From d68e564961ae3797f021e598fb52779a6e7e7cc4 Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 28 Jul 2026 04:48:28 +0300 Subject: [PATCH 001/320] feat(azure.ai.evaluations): scaffold the Foundry evaluations extension New azd extension exposing azd ai eval, registering the azure.ai.eval service-target provider. Scaffold only: manifest, entrypoint, root command. --- .../azure.ai.evaluations/extension.yaml | 29 ++ .../extensions/azure.ai.evaluations/go.mod | 104 ++++++ .../extensions/azure.ai.evaluations/go.sum | 314 ++++++++++++++++++ .../azure.ai.evaluations/internal/cmd/root.go | 29 ++ .../extensions/azure.ai.evaluations/main.go | 14 + .../azure.ai.evaluations/version.txt | 1 + 6 files changed, 491 insertions(+) create mode 100644 cli/azd/extensions/azure.ai.evaluations/extension.yaml create mode 100644 cli/azd/extensions/azure.ai.evaluations/go.mod create mode 100644 cli/azd/extensions/azure.ai.evaluations/go.sum create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/root.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/main.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/version.txt diff --git a/cli/azd/extensions/azure.ai.evaluations/extension.yaml b/cli/azd/extensions/azure.ai.evaluations/extension.yaml new file mode 100644 index 00000000000..2b8e8bbb58f --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/extension.yaml @@ -0,0 +1,29 @@ +# yaml-language-server: $schema=../extension.schema.json +id: azure.ai.evaluations +namespace: ai.eval +displayName: Foundry evaluations (Beta) +description: Define and run Foundry evaluations from your terminal. (Beta) +usage: azd ai eval [options] +# NOTE: Make sure version.txt is in sync with this version. +version: 1.0.0-beta.1 +requiredAzdVersion: ">=1.27.1" +dependencies: + - id: azure.ai.projects + version: "~1.0.0-beta.3" +language: go +capabilities: + - custom-commands + - lifecycle-events + - service-target-provider + - metadata +providers: + - name: azure.ai.eval + type: service-target + description: Deploys evaluation datasets, evaluators, and eval groups to Foundry +examples: + - name: init + description: Scaffold evaluation config for an agent. + usage: azd ai eval init + - name: run + description: Run an evaluation and summarize the results. + usage: azd ai eval run diff --git a/cli/azd/extensions/azure.ai.evaluations/go.mod b/cli/azd/extensions/azure.ai.evaluations/go.mod new file mode 100644 index 00000000000..d1099806c54 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/go.mod @@ -0,0 +1,104 @@ +module azureaieval + +go 1.26.4 + +require ( + github.com/azure/azure-dev/cli/azd v1.28.0 + github.com/fatih/color v1.18.0 + github.com/spf13/cobra v1.10.1 +) + +require ( + dario.cat/mergo v1.0.2 // indirect + github.com/AlecAivazis/survey/v2 v2.3.7 // indirect + github.com/Azure/azure-sdk-for-go/sdk/azcore v1.21.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.14.0-beta.3 // indirect + github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2 // indirect + github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/keyvault/armkeyvault v1.5.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armsubscriptions v1.3.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azsecrets v1.4.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/internal v1.2.0 // indirect + github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0 // indirect + github.com/Masterminds/semver/v3 v3.4.0 // indirect + github.com/adam-lavrik/go-imath v0.0.0-20210910152346-265a42a96f0b // indirect + github.com/alecthomas/chroma/v2 v2.20.0 // indirect + github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect + github.com/aymerick/douceur v0.2.0 // indirect + github.com/bahlo/generic-list-go v0.2.0 // indirect + github.com/blang/semver/v4 v4.0.0 // indirect + github.com/braydonk/yaml v0.9.0 // indirect + github.com/buger/goterm v1.0.4 // indirect + github.com/buger/jsonparser v1.1.2 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/charmbracelet/colorprofile v0.3.2 // indirect + github.com/charmbracelet/glamour v0.10.0 // indirect + github.com/charmbracelet/lipgloss v1.1.1-0.20250404203927-76690c660834 // indirect + github.com/charmbracelet/x/ansi v0.10.2 // indirect + github.com/charmbracelet/x/cellbuf v0.0.13 // indirect + github.com/charmbracelet/x/exp/slice v0.0.0-20251008171431-5d3777519489 // indirect + github.com/charmbracelet/x/term v0.2.1 // indirect + github.com/cli/browser v1.3.0 // indirect + github.com/clipperhouse/uax29/v2 v2.2.0 // indirect + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + github.com/dlclark/regexp2 v1.11.5 // indirect + github.com/drone/envsubst v1.0.3 // indirect + github.com/go-logr/logr v1.4.3 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/gofrs/flock v0.12.1 // indirect + github.com/golang-jwt/jwt/v5 v5.3.0 // indirect + github.com/golobby/container/v3 v3.3.2 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/gorilla/css v1.0.1 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/invopop/jsonschema v0.13.0 // indirect + github.com/jmespath-community/go-jmespath v1.1.1 // indirect + github.com/joho/godotenv v1.5.1 // indirect + github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect + github.com/kylelemons/godebug v1.1.0 // indirect + github.com/lucasb-eyer/go-colorful v1.3.0 // indirect + github.com/mailru/easyjson v0.9.1 // indirect + github.com/mark3labs/mcp-go v0.41.1 // indirect + github.com/mattn/go-colorable v0.1.14 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect + github.com/mattn/go-runewidth v0.0.19 // indirect + github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d // indirect + github.com/microcosm-cc/bluemonday v1.0.27 // indirect + github.com/microsoft/ApplicationInsights-Go v0.4.4 // indirect + github.com/microsoft/go-deviceid v1.0.0 // indirect + github.com/muesli/reflow v0.3.0 // indirect + github.com/muesli/termenv v0.16.0 // indirect + github.com/nathan-fiscaletti/consolesize-go v0.0.0-20220204101620-317176b6684d // indirect + github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect + github.com/rivo/uniseg v0.4.7 // indirect + github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 // indirect + github.com/sethvargo/go-retry v0.3.0 // indirect + github.com/spf13/cast v1.10.0 // indirect + github.com/spf13/pflag v1.0.10 // indirect + github.com/stretchr/testify v1.11.1 // indirect + github.com/theckman/yacspin v0.13.12 // indirect + github.com/wk8/go-ordered-map/v2 v2.1.8 // indirect + github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect + github.com/yosida95/uritemplate/v3 v3.0.2 // indirect + github.com/yuin/goldmark v1.7.13 // indirect + github.com/yuin/goldmark-emoji v1.0.6 // indirect + go.opentelemetry.io/auto/sdk v1.2.1 // indirect + go.opentelemetry.io/otel v1.43.0 // indirect + go.opentelemetry.io/otel/metric v1.43.0 // indirect + go.opentelemetry.io/otel/sdk v1.43.0 // indirect + go.opentelemetry.io/otel/trace v1.43.0 // indirect + go.uber.org/atomic v1.11.0 // indirect + go.uber.org/multierr v1.11.0 // indirect + golang.org/x/crypto v0.53.0 // indirect + golang.org/x/exp v0.0.0-20250911091902-df9299821621 // indirect + golang.org/x/net v0.56.0 // indirect + golang.org/x/sync v0.21.0 // indirect + golang.org/x/sys v0.46.0 // indirect + golang.org/x/term v0.44.0 // indirect + golang.org/x/text v0.38.0 // indirect + golang.org/x/time v0.9.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260401024825-9d38bb4040a9 // indirect + google.golang.org/grpc v1.80.0 // indirect + google.golang.org/protobuf v1.36.11 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/cli/azd/extensions/azure.ai.evaluations/go.sum b/cli/azd/extensions/azure.ai.evaluations/go.sum new file mode 100644 index 00000000000..6d0434c4767 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/go.sum @@ -0,0 +1,314 @@ +code.cloudfoundry.org/clock v0.0.0-20180518195852-02e53af36e6c/go.mod h1:QD9Lzhd/ux6eNQVUDVRJX/RKTigpewimNYBi7ivZKY8= +dario.cat/mergo v1.0.2 h1:85+piFYR1tMbRrLcDwR18y4UKJ3aH1Tbzi24VRW1TK8= +dario.cat/mergo v1.0.2/go.mod h1:E/hbnu0NxMFBjpMIE34DRGLWqDy0g5FuKDhCb31ngxA= +github.com/AlecAivazis/survey/v2 v2.3.7 h1:6I/u8FvytdGsgonrYsVn2t8t4QiRnh6QSTqkkhIiSjQ= +github.com/AlecAivazis/survey/v2 v2.3.7/go.mod h1:xUTIdE4KCOIjsBAE1JYsUPoCqYdZ1reCfTwbto0Fduo= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.21.0 h1:fou+2+WFTib47nS+nz/ozhEBnvU96bKHy6LjRsY4E28= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.21.0/go.mod h1:t76Ruy8AHvUAC8GfMWJMa0ElSbuIcO03NLpynfbgsPA= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.14.0-beta.3 h1:0g4UTtvRA9goC37cmD9ZHdW6CCNJR4cOXBnHz0r4ubM= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.14.0-beta.3/go.mod h1:fEiHi0sbYqbo3shUkIF1SNxm8GyeEJl+Poc/djOvbdE= +github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.3.2 h1:yz1bePFlP5Vws5+8ez6T3HWXPmwOK7Yvq8QxDBD3SKY= +github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.3.2/go.mod h1:Pa9ZNPuoNu/GztvBSKk9J1cDJW6vk/n0zLtV4mgd8N8= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2 h1:9iefClla7iYpfYWdzPCRDozdmndjTm8DXdpCzPajMgA= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2/go.mod h1:XtLgD3ZD34DAaVIIAyG3objl5DynM3CQ/vMcbBNJZGI= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/internal/v2 v2.0.0 h1:PTFGRSlMKCQelWwxUyYVEUqseBJVemLyqWJjvMyt0do= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/internal/v2 v2.0.0/go.mod h1:LRr2FzBTQlONPPa5HREE5+RjSCTXl7BwOvYOaWTqCaI= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/internal/v3 v3.1.0 h1:2qsIIvxVT+uE6yrNldntJKlLRgxGbZ85kgtz5SNBhMw= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/internal/v3 v3.1.0/go.mod h1:AW8VEadnhw9xox+VaVd9sP7NjzOAnaZBLRH6Tq3cJ38= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/keyvault/armkeyvault v1.5.0 h1:nnQ9vXH039UrEFxi08pPuZBE7VfqSJt343uJLw0rhWI= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/keyvault/armkeyvault v1.5.0/go.mod h1:4YIVtzMFVsPwBvitCDX7J9sqthSj43QD1sP6fYc1egc= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.2.0 h1:Dd+RhdJn0OTtVGaeDLZpcumkIVCtA/3/Fo42+eoYvVM= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.2.0/go.mod h1:5kakwfW5CjC9KK+Q4wjXAg+ShuIm2mBMua0ZFj2C8PE= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armsubscriptions v1.3.0 h1:wxQx2Bt4xzPIKvW59WQf1tJNx/ZZKPfN+EhPX3Z6CYY= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armsubscriptions v1.3.0/go.mod h1:TpiwjwnW/khS0LKs4vW5UmmT9OWcxaveS8U7+tlknzo= +github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azsecrets v1.4.0 h1:/g8S6wk65vfC6m3FIxJ+i5QDyN9JWwXI8Hb0Img10hU= +github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azsecrets v1.4.0/go.mod h1:gpl+q95AzZlKVI3xSoseF9QPrypk0hQqBiJYeB/cR/I= +github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/internal v1.2.0 h1:nCYfgcSyHZXJI8J0IWE5MsCGlb2xp9fJiXyxWgmOFg4= +github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/internal v1.2.0/go.mod h1:ucUjca2JtSZboY8IoUqyQyuuXvwbMBVwFOm0vdQPNhA= +github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1 h1:WJTmL004Abzc5wDB5VtZG2PJk5ndYDgVacGqfirKxjM= +github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1/go.mod h1:tCcJZ0uHAmvjsVYzEFivsRTN00oz5BEsRgQHu5JZ9WE= +github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0 h1:XRzhVemXdgvJqCH0sFfrBUTnUJSBrBf7++ypk+twtRs= +github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0/go.mod h1:HKpQxkWaGLJ+D/5H8QRpyQXA1eKjxkFlOMwck5+33Jk= +github.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1Xbatp0= +github.com/Masterminds/semver/v3 v3.4.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= +github.com/Netflix/go-expect v0.0.0-20220104043353-73e0943537d2 h1:+vx7roKuyA63nhn5WAunQHLTznkw5W8b1Xc0dNjp83s= +github.com/Netflix/go-expect v0.0.0-20220104043353-73e0943537d2/go.mod h1:HBCaDeC1lPdgDeDbhX8XFpy1jqjK0IBG8W5K+xYqA0w= +github.com/adam-lavrik/go-imath v0.0.0-20210910152346-265a42a96f0b h1:g9SuFmxM/WucQFKTMSP+irxyf5m0RiUJreBDhGI6jSA= +github.com/adam-lavrik/go-imath v0.0.0-20210910152346-265a42a96f0b/go.mod h1:XjvqMUpGd3Xn9Jtzk/4GEBCSoBX0eB2RyriXgne0IdM= +github.com/alecthomas/assert/v2 v2.11.0 h1:2Q9r3ki8+JYXvGsDyBXwH3LcJ+WK5D0gc5E8vS6K3D0= +github.com/alecthomas/assert/v2 v2.11.0/go.mod h1:Bze95FyfUr7x34QZrjL+XP+0qgp/zg8yS+TtBj1WA3k= +github.com/alecthomas/chroma/v2 v2.20.0 h1:sfIHpxPyR07/Oylvmcai3X/exDlE8+FA820NTz+9sGw= +github.com/alecthomas/chroma/v2 v2.20.0/go.mod h1:e7tViK0xh/Nf4BYHl00ycY6rV7b8iXBksI9E359yNmA= +github.com/alecthomas/repr v0.5.1 h1:E3G4t2QbHTSNpPKBgMTln5KLkZHLOcU7r37J4pXBuIg= +github.com/alecthomas/repr v0.5.1/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4= +github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k= +github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8= +github.com/aymanbagabas/go-udiff v0.2.0 h1:TK0fH4MteXUDspT88n8CKzvK0X9O2xu9yQjWpi6yML8= +github.com/aymanbagabas/go-udiff v0.2.0/go.mod h1:RE4Ex0qsGkTAJoQdQQCA0uG+nAzJO/pI/QwceO5fgrA= +github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= +github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4= +github.com/azure/azure-dev/cli/azd v1.28.0 h1:mqqyV85m7A1XfWJFjV/Ut0QoIEImFeF++1Ruq/cRp0s= +github.com/azure/azure-dev/cli/azd v1.28.0/go.mod h1:Ge7QaU9PoJM7i6J0xArDoQCf2tUn6O7OIKkoItxFTA8= +github.com/bahlo/generic-list-go v0.2.0 h1:5sz/EEAK+ls5wF+NeqDpk5+iNdMDXrh3z3nPnH1Wvgk= +github.com/bahlo/generic-list-go v0.2.0/go.mod h1:2KvAjgMlE5NNynlg/5iLrrCCZ2+5xWbdbCW3pNTGyYg= +github.com/benbjohnson/clock v1.3.5 h1:VvXlSJBzZpA/zum6Sj74hxwYI2DIxRWuNIoXAzHZz5o= +github.com/benbjohnson/clock v1.3.5/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= +github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= +github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= +github.com/bradleyjkemp/cupaloy/v2 v2.8.0 h1:any4BmKE+jGIaMpnU8YgH/I2LPiLBufr6oMMlVBbn9M= +github.com/bradleyjkemp/cupaloy/v2 v2.8.0/go.mod h1:bm7JXdkRd4BHJk9HpwqAI8BoAY1lps46Enkdqw6aRX0= +github.com/braydonk/yaml v0.9.0 h1:ewGMrVmEVpsm3VwXQDR388sLg5+aQ8Yihp6/hc4m+h4= +github.com/braydonk/yaml v0.9.0/go.mod h1:hcm3h581tudlirk8XEUPDBAimBPbmnL0Y45hCRl47N4= +github.com/buger/goterm v1.0.4 h1:Z9YvGmOih81P0FbVtEYTFF6YsSgxSUKEhf/f9bTMXbY= +github.com/buger/goterm v1.0.4/go.mod h1:HiFWV3xnkolgrBV3mY8m0X0Pumt4zg4QhbdOzQtB8tE= +github.com/buger/jsonparser v1.1.2 h1:frqHqw7otoVbk5M8LlE/L7HTnIq2v9RX6EJ48i9AxJk= +github.com/buger/jsonparser v1.1.2/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/charmbracelet/colorprofile v0.3.2 h1:9J27WdztfJQVAQKX2WOlSSRB+5gaKqqITmrvb1uTIiI= +github.com/charmbracelet/colorprofile v0.3.2/go.mod h1:mTD5XzNeWHj8oqHb+S1bssQb7vIHbepiebQ2kPKVKbI= +github.com/charmbracelet/glamour v0.10.0 h1:MtZvfwsYCx8jEPFJm3rIBFIMZUfUJ765oX8V6kXldcY= +github.com/charmbracelet/glamour v0.10.0/go.mod h1:f+uf+I/ChNmqo087elLnVdCiVgjSKWuXa/l6NU2ndYk= +github.com/charmbracelet/lipgloss v1.1.1-0.20250404203927-76690c660834 h1:ZR7e0ro+SZZiIZD7msJyA+NjkCNNavuiPBLgerbOziE= +github.com/charmbracelet/lipgloss v1.1.1-0.20250404203927-76690c660834/go.mod h1:aKC/t2arECF6rNOnaKaVU6y4t4ZeHQzqfxedE/VkVhA= +github.com/charmbracelet/x/ansi v0.10.2 h1:ith2ArZS0CJG30cIUfID1LXN7ZFXRCww6RUvAPA+Pzw= +github.com/charmbracelet/x/ansi v0.10.2/go.mod h1:HbLdJjQH4UH4AqA2HpRWuWNluRE6zxJH/yteYEYCFa8= +github.com/charmbracelet/x/cellbuf v0.0.13 h1:/KBBKHuVRbq1lYx5BzEHBAFBP8VcQzJejZ/IA3iR28k= +github.com/charmbracelet/x/cellbuf v0.0.13/go.mod h1:xe0nKWGd3eJgtqZRaN9RjMtK7xUYchjzPr7q6kcvCCs= +github.com/charmbracelet/x/exp/golden v0.0.0-20240806155701-69247e0abc2a h1:G99klV19u0QnhiizODirwVksQB91TJKV/UaTnACcG30= +github.com/charmbracelet/x/exp/golden v0.0.0-20240806155701-69247e0abc2a/go.mod h1:wDlXFlCrmJ8J+swcL/MnGUuYnqgQdW9rhSD61oNMb6U= +github.com/charmbracelet/x/exp/slice v0.0.0-20251008171431-5d3777519489 h1:a5q2sWiet6kgqucSGjYN1jhT2cn4bMKUwprtm2IGRto= +github.com/charmbracelet/x/exp/slice v0.0.0-20251008171431-5d3777519489/go.mod h1:vqEfX6xzqW1pKKZUUiFOKg0OQ7bCh54Q2vR/tserrRA= +github.com/charmbracelet/x/term v0.2.1 h1:AQeHeLZ1OqSXhrAWpYUtZyX1T3zVxfpZuEQMIQaGIAQ= +github.com/charmbracelet/x/term v0.2.1/go.mod h1:oQ4enTYFV7QN4m0i9mzHrViD7TQKvNEEkHUMCmsxdUg= +github.com/cli/browser v1.3.0 h1:LejqCrpWr+1pRqmEPDGnTZOjsMe7sehifLynZJuqJpo= +github.com/cli/browser v1.3.0/go.mod h1:HH8s+fOAxjhQoBUAsKuPCbqUuxZDhQ2/aD+SzsEfBTk= +github.com/clipperhouse/uax29/v2 v2.2.0 h1:ChwIKnQN3kcZteTXMgb1wztSgaU+ZemkgWdohwgs8tY= +github.com/clipperhouse/uax29/v2 v2.2.0/go.mod h1:EFJ2TJMRUaplDxHKj1qAEhCtQPW2tJSwu5BF98AuoVM= +github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= +github.com/creack/pty v1.1.17 h1:QeVUsEDNrLBW4tMgZHvxy18sKtr6VI492kBhUfhDJNI= +github.com/creack/pty v1.1.17/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dlclark/regexp2 v1.11.5 h1:Q/sSnsKerHeCkc/jSTNq1oCm7KiVgUMZRDUoRu0JQZQ= +github.com/dlclark/regexp2 v1.11.5/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= +github.com/drone/envsubst v1.0.3 h1:PCIBwNDYjs50AsLZPYdfhSATKaRg/FJmDc2D6+C2x8g= +github.com/drone/envsubst v1.0.3/go.mod h1:N2jZmlMufstn1KEqvbHjw40h1KyTmnVzHcSc9bFiJ2g= +github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM= +github.com/fatih/color v1.18.0/go.mod h1:4FelSpRwEGDpQ12mAdzqdOukCy4u8WUtOY6lkT/6HfU= +github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= +github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/gofrs/flock v0.12.1 h1:MTLVXXHf8ekldpJk3AKicLij9MdwOWkZ+a/jHHZby9E= +github.com/gofrs/flock v0.12.1/go.mod h1:9zxTsyu5xtJ9DK+1tFZyibEV7y3uwDxPPfbxeeHCoD0= +github.com/gofrs/uuid v3.3.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= +github.com/golang-jwt/jwt/v5 v5.3.0 h1:pv4AsKCKKZuqlgs5sUmn4x8UlGa0kEVt/puTpKx9vvo= +github.com/golang-jwt/jwt/v5 v5.3.0/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/golobby/container/v3 v3.3.2 h1:7u+RgNnsdVlhGoS8gY4EXAG601vpMMzLZlYqSp77Quw= +github.com/golobby/container/v3 v3.3.2/go.mod h1:RDdKpnKpV1Of11PFBe7Dxc2C1k2KaLE4FD47FflAmj0= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/gorilla/css v1.0.1 h1:ntNaBIghp6JmvWnxbZKANoLyuXTPZ4cAMlo6RyhlbO8= +github.com/gorilla/css v1.0.1/go.mod h1:BvnYkspnSzMmwRK+b8/xgNPLiIuNZr6vbZBTPQ2A3b0= +github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM= +github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg= +github.com/hinshun/vt10x v0.0.0-20220119200601-820417d04eec h1:qv2VnGeEQHchGaZ/u7lxST/RaJw+cv273q79D81Xbog= +github.com/hinshun/vt10x v0.0.0-20220119200601-820417d04eec/go.mod h1:Q48J4R4DvxnHolD5P8pOtXigYlRuPLGl6moFx3ulM68= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/invopop/jsonschema v0.13.0 h1:KvpoAJWEjR3uD9Kbm2HWJmqsEaHt8lBUpd0qHcIi21E= +github.com/invopop/jsonschema v0.13.0/go.mod h1:ffZ5Km5SWWRAIN6wbDXItl95euhFz2uON45H2qjYt+0= +github.com/jmespath-community/go-jmespath v1.1.1 h1:bFikPhsi/FdmlZhVgSCd2jj1e7G/rw+zyQfyg5UF+L4= +github.com/jmespath-community/go-jmespath v1.1.1/go.mod h1:4gOyFJsR/Gk+05RgTKYrifT7tBPWD8Lubtb5jRrfy9I= +github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= +github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= +github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs= +github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8= +github.com/keybase/go-keychain v0.0.1 h1:way+bWYa6lDppZoZcgMbYsvC7GxljxrskdNInRtuthU= +github.com/keybase/go-keychain v0.0.1/go.mod h1:PdEILRW3i9D8JcdM+FmY6RwkHGnhHxXwkPPMeUgOK1k= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/lucasb-eyer/go-colorful v1.3.0 h1:2/yBRLdWBZKrf7gB40FoiKfAWYQ0lqNcbuQwVHXptag= +github.com/lucasb-eyer/go-colorful v1.3.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= +github.com/mailru/easyjson v0.9.1 h1:LbtsOm5WAswyWbvTEOqhypdPeZzHavpZx96/n553mR8= +github.com/mailru/easyjson v0.9.1/go.mod h1:1+xMtQp2MRNVL/V1bOzuP3aP8VNwRW55fQUto+XFtTU= +github.com/mark3labs/mcp-go v0.41.1 h1:w78eWfiQam2i8ICL7AL0WFiq7KHNJQ6UB53ZVtH4KGA= +github.com/mark3labs/mcp-go v0.41.1/go.mod h1:T7tUa2jO6MavG+3P25Oy/jR7iCeJPHImCZHRymCn39g= +github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE= +github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= +github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-runewidth v0.0.12/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk= +github.com/mattn/go-runewidth v0.0.19 h1:v++JhqYnZuu5jSKrk9RbgF5v4CGUjqRfBm05byFGLdw= +github.com/mattn/go-runewidth v0.0.19/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhgLpndooCuJAs= +github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= +github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d h1:5PJl274Y63IEHC+7izoQE9x6ikvDFZS2mDVS3drnohI= +github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= +github.com/microcosm-cc/bluemonday v1.0.27 h1:MpEUotklkwCSLeH+Qdx1VJgNqLlpY2KXwXFM08ygZfk= +github.com/microcosm-cc/bluemonday v1.0.27/go.mod h1:jFi9vgW+H7c3V0lb6nR74Ib/DIB5OBs92Dimizgw2cA= +github.com/microsoft/ApplicationInsights-Go v0.4.4 h1:G4+H9WNs6ygSCe6sUyxRc2U81TI5Es90b2t/MwX5KqY= +github.com/microsoft/ApplicationInsights-Go v0.4.4/go.mod h1:fKRUseBqkw6bDiXTs3ESTiU/4YTIHsQS4W3fP2ieF4U= +github.com/microsoft/go-deviceid v1.0.0 h1:i5AQ654Xk9kfvwJeKQm3w2+eT1+ImBDVEpAR0AjpP40= +github.com/microsoft/go-deviceid v1.0.0/go.mod h1:KY13FeVdHkzD8gy+6T8+kVmD/7RMpTaWW75K+T4uZWg= +github.com/muesli/reflow v0.3.0 h1:IFsN6K9NfGtjeggFP+68I4chLZV2yIKsXJFNZ+eWh6s= +github.com/muesli/reflow v0.3.0/go.mod h1:pbwTDkVPibjO2kyvBQRBxTWEEGDGq0FlB1BIKtnHY/8= +github.com/muesli/termenv v0.16.0 h1:S5AlUN9dENB57rsbnkPyfdGuWIlkmzJjbFf0Tf5FWUc= +github.com/muesli/termenv v0.16.0/go.mod h1:ZRfOIKPFDYQoDFF4Olj7/QJbW60Ol/kL1pU3VfY/Cnk= +github.com/nathan-fiscaletti/consolesize-go v0.0.0-20220204101620-317176b6684d h1:NqRhLdNVlozULwM1B3VaHhcXYSgrOAv8V5BE65om+1Q= +github.com/nathan-fiscaletti/consolesize-go v0.0.0-20220204101620-317176b6684d/go.mod h1:cxIIfNMTwff8f/ZvRouvWYF6wOoO7nj99neWSx2q/Es= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ= +github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= +github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= +github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= +github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= +github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= +github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 h1:KRzFb2m7YtdldCEkzs6KqmJw4nqEVZGK7IN2kJkjTuQ= +github.com/santhosh-tekuri/jsonschema/v6 v6.0.2/go.mod h1:JXeL+ps8p7/KNMjDQk3TCwPpBy0wYklyWTfbkIzdIFU= +github.com/sethvargo/go-retry v0.3.0 h1:EEt31A35QhrcRZtrYFDTBg91cqZVnFL2navjDrah2SE= +github.com/sethvargo/go-retry v0.3.0/go.mod h1:mNX17F0C/HguQMyMyJxcnU471gOZGxCLyYaFyAZraas= +github.com/spf13/cast v1.10.0 h1:h2x0u2shc1QuLHfxi+cTJvs30+ZAHOGRic8uyGTDWxY= +github.com/spf13/cast v1.10.0/go.mod h1:jNfB8QC9IA6ZuY2ZjDp0KtFO2LZZlg4S/7bzP6qqeHo= +github.com/spf13/cobra v1.10.1 h1:lJeBwCfmrnXthfAupyUTzJ/J4Nc1RsHC/mSRU2dll/s= +github.com/spf13/cobra v1.10.1/go.mod h1:7SmJGaTHFVBY0jW4NXGluQoLvhqFQM+6XSKD+P4XaB0= +github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk= +github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/tedsuo/ifrit v0.0.0-20180802180643-bea94bb476cc/go.mod h1:eyZnKCc955uh98WQvzOm0dgAeLnf2O0Rz0LPoC5ze+0= +github.com/theckman/yacspin v0.13.12 h1:CdZ57+n0U6JMuh2xqjnjRq5Haj6v1ner2djtLQRzJr4= +github.com/theckman/yacspin v0.13.12/go.mod h1:Rd2+oG2LmQi5f3zC3yeZAOl245z8QOvrH4OPOJNZxLg= +github.com/wk8/go-ordered-map/v2 v2.1.8 h1:5h/BUHu93oj4gIdvHHHGsScSTMijfx5PeYkE/fJgbpc= +github.com/wk8/go-ordered-map/v2 v2.1.8/go.mod h1:5nJHM5DyteebpVlHnWMV0rPz6Zp7+xBAnxjb1X5vnTw= +github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavMF/ppJZNG9ZpyihvCd0w101no= +github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJuqunuUZ/Dhy/avygyECGrLceyNeo4LiM= +github.com/yosida95/uritemplate/v3 v3.0.2 h1:Ed3Oyj9yrmi9087+NczuL5BwkIc4wvTb5zIM+UJPGz4= +github.com/yosida95/uritemplate/v3 v3.0.2/go.mod h1:ILOh0sOhIJR3+L/8afwt/kE++YT040gmv5BQTMR2HP4= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +github.com/yuin/goldmark v1.7.13 h1:GPddIs617DnBLFFVJFgpo1aBfe/4xcvMc3SB5t/D0pA= +github.com/yuin/goldmark v1.7.13/go.mod h1:ip/1k0VRfGynBgxOz0yCqHrbZXhcjxyuS66Brc7iBKg= +github.com/yuin/goldmark-emoji v1.0.6 h1:QWfF2FYaXwL74tfGOW5izeiZepUDroDJfWubQI9HTHs= +github.com/yuin/goldmark-emoji v1.0.6/go.mod h1:ukxJDKFpdFb5x0a5HqbdlcKtebh086iJpI31LTKmWuA= +go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= +go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= +go.opentelemetry.io/otel v1.43.0 h1:mYIM03dnh5zfN7HautFE4ieIig9amkNANT+xcVxAj9I= +go.opentelemetry.io/otel v1.43.0/go.mod h1:JuG+u74mvjvcm8vj8pI5XiHy1zDeoCS2LB1spIq7Ay0= +go.opentelemetry.io/otel/metric v1.43.0 h1:d7638QeInOnuwOONPp4JAOGfbCEpYb+K6DVWvdxGzgM= +go.opentelemetry.io/otel/metric v1.43.0/go.mod h1:RDnPtIxvqlgO8GRW18W6Z/4P462ldprJtfxHxyKd2PY= +go.opentelemetry.io/otel/sdk v1.43.0 h1:pi5mE86i5rTeLXqoF/hhiBtUNcrAGHLKQdhg4h4V9Dg= +go.opentelemetry.io/otel/sdk v1.43.0/go.mod h1:P+IkVU3iWukmiit/Yf9AWvpyRDlUeBaRg6Y+C58QHzg= +go.opentelemetry.io/otel/sdk/metric v1.43.0 h1:S88dyqXjJkuBNLeMcVPRFXpRw2fuwdvfCGLEo89fDkw= +go.opentelemetry.io/otel/sdk/metric v1.43.0/go.mod h1:C/RJtwSEJ5hzTiUz5pXF1kILHStzb9zFlIEe85bhj6A= +go.opentelemetry.io/otel/trace v1.43.0 h1:BkNrHpup+4k4w+ZZ86CZoHHEkohws8AY+WTX09nk+3A= +go.opentelemetry.io/otel/trace v1.43.0/go.mod h1:/QJhyVBUUswCphDVxq+8mld+AvhXZLhe+8WVFxiFff0= +go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= +go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.53.0 h1:QZ4Muo8THX6CizN2vPPd5fBGHyogrdK9fG4wLPFUsto= +golang.org/x/crypto v0.53.0/go.mod h1:DNLU434OwVakk9PzuwV8w62mAJpRJL3vsgcfp4Qnsio= +golang.org/x/exp v0.0.0-20250911091902-df9299821621 h1:2id6c1/gto0kaHYyrixvknJ8tUK/Qs5IsmBtrc+FtgU= +golang.org/x/exp v0.0.0-20250911091902-df9299821621/go.mod h1:TwQYMMnGpvZyc+JpB/UAuTNIsVJifOlSkrZkhcvpVUk= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.56.0 h1:Rw8j/hFzGvJUZwNBXnAtf5sVDVt+65SK2C7IxCxZt5o= +golang.org/x/net v0.56.0/go.mod h1:D3Ku6r+V6JROoZK144D2XfMHFcMq/0zSfLelVTCFKec= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM= +golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210331175145-43e1dd70ce54/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw= +golang.org/x/sys v0.46.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.44.0 h1:0rLvDRCtNj0gZkyIXhCyOb2OAzEhLVqc4B+hrsBhrmc= +golang.org/x/term v0.44.0/go.mod h1:7ze4MdzUzLXpSAoFP1H0bOI9aXDqveSvatT5vKcFh2Y= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.38.0 h1:sXmwo9DwP3OK9EZ7PqAdaooSGozfl/3a6/xJcbzPRhE= +golang.org/x/text v0.38.0/go.mod h1:YXZt3QhHUKYT53r2lLKFIVi6Ao1jdzrTR/KQ09qyxF4= +golang.org/x/time v0.9.0 h1:EsRrnYcQiGH+5FfbgvV4AP7qEZstoyrHB0DzarOQ4ZY= +golang.org/x/time v0.9.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= +gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260401024825-9d38bb4040a9 h1:m8qni9SQFH0tJc1X0vmnpw/0t+AImlSvp30sEupozUg= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260401024825-9d38bb4040a9/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= +google.golang.org/grpc v1.80.0 h1:Xr6m2WmWZLETvUNvIUmeD5OAagMw3FiKmMlTdViWsHM= +google.golang.org/grpc v1.80.0/go.mod h1:ho/dLnxwi3EDJA4Zghp7k2Ec1+c2jqup0bFkw07bwF4= +google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= +google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/root.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/root.go new file mode 100644 index 00000000000..6654435f055 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/root.go @@ -0,0 +1,29 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "fmt" + + "github.com/azure/azure-dev/cli/azd/pkg/azdext" + "github.com/fatih/color" + "github.com/spf13/cobra" +) + +// NewRootCommand builds the `azd ai eval` command tree. +func NewRootCommand() *cobra.Command { + rootCmd, _ := azdext.NewExtensionRootCommand(azdext.ExtensionCommandOptions{ + Name: "eval", + Use: "eval [options]", + Short: fmt.Sprintf( + "Define and run Foundry evaluations from your terminal. %s", + color.YellowString("(Beta)"), + ), + }) + rootCmd.SilenceUsage = true + rootCmd.SilenceErrors = true + rootCmd.CompletionOptions.DisableDefaultCmd = true + + return rootCmd +} diff --git a/cli/azd/extensions/azure.ai.evaluations/main.go b/cli/azd/extensions/azure.ai.evaluations/main.go new file mode 100644 index 00000000000..993d2e8816e --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/main.go @@ -0,0 +1,14 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package main + +import ( + "azureaieval/internal/cmd" + + "github.com/azure/azure-dev/cli/azd/pkg/azdext" +) + +func main() { + azdext.Run(cmd.NewRootCommand()) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/version.txt b/cli/azd/extensions/azure.ai.evaluations/version.txt new file mode 100644 index 00000000000..ffbc9939864 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/version.txt @@ -0,0 +1 @@ +1.0.0-beta.1 From a9607265b07dc9fd8ee20d3d4f03619730b62ee5 Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 28 Jul 2026 04:54:49 +0300 Subject: [PATCH 002/320] feat(azure.ai.evaluations): port the eval and dataset data-plane clients Lifted eval_api (models, operations, poller, generation, portal_urls) and dataset_api from azure.ai.agents, de-agent-scoped. Added evalcore with IsTransientError and an EvaluatorList that accepts either a bare string or a mapping with a threshold. Skipped artifacts.go and eval_config.go since the config model differs. --- .../extensions/azure.ai.evaluations/go.mod | 6 +- .../extensions/azure.ai.evaluations/go.sum | 4 + .../internal/pkg/dataset_api/models.go | 192 ++++++++ .../internal/pkg/dataset_api/operations.go | 442 ++++++++++++++++++ .../internal/pkg/eval_api/generation.go | 140 ++++++ .../internal/pkg/eval_api/models.go | 412 ++++++++++++++++ .../internal/pkg/eval_api/operations.go | 290 ++++++++++++ .../internal/pkg/eval_api/poller.go | 195 ++++++++ .../internal/pkg/eval_api/portal_urls.go | 79 ++++ .../internal/pkg/evalcore/evaluator.go | 89 ++++ .../internal/pkg/evalcore/transient.go | 29 ++ .../internal/version/version.go | 11 + 12 files changed, 1887 insertions(+), 2 deletions(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/models.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/operations.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/generation.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/models.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/operations.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/poller.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/portal_urls.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/evaluator.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/transient.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/version/version.go diff --git a/cli/azd/extensions/azure.ai.evaluations/go.mod b/cli/azd/extensions/azure.ai.evaluations/go.mod index d1099806c54..c63d3067dfb 100644 --- a/cli/azd/extensions/azure.ai.evaluations/go.mod +++ b/cli/azd/extensions/azure.ai.evaluations/go.mod @@ -3,17 +3,20 @@ module azureaieval go 1.26.4 require ( + github.com/Azure/azure-sdk-for-go/sdk/azcore v1.21.0 github.com/azure/azure-dev/cli/azd v1.28.0 github.com/fatih/color v1.18.0 + github.com/google/uuid v1.6.0 github.com/spf13/cobra v1.10.1 + go.yaml.in/yaml/v3 v3.0.4 ) require ( dario.cat/mergo v1.0.2 // indirect github.com/AlecAivazis/survey/v2 v2.3.7 // indirect - github.com/Azure/azure-sdk-for-go/sdk/azcore v1.21.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.14.0-beta.3 // indirect github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2 // indirect + github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/appservice/armappservice/v2 v2.3.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/keyvault/armkeyvault v1.5.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armsubscriptions v1.3.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azsecrets v1.4.0 // indirect @@ -47,7 +50,6 @@ require ( github.com/gofrs/flock v0.12.1 // indirect github.com/golang-jwt/jwt/v5 v5.3.0 // indirect github.com/golobby/container/v3 v3.3.2 // indirect - github.com/google/uuid v1.6.0 // indirect github.com/gorilla/css v1.0.1 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/invopop/jsonschema v0.13.0 // indirect diff --git a/cli/azd/extensions/azure.ai.evaluations/go.sum b/cli/azd/extensions/azure.ai.evaluations/go.sum index 6d0434c4767..81c10e45793 100644 --- a/cli/azd/extensions/azure.ai.evaluations/go.sum +++ b/cli/azd/extensions/azure.ai.evaluations/go.sum @@ -11,6 +11,8 @@ github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.3.2 h1:yz1bePFlP5Vws5+ github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.3.2/go.mod h1:Pa9ZNPuoNu/GztvBSKk9J1cDJW6vk/n0zLtV4mgd8N8= github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2 h1:9iefClla7iYpfYWdzPCRDozdmndjTm8DXdpCzPajMgA= github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2/go.mod h1:XtLgD3ZD34DAaVIIAyG3objl5DynM3CQ/vMcbBNJZGI= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/appservice/armappservice/v2 v2.3.0 h1:JI8PcWOImyvIUEZ0Bbmfe05FOlWkMi2KhjG+cAKaUms= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/appservice/armappservice/v2 v2.3.0/go.mod h1:nJLFPGJkyKfDDyJiPuHIXsCi/gpJkm07EvRgiX7SGlI= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/internal/v2 v2.0.0 h1:PTFGRSlMKCQelWwxUyYVEUqseBJVemLyqWJjvMyt0do= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/internal/v2 v2.0.0/go.mod h1:LRr2FzBTQlONPPa5HREE5+RjSCTXl7BwOvYOaWTqCaI= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/internal/v3 v3.1.0 h1:2qsIIvxVT+uE6yrNldntJKlLRgxGbZ85kgtz5SNBhMw= @@ -247,6 +249,8 @@ go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= +go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.53.0 h1:QZ4Muo8THX6CizN2vPPd5fBGHyogrdK9fG4wLPFUsto= diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/models.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/models.go new file mode 100644 index 00000000000..cee6bbcd886 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/models.go @@ -0,0 +1,192 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package dataset_api + +import ( + "fmt" + "math" + "os" + "path/filepath" + "strconv" + "strings" +) + +// CreateDatasetRequest is the request body for creating (uploading) a dataset. +type CreateDatasetRequest struct { + Name string `json:"name"` + Version string `json:"version"` + Format string `json:"format"` + Content string `json:"content"` +} + +// Dataset is the response for dataset operations. +// Note: The GET /datasets API returns snake_case field names (data_uri, blob_uri, +// content_uri), while the POST /finalize API accepts camelCase (dataUri). +// Both conventions are correct for their respective endpoints. +type Dataset struct { + Name string `json:"name"` + Version string `json:"version"` + BlobURI string `json:"blob_uri,omitempty"` + Format string `json:"format,omitempty"` + DataURI string `json:"data_uri,omitempty"` + ContentURI string `json:"content_uri,omitempty"` +} + +// ResolvedBlobURI returns the best available blob URI. Prefers blob_uri, +// falls back to data_uri, then content_uri. +func (d *Dataset) ResolvedBlobURI() string { + if d.BlobURI != "" { + return d.BlobURI + } + if d.DataURI != "" { + return d.DataURI + } + return d.ContentURI +} + +// DatasetCredential is the response for dataset credential (SAS token) requests. +// The API returns a nested structure with blobReference and blobReferenceForConsumption. +type DatasetCredential struct { + // Flat fields (legacy format). + BlobURI string `json:"blob_uri,omitempty"` + SAS string `json:"sas,omitempty"` + SASUri string `json:"sas_uri,omitempty"` + + // Nested fields (current API format). + BlobReference *BlobReference `json:"blobReference,omitempty"` + BlobReferenceConsumption *BlobReference `json:"blobReferenceForConsumption,omitempty"` +} + +// BlobReference represents a blob storage reference with credentials. +type BlobReference struct { + BlobURI string `json:"blobUri,omitempty"` + StorageAccountARM string `json:"storageAccountArmId,omitempty"` + Credential *BlobCredential `json:"credential,omitempty"` +} + +// BlobCredential holds SAS credential details for blob access. +type BlobCredential struct { + Type string `json:"type,omitempty"` + SASUri string `json:"sasUri,omitempty"` + SASPath string `json:"sas,omitempty"` +} + +// ResolvedDownloadURI returns the URL to download the dataset. +// Prefers blobReferenceForConsumption.credential.sasUri (current API), +// then blobReference.credential.sasUri, then flat sas_uri, then blob_uri + sas. +func (c *DatasetCredential) ResolvedDownloadURI() string { + // Current API format: nested blob references. + if c.BlobReferenceConsumption != nil && c.BlobReferenceConsumption.Credential != nil { + if uri := c.BlobReferenceConsumption.Credential.SASUri; uri != "" { + return uri + } + } + if c.BlobReference != nil && c.BlobReference.Credential != nil { + if uri := c.BlobReference.Credential.SASUri; uri != "" { + return uri + } + } + // Legacy flat format. + if c.SASUri != "" { + return c.SASUri + } + if c.BlobURI != "" && c.SAS != "" { + return c.BlobURI + "?" + c.SAS + } + return c.BlobURI +} + +// PendingUploadResponse is returned by the startPendingUpload endpoint. +// It contains a SAS URI for uploading blob data and the blob container URI. +type PendingUploadResponse struct { + BlobReference *BlobReference `json:"blobReference,omitempty"` + BlobReferenceConsumption *BlobReference `json:"blobReferenceForConsumption,omitempty"` + PendingUploadID *string `json:"pendingUploadId,omitempty"` + PendingUploadType string `json:"pendingUploadType,omitempty"` + Version string `json:"version,omitempty"` +} + +// ResolvedUploadURI returns the SAS URI for uploading blobs. +func (p *PendingUploadResponse) ResolvedUploadURI() string { + if p.BlobReference != nil && p.BlobReference.Credential != nil { + if uri := p.BlobReference.Credential.SASUri; uri != "" { + return uri + } + } + return "" +} + +// ResolvedBlobURI returns the blob container URI (without SAS) for the finalize request. +func (p *PendingUploadResponse) ResolvedBlobURI() string { + if p.BlobReference != nil { + return p.BlobReference.BlobURI + } + return "" +} + +// FinalizeDatasetRequest is the request body for finalizing a dataset version +// after blob upload. +type FinalizeDatasetRequest struct { + Name string `json:"name"` + Version string `json:"version"` + Description string `json:"description"` + Type string `json:"type"` + IsReference bool `json:"isReference"` + DataURI string `json:"dataUri"` +} + +// NextVersion computes the next dataset version string. +// +// Rules: +// 1. Empty → "1.0" +// 2. Parsable as a decimal number → increment by 1, format as "N.0" +// 3. Ends with trailing digits → increment the trailing numeric part +// 4. Otherwise → append ".1" +func NextVersion(current string) string { + current = strings.TrimSpace(current) + if current == "" { + return "1.0" + } + + // Try parsing as a decimal number (e.g. "1", "1.0", "2.0"). + if f, err := strconv.ParseFloat(current, 64); err == nil { + return strconv.FormatFloat(math.Floor(f)+1, 'f', 1, 64) + } + + // Find trailing digits and increment them. + i := len(current) - 1 + for i >= 0 && current[i] >= '0' && current[i] <= '9' { + i-- + } + if i < len(current)-1 { + prefix := current[:i+1] + n, err := strconv.Atoi(current[i+1:]) + if err == nil { + return prefix + strconv.Itoa(n+1) + } + } + + return current + ".1" +} + +// ReadFirstJSONLFile finds and reads the first .jsonl file in a directory. +func ReadFirstJSONLFile(dir string) (string, error) { + entries, err := os.ReadDir(dir) + if err != nil { + return "", fmt.Errorf("reading directory: %w", err) + } + for _, e := range entries { + if e.IsDir() { + continue + } + if filepath.Ext(e.Name()) == ".jsonl" { + data, err := os.ReadFile(filepath.Join(dir, e.Name())) //nolint:gosec // local artifact path + if err != nil { + return "", fmt.Errorf("reading %s: %w", e.Name(), err) + } + return string(data), nil + } + } + return "", fmt.Errorf("no .jsonl file found in %s", dir) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/operations.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/operations.go new file mode 100644 index 00000000000..e29b6710f12 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/operations.go @@ -0,0 +1,442 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package dataset_api + +import ( + "bytes" + "context" + "encoding/json" + "encoding/xml" + "fmt" + "io" + "log" + "net/http" + "net/url" + "strings" + + "azureaieval/internal/version" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/streaming" + "github.com/azure/azure-dev/cli/azd/pkg/azsdk" +) + +// API path prefix for dataset endpoints. +const pathDatasets = "/datasets" + +// DatasetClient provides methods for dataset upload, download, and metadata retrieval. +type DatasetClient struct { + endpoint string + pipeline runtime.Pipeline +} + +// NewDatasetClient creates a new DatasetClient. +func NewDatasetClient(endpoint string, cred azcore.TokenCredential) *DatasetClient { + userAgent := fmt.Sprintf("azd-ext-azure-ai-evaluations/%s", version.Version) + + clientOptions := &policy.ClientOptions{ + Logging: policy.LogOptions{ + AllowedHeaders: []string{"X-Ms-Correlation-Request-Id", "X-Request-Id"}, + IncludeBody: false, + }, + PerCallPolicies: []policy.Policy{ + runtime.NewBearerTokenPolicy(cred, []string{"https://ai.azure.com/.default"}, nil), + azsdk.NewMsCorrelationPolicy(), + azsdk.NewUserAgentPolicy(userAgent), + }, + } + + pipeline := runtime.NewPipeline( + "azure-ai-datasets", + "v1.0.0", + runtime.PipelineOptions{}, + clientOptions, + ) + + return &DatasetClient{ + endpoint: endpoint, + pipeline: pipeline, + } +} + +// NewDatasetClientFromPipeline creates a DatasetClient with a pre-built pipeline. +// This is intended for tests that need to bypass auth policies. +func NewDatasetClientFromPipeline(endpoint string, pipeline runtime.Pipeline) *DatasetClient { + return &DatasetClient{ + endpoint: endpoint, + pipeline: pipeline, + } +} + +// CreateDataset registers a dataset with inline content (upload). +func (c *DatasetClient) CreateDataset( + ctx context.Context, + request *CreateDatasetRequest, + apiVersion string, +) (*Dataset, error) { + return doRequestTyped[Dataset](c, ctx, http.MethodPost, pathDatasets, nil, request, apiVersion) +} + +// UploadNewVersion reads the first JSONL file from localDir, computes the next +// version from currentVersion, and uploads it as a new dataset version using +// the 3-step pending upload flow: +// 1. startPendingUpload → get SAS URI +// 2. Upload blob to SAS URI +// 3. Finalize dataset version with dataUri +func (c *DatasetClient) UploadNewVersion( + ctx context.Context, + name string, + currentVersion string, + localDir string, + apiVersion string, +) (*Dataset, error) { + content, err := ReadFirstJSONLFile(localDir) + if err != nil { + return nil, fmt.Errorf("reading dataset from %s: %w", localDir, err) + } + + newVersion := NextVersion(currentVersion) + + // Step 1: Start pending upload to get a SAS URI. + pending, err := c.StartPendingUpload(ctx, name, newVersion, apiVersion) + if err != nil { + return nil, fmt.Errorf("starting pending upload: %w", err) + } + + uploadURI := pending.ResolvedUploadURI() + if uploadURI == "" { + return nil, fmt.Errorf("no upload SAS URI returned from startPendingUpload") + } + + // Step 2: Upload the JSONL file to blob storage. + blobName := name + ".jsonl" + if err := c.UploadBlob(ctx, uploadURI, blobName, []byte(content)); err != nil { + return nil, fmt.Errorf("uploading blob: %w", err) + } + + // Step 3: Finalize the dataset version with the full blob URI. + dataURI := strings.TrimSuffix(pending.ResolvedBlobURI(), "/") + "/" + blobName + return c.FinalizeDatasetVersion(ctx, name, newVersion, dataURI, apiVersion) +} + +// StartPendingUpload initiates a pending upload for a dataset version. +// Returns the SAS URI and blob reference for uploading data. +func (c *DatasetClient) StartPendingUpload( + ctx context.Context, + name string, + version string, + apiVersion string, +) (*PendingUploadResponse, error) { + path := fmt.Sprintf( + "%s/%s/versions/%s/startPendingUpload", + pathDatasets, url.PathEscape(name), url.PathEscape(version), + ) + return doRequestTyped[PendingUploadResponse](c, ctx, http.MethodPost, path, nil, json.RawMessage(`{}`), apiVersion) +} + +// UploadBlob uploads data to a container SAS URI as a block blob. +func (c *DatasetClient) UploadBlob(ctx context.Context, containerSASUri, blobName string, data []byte) error { + u, err := url.Parse(containerSASUri) + if err != nil { + return fmt.Errorf("invalid container SAS URI: %w", err) + } + + // Append blob name to the container path. + u.Path = strings.TrimSuffix(u.Path, "/") + "/" + blobName + + req, err := http.NewRequestWithContext(ctx, http.MethodPut, u.String(), bytes.NewReader(data)) + if err != nil { + return fmt.Errorf("failed to create upload request: %w", err) + } + req.Header.Set("x-ms-blob-type", "BlockBlob") + req.Header.Set("Content-Type", "application/octet-stream") + + httpClient := &http.Client{} + resp, err := httpClient.Do(req) + if err != nil { + return fmt.Errorf("failed to upload blob: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusCreated && resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(resp.Body) + return fmt.Errorf("blob upload failed with status %d: %s", resp.StatusCode, string(body)) + } + + return nil +} + +// FinalizeDatasetVersion completes the dataset version after blob upload +// by sending the metadata (name, version, dataUri) to the API. +func (c *DatasetClient) FinalizeDatasetVersion( + ctx context.Context, + name string, + version string, + dataURI string, + apiVersion string, +) (*Dataset, error) { + path := fmt.Sprintf("%s/%s/versions/%s", pathDatasets, url.PathEscape(name), url.PathEscape(version)) + request := &FinalizeDatasetRequest{ + Name: name, + Version: version, + Type: "uri_file", + DataURI: dataURI, + } + return doRequestTyped[Dataset](c, ctx, http.MethodPut, path, nil, request, apiVersion) +} + +// GetDataset retrieves metadata for a dataset by name and version. +func (c *DatasetClient) GetDataset( + ctx context.Context, + name string, + version string, + apiVersion string, +) (*Dataset, error) { + path := fmt.Sprintf("%s/%s/versions/%s", pathDatasets, url.PathEscape(name), url.PathEscape(version)) + return doRequestTyped[Dataset](c, ctx, http.MethodGet, path, nil, nil, apiVersion) +} + +// GetDatasetCredential retrieves a SAS credential for downloading a dataset from blob storage. +func (c *DatasetClient) GetDatasetCredential( + ctx context.Context, + name string, + version string, + apiVersion string, +) (*DatasetCredential, error) { + path := fmt.Sprintf( + "%s/%s/versions/%s/credentials", + pathDatasets, url.PathEscape(name), url.PathEscape(version), + ) + return doRequestTyped[DatasetCredential](c, ctx, http.MethodPost, path, nil, nil, apiVersion) +} + +// DownloadDataset downloads dataset content from blob storage using a SAS-authenticated URL. +// Returns the raw content as bytes. The downloadURL should be the full URL with SAS token +// (e.g., from DatasetCredential.ResolvedDownloadURI()). +func (c *DatasetClient) DownloadDataset(ctx context.Context, downloadURL string) ([]byte, error) { + req, err := runtime.NewRequest(ctx, http.MethodGet, downloadURL) + if err != nil { + return nil, fmt.Errorf("failed to create download request: %w", err) + } + + // Use a plain HTTP client for blob downloads — the SAS token in the URL provides + // authentication, and Azure SDK pipeline policies (bearer token, correlation ID) + // should not be sent to Azure Blob Storage endpoints. + httpClient := &http.Client{} + resp, err := httpClient.Do(req.Raw()) + if err != nil { + return nil, fmt.Errorf("failed to download dataset from blob: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("blob download failed with status %d", resp.StatusCode) + } + + data, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("failed to read dataset content: %w", err) + } + + log.Printf("[dataset_api] downloaded %d bytes", len(data)) + return data, nil +} + +// ListContainerBlobs lists blobs in a container using a container-level SAS URI. +// The containerSASUri should include the SAS token (e.g., from credential.sasUri with sr=c). +// Returns a list of blob names found in the container. +func (c *DatasetClient) ListContainerBlobs(ctx context.Context, containerSASUri string) ([]string, error) { + // Parse the container URI and append list query parameters. + u, err := url.Parse(containerSASUri) + if err != nil { + return nil, fmt.Errorf("invalid container SAS URI: %w", err) + } + + q := u.Query() + q.Set("restype", "container") // cspell:ignore restype — Azure Storage API query parameter + q.Set("comp", "list") + u.RawQuery = q.Encode() + + log.Printf("[dataset_api] listing blobs: %s", u.Redacted()) + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil) + if err != nil { + return nil, fmt.Errorf("failed to create list request: %w", err) + } + + httpClient := &http.Client{} + resp, err := httpClient.Do(req) + if err != nil { + return nil, fmt.Errorf("failed to list container blobs: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("container list failed with status %d", resp.StatusCode) + } + + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("failed to read list response: %w", err) + } + + // Parse XML blob listing to extract blob names. + names := parseBlobNames(string(body)) + log.Printf("[dataset_api] found %d blobs in container", len(names)) + return names, nil +} + +// DownloadBlob downloads a single blob from a container using the container SAS URI +// and the blob name. Returns the blob content as bytes. +func (c *DatasetClient) DownloadBlob(ctx context.Context, containerSASUri, blobName string) ([]byte, error) { + u, err := url.Parse(containerSASUri) + if err != nil { + return nil, fmt.Errorf("invalid container SAS URI: %w", err) + } + + // Append blob name to the container path. + u.Path = strings.TrimSuffix(u.Path, "/") + "/" + blobName + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil) + if err != nil { + return nil, fmt.Errorf("failed to create blob download request: %w", err) + } + + httpClient := &http.Client{} + resp, err := httpClient.Do(req) + if err != nil { + return nil, fmt.Errorf("failed to download blob: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("blob download failed with status %d for %s", resp.StatusCode, blobName) + } + + data, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("failed to read blob content: %w", err) + } + + log.Printf("[dataset_api] downloaded blob %s (%d bytes)", blobName, len(data)) + return data, nil +} + +// parseBlobNames extracts blob names from the Azure Blob Storage XML list response +// using proper XML parsing against the EnumerationResults schema. +func parseBlobNames(xmlBody string) []string { + type blob struct { + Name string `xml:"Name"` + } + type blobs struct { + Blob []blob `xml:"Blob"` + } + type enumerationResults struct { + Blobs blobs `xml:"Blobs"` + } + + var result enumerationResults + if err := xml.Unmarshal([]byte(xmlBody), &result); err != nil { + return nil + } + + names := make([]string, 0, len(result.Blobs.Blob)) + for _, b := range result.Blobs.Blob { + if b.Name != "" { + names = append(names, b.Name) + } + } + return names +} + +// doRequest performs an HTTP request against the dataset API and returns the raw response body. +func (c *DatasetClient) doRequest( + ctx context.Context, + method string, + path string, + query map[string]string, + body any, + apiVersion string, +) ([]byte, error) { + u, err := url.Parse(c.endpoint) + if err != nil { + return nil, fmt.Errorf("invalid endpoint URL: %w", err) + } + + u.Path += path + q := u.Query() + if apiVersion != "" { + q.Set("api-version", apiVersion) + } + for k, v := range query { + q.Set(k, v) + } + u.RawQuery = q.Encode() + + req, err := runtime.NewRequest(ctx, method, u.String()) + if err != nil { + return nil, fmt.Errorf("failed to create request: %w", err) + } + + log.Printf("[dataset_api] %s %s", method, u.Redacted()) + + if body != nil { + payload, err := json.Marshal(body) + if err != nil { + return nil, fmt.Errorf("failed to marshal request: %w", err) + } + if err := req.SetBody(streaming.NopCloser(bytes.NewReader(payload)), "application/json"); err != nil { + return nil, fmt.Errorf("failed to set request body: %w", err) + } + } + + resp, err := c.pipeline.Do(req) + if err != nil { + return nil, fmt.Errorf("HTTP request failed: %w", err) + } + defer resp.Body.Close() + + respBody, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("failed to read response body: %w", err) + } + + log.Printf("[dataset_api] response status: %d", resp.StatusCode) + + if !runtime.HasStatusCode(resp, http.StatusOK, http.StatusCreated, http.StatusAccepted) { + resp.Body = io.NopCloser(bytes.NewReader(respBody)) + return nil, runtime.NewResponseError(resp) + } + + return respBody, nil +} + +// doRequestTyped performs an HTTP request and unmarshals the response into T. +func doRequestTyped[T any]( + c *DatasetClient, + ctx context.Context, + method string, + path string, + query map[string]string, + body any, + apiVersion string, +) (*T, error) { + respBody, err := c.doRequest(ctx, method, path, query, body, apiVersion) + if err != nil { + return nil, err + } + + if len(respBody) == 0 { + return new(T), nil + } + + var result T + if err := json.Unmarshal(respBody, &result); err != nil { + return nil, fmt.Errorf("failed to parse response: %w", err) + } + + return &result, nil +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/generation.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/generation.go new file mode 100644 index 00000000000..2b12968065a --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/generation.go @@ -0,0 +1,140 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package eval_api + +import ( + "path/filepath" + "strings" + "time" + + "azureaieval/internal/pkg/evalcore" +) + +// --------------------------------------------------------------------------- +// Generation source building +// --------------------------------------------------------------------------- + +// TraceOptions holds optional trace inclusion parameters for generation sources. +type TraceOptions struct { + Days int +} + +// BuildGenerationSources constructs the sources array for generation jobs. +// A prompt source is included when instruction is non-empty, along with the +// agent source. When traces is non-nil and Days > 0, a traces source is +// appended with start_time computed from the current time. +func BuildGenerationSources(agentKind, agentName, version, instruction string, traces *TraceOptions) []GenerationSource { + var sources []GenerationSource + + if instruction != "" { + sources = append(sources, GenerationSource{ + Type: "prompt", + Prompt: instruction, + }) + } + + agentSource := GenerationSource{ + Type: "agent", + AgentName: agentName, + } + if version != "" { + agentSource.AgentVersion = version + } + sources = append(sources, agentSource) + + if traces != nil && traces.Days > 0 { + startTime := time.Now().AddDate(0, 0, -traces.Days).Unix() + sources = append(sources, GenerationSource{ + Type: "traces", + AgentName: agentName, + StartTime: startTime, + }) + } + + return sources +} + +// --------------------------------------------------------------------------- +// Request builders +// --------------------------------------------------------------------------- + +// NewDataGenerationJobRequest builds a DataGenerationJobRequest from the +// provided parameters. Currently, it's always "simple_qna" type with multiple sources +func NewDataGenerationJobRequest( + name, evalModel string, + maxSamples int, + sources []GenerationSource, +) *DataGenerationJobRequest { + return &DataGenerationJobRequest{ + Inputs: DataGenerationInputs{ + Name: name, + Scenario: "evaluation", + Options: DataGenerationOptions{ + Type: "simple_qna", + MaxSamples: maxSamples, + ModelOptions: ModelOptions{ + Model: evalModel, + }, + }, + Sources: sources, + }, + } +} + +// NewEvaluatorGenerationJobRequest builds an EvaluatorGenerationJobRequest +// from the provided parameters. +func NewEvaluatorGenerationJobRequest( + name, evalModel string, + sources []GenerationSource, +) *EvaluatorGenerationJobRequest { + return &EvaluatorGenerationJobRequest{ + Inputs: EvaluatorGenerationInputs{ + Name: name, + EvaluatorName: name, + Model: evalModel, + Sources: sources, + }, + } +} + +// --------------------------------------------------------------------------- +// Evaluator classification +// --------------------------------------------------------------------------- + +// IsBuiltinEvaluator returns true when the evaluator name has the "builtin." +// prefix. +func IsBuiltinEvaluator(name string) bool { + return strings.HasPrefix(name, "builtin.") +} + +// SplitEvaluators partitions evaluators into generated (non-builtin) and +// built-in lists. +func SplitEvaluators(evaluators evalcore.EvaluatorList) (generated, builtin evalcore.EvaluatorList) { + for _, e := range evaluators { + if IsBuiltinEvaluator(e.Name) { + builtin = append(builtin, e) + } else { + generated = append(generated, e) + } + } + return generated, builtin +} + +// --------------------------------------------------------------------------- +// Dataset name detection +// --------------------------------------------------------------------------- + +// IsDatasetName returns true when the value looks like a registered dataset +// name rather than a local file path. A name has no path separators and no +// common data-file extension (.jsonl, .json, .csv). +func IsDatasetName(value string) bool { + if value == "" { + return false + } + if strings.ContainsAny(value, "/\\") { + return false + } + ext := strings.ToLower(filepath.Ext(value)) + return ext != ".jsonl" && ext != ".json" && ext != ".csv" +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/models.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/models.go new file mode 100644 index 00000000000..cf51017746c --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/models.go @@ -0,0 +1,412 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package eval_api + +import "encoding/json" + +// --------------------------------------------------------------------------- +// Data Generation Jobs +// --------------------------------------------------------------------------- + +// DataGenerationJobRequest is the request body for CreateDataGenerationJob. +type DataGenerationJobRequest struct { + Inputs DataGenerationInputs `json:"inputs"` +} + +// DataGenerationInputs holds the inputs for a data generation job. +type DataGenerationInputs struct { + Name string `json:"name"` + Scenario string `json:"scenario"` + Options DataGenerationOptions `json:"options"` + Sources []GenerationSource `json:"sources"` +} + +// DataGenerationOptions holds configuration for data generation. +type DataGenerationOptions struct { + Type string `json:"type"` + MaxSamples int `json:"max_samples"` + ModelOptions ModelOptions `json:"model_options"` +} + +// ModelOptions holds the model selection for generation. +type ModelOptions struct { + Model string `json:"model"` +} + +// GenerationSource describes a source used for dataset or evaluator generation. +type GenerationSource struct { + Type string `json:"type"` + Prompt string `json:"prompt,omitempty"` + AgentName string `json:"agent_name,omitempty"` + AgentVersion string `json:"agent_version,omitempty"` + StartTime int64 `json:"start_time,omitempty"` +} + +// GenerationJob is the response for data and evaluator generation job operations. +type GenerationJob struct { + ID string `json:"id"` + Status string `json:"status"` + Result json.RawMessage `json:"result,omitempty"` + Error *JobError `json:"error,omitempty"` +} + +// JobError captures error details from a failed generation job. +type JobError struct { + Code string `json:"code,omitempty"` + Message string `json:"message,omitempty"` +} + +// OperationID returns the job's operation identifier. +func (j *GenerationJob) OperationID() string { + return j.ID +} + +// NormalizedStatus returns the lowercase status, defaulting to "running". +func (j *GenerationJob) NormalizedStatus() string { + if j.Status == "" { + return "running" + } + return j.Status +} + +// ResolvedNameVersion extracts the name and version from the generation job result. +// If name is empty, both return values are empty (caller should treat as no result). +// If version is empty, it defaults to "latest". +func (j *GenerationJob) ResolvedNameVersion() (string, string) { + name := j.resultStringField("name") + if name == "" { + return "", "" + } + version := j.resultStringField("version") + if version == "" { + version = "latest" + } + return name, version +} + +// resultStringField extracts a string field from the raw Result JSON. +// It first checks for a top-level key, then falls back to outputs[0].key +// to handle the nested response format. +func (j *GenerationJob) resultStringField(key string) string { + if len(j.Result) == 0 { + return "" + } + var m map[string]json.RawMessage + if err := json.Unmarshal(j.Result, &m); err != nil { + return "" + } + + // Try top-level field first. + if raw, ok := m[key]; ok { + var s string + if err := json.Unmarshal(raw, &s); err == nil && s != "" { + return s + } + } + + // Fall back to outputs[0].key for nested response format. + if rawOutputs, ok := m["outputs"]; ok { + var outputs []map[string]json.RawMessage + if err := json.Unmarshal(rawOutputs, &outputs); err == nil && len(outputs) > 0 { + if raw, ok := outputs[0][key]; ok { + var s string + if err := json.Unmarshal(raw, &s); err == nil { + return s + } + } + } + } + + return "" +} + +// --------------------------------------------------------------------------- +// Evaluator Generation Jobs +// --------------------------------------------------------------------------- + +// EvaluatorGenerationJobRequest is the request body for CreateEvaluatorGenerationJob. +type EvaluatorGenerationJobRequest struct { + Inputs EvaluatorGenerationInputs `json:"inputs"` +} + +// EvaluatorGenerationInputs holds the inputs for an evaluator generation job. +type EvaluatorGenerationInputs struct { + Name string `json:"name"` + EvaluatorName string `json:"evaluator_name"` + Category string `json:"category,omitempty"` + Model string `json:"model"` + Sources []GenerationSource `json:"sources"` +} + +// --------------------------------------------------------------------------- +// Evaluator Versions +// --------------------------------------------------------------------------- + +// EvaluatorVersion is the response for evaluator version operations. +type EvaluatorVersion struct { + Name string `json:"name"` + Version string `json:"version"` +} + +// --------------------------------------------------------------------------- +// Evaluator Definition (Rubric) +// --------------------------------------------------------------------------- + +// EvaluatorResult is the top-level response from evaluator generation, +// containing the evaluator's definition. +type EvaluatorResult struct { + Name string `json:"name"` + Version string `json:"version,omitempty"` + Definition EvaluatorDefinition `json:"definition"` +} + +// EvaluatorDefinition describes an evaluator's scoring rubric. +type EvaluatorDefinition struct { + Type string `json:"type"` + Dimensions []EvaluatorDimension `json:"dimensions"` +} + +// EvaluatorDimension is a single scoring dimension within a rubric evaluator. +type EvaluatorDimension struct { + ID string `json:"id"` + Description string `json:"description,omitempty"` + Weight int `json:"weight"` + AlwaysApplicable bool `json:"always_applicable,omitempty"` +} + +// ParseEvaluatorResult parses a GenerationJob result into a structured EvaluatorResult. +// Returns nil if the result cannot be parsed. +func ParseEvaluatorResult(result json.RawMessage) *EvaluatorResult { + if len(result) == 0 { + return nil + } + var r EvaluatorResult + if err := json.Unmarshal(result, &r); err != nil { + return nil + } + if len(r.Definition.Dimensions) == 0 { + return nil + } + return &r +} + +// --------------------------------------------------------------------------- +// Datasets +// --------------------------------------------------------------------------- + +// CreateDatasetRequest is the request body for CreateDataset. +type CreateDatasetRequest struct { + Name string `json:"name"` + Version string `json:"version"` + Format string `json:"format"` + Content string `json:"content"` +} + +// Dataset is the response for dataset operations. +type Dataset struct { + Name string `json:"name"` + Version string `json:"version"` +} + +// --------------------------------------------------------------------------- +// OpenAI Evals +// --------------------------------------------------------------------------- + +// DataSourceConfig describes the data source for an OpenAI eval. +type DataSourceConfig struct { + Type string `json:"type"` + ItemSchema map[string]any `json:"item_schema"` + IncludeSampleSchema bool `json:"include_sample_schema"` +} + +// DataSourceSchema defines the item and sample schemas for an eval data source. +type DataSourceSchema struct { + Item map[string]any `json:"item,omitempty"` + Sample map[string]any `json:"sample,omitempty"` +} + +// TestingCriterion describes a single evaluator in testing_criteria. +type TestingCriterion struct { + Type string `json:"type"` + Name string `json:"name"` + EvaluatorName string `json:"evaluator_name"` + InitializationParameters map[string]any `json:"initialization_parameters,omitempty"` + DataMapping map[string]string `json:"data_mapping,omitempty"` +} + +// CreateOpenAIEvalRequest is the request body for CreateOpenAIEval. +type CreateOpenAIEvalRequest struct { + Name string `json:"name"` + Metadata map[string]string `json:"metadata,omitempty"` + DataSourceConfig *DataSourceConfig `json:"data_source_config,omitempty"` + TestingCriteria []TestingCriterion `json:"testing_criteria,omitempty"` +} + +// OpenAIEval is the response for an OpenAI eval definition. +type OpenAIEval struct { + ID string `json:"id"` + Name string `json:"name,omitempty"` + CreatedAt any `json:"created_at,omitempty"` + ModifiedAt any `json:"modified_at,omitempty"` + CreatedBy string `json:"created_by,omitempty"` + Metadata map[string]string `json:"metadata,omitempty"` +} + +// ResolvedID returns the eval's ID, falling back to name. +func (e *OpenAIEval) ResolvedID() string { + if e.ID != "" { + return e.ID + } + return e.Name +} + +// OpenAIEvalList is the response for listing OpenAI eval definitions. +type OpenAIEvalList struct { + Data []OpenAIEval `json:"data"` +} + +// --------------------------------------------------------------------------- +// OpenAI Eval Runs +// --------------------------------------------------------------------------- + +// CreateOpenAIEvalRunRequest is the request body for CreateOpenAIEvalRun. +type CreateOpenAIEvalRunRequest struct { + Name string `json:"name"` + DataSource *EvalRunDataSource `json:"data_source,omitempty"` + Metadata map[string]string `json:"metadata,omitempty"` +} + +// EvalRunDataSourceType defines the type for an eval run data source. +type EvalRunDataSourceType string + +const ( + // EvalRunDataSourceTypeAgentTarget is the data source type for agent target completions. + EvalRunDataSourceTypeAgentTarget EvalRunDataSourceType = "azure_ai_target_completions" +) + +// EvalRunDataContentType defines the source type for eval run data content. +type EvalRunDataContentType string + +const ( + EvalRunDataContentTypeFileContent EvalRunDataContentType = "file_content" + EvalRunDataContentTypeFileID EvalRunDataContentType = "file_id" +) + +// EvalRunDataSource describes the data source for an eval run with agent target completions. +type EvalRunDataSource struct { + Type EvalRunDataSourceType `json:"type"` + InputMessages *EvalRunInputMessages `json:"input_messages,omitempty"` + Source *EvalRunDataContent `json:"source,omitempty"` + Target *EvalRunTarget `json:"target,omitempty"` +} + +// EvalRunInputMessages describes how input messages are constructed from dataset items. +type EvalRunInputMessages struct { + Type string `json:"type"` + Template []EvalRunMessageTemplate `json:"template"` +} + +// EvalRunMessageTemplate describes a single message in the input template. +type EvalRunMessageTemplate struct { + Role string `json:"role"` + Content string `json:"content"` + Type string `json:"type"` +} + +// EvalRunTarget describes the agent target for completions. +type EvalRunTarget struct { + Type string `json:"type"` + Name string `json:"name"` + Version *string `json:"version"` + ToolDescriptions []string `json:"tool_descriptions"` +} + +// EvalRunDataContent holds the source reference within an EvalRunDataSource. +type EvalRunDataContent struct { + Type EvalRunDataContentType `json:"type"` + ID string `json:"id,omitempty"` + Content []map[string]any `json:"content,omitempty"` +} + +// NewAgentTargetDataSource builds an EvalRunDataSource configured for agent target completions. +// The source field must be set separately via SetFileContent or SetFileID. +func NewAgentTargetDataSource(agentName string, agentVersion *string) *EvalRunDataSource { + return &EvalRunDataSource{ + Type: EvalRunDataSourceTypeAgentTarget, + InputMessages: &EvalRunInputMessages{ + Type: "template", + Template: []EvalRunMessageTemplate{ + { + Role: "user", + Content: "{{item.query}}", + Type: "message", + }, + }, + }, + Target: &EvalRunTarget{ + Type: "azure_ai_agent", + Name: agentName, + Version: agentVersion, + ToolDescriptions: []string{}, + }, + } +} + +// SetFileContent sets the data source to use inline file content. +func (ds *EvalRunDataSource) SetFileContent(items []map[string]any) { + ds.Source = &EvalRunDataContent{ + Type: EvalRunDataContentTypeFileContent, + Content: items, + } +} + +// SetFileID sets the data source to reference a remote dataset by ID. +func (ds *EvalRunDataSource) SetFileID(fileID string) { + ds.Source = &EvalRunDataContent{ + Type: EvalRunDataContentTypeFileID, + ID: fileID, + } +} + +// OpenAIEvalRun is the response for an OpenAI eval run. +type OpenAIEvalRun struct { + ID string `json:"id"` + EvalID string `json:"eval_id,omitempty"` + Name string `json:"name,omitempty"` + Status string `json:"status,omitempty"` + CreatedAt any `json:"created_at,omitempty"` + ModifiedAt any `json:"modified_at,omitempty"` + CreatedBy string `json:"created_by,omitempty"` + DataSource *EvalRunDataSource `json:"data_source,omitempty"` + Metadata map[string]string `json:"metadata,omitempty"` + ReportURL string `json:"report_url,omitempty"` + + // Result summary + ResultCounts *EvalRunResultCounts `json:"result_counts,omitempty"` + PerTestingCriteria []EvalRunCriteriaResult `json:"per_testing_criteria_results,omitempty"` + Error any `json:"error,omitempty"` +} + +// EvalRunResultCounts holds pass/fail/error/skip counts for a run. +type EvalRunResultCounts struct { + Total int `json:"total"` + Passed int `json:"passed"` + Failed int `json:"failed"` + Errored int `json:"errored"` + Skipped int `json:"skipped"` +} + +// EvalRunCriteriaResult holds per-testing-criteria pass/fail counts. +type EvalRunCriteriaResult struct { + TestingCriteria string `json:"testing_criteria"` + Passed int `json:"passed"` + Failed int `json:"failed"` + Errored int `json:"errored"` + Skipped int `json:"skipped"` +} + +// OpenAIEvalRunList is the response for listing OpenAI eval runs. +type OpenAIEvalRunList struct { + Data []OpenAIEvalRun `json:"data"` +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/operations.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/operations.go new file mode 100644 index 00000000000..510b2680b31 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/operations.go @@ -0,0 +1,290 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package eval_api + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "log" + "net/http" + "net/url" + "strconv" + + "azureaieval/internal/version" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/streaming" + "github.com/azure/azure-dev/cli/azd/pkg/azsdk" +) + +// API path prefixes for eval service endpoints. +const ( + pathDataGenerationJobs = "/data_generation_jobs" + pathEvaluatorGenerationJobs = "/evaluator_generation_jobs" + pathEvaluators = "/evaluators" + pathDatasets = "/datasets" + pathOpenAIEvals = "/openai/v1/evals" +) + +// EvalClient provides methods for interacting with the Azure AI eval APIs. +type EvalClient struct { + endpoint string + pipeline runtime.Pipeline +} + +// NewEvalClient creates a new EvalClient. +func NewEvalClient(endpoint string, cred azcore.TokenCredential) *EvalClient { + userAgent := fmt.Sprintf("azd-ext-azure-ai-evaluations/%s", version.Version) + + clientOptions := &policy.ClientOptions{ + Logging: policy.LogOptions{ + AllowedHeaders: []string{"X-Ms-Correlation-Request-Id", "X-Request-Id"}, + IncludeBody: false, + }, + PerCallPolicies: []policy.Policy{ + runtime.NewBearerTokenPolicy(cred, []string{"https://ai.azure.com/.default"}, nil), + azsdk.NewMsCorrelationPolicy(), + azsdk.NewUserAgentPolicy(userAgent), + }, + } + + pipeline := runtime.NewPipeline( + "azure-ai-evals", + "v1.0.0", + runtime.PipelineOptions{}, + clientOptions, + ) + + return &EvalClient{ + endpoint: endpoint, + pipeline: pipeline, + } +} + +// NewEvalClientFromPipeline creates an EvalClient with a pre-built pipeline. +// This is intended for tests that need to bypass auth policies. +func NewEvalClientFromPipeline(endpoint string, pipeline runtime.Pipeline) *EvalClient { + return &EvalClient{ + endpoint: endpoint, + pipeline: pipeline, + } +} + +// CreateDataGenerationJob starts a dataset generation job for eval onboarding. +func (c *EvalClient) CreateDataGenerationJob( + ctx context.Context, + request *DataGenerationJobRequest, + apiVersion string, +) (*GenerationJob, error) { + return doRequestTyped[GenerationJob](c, ctx, http.MethodPost, pathDataGenerationJobs, nil, request, apiVersion) +} + +// GetDataGenerationJob gets the current state of a dataset generation job. +func (c *EvalClient) GetDataGenerationJob( + ctx context.Context, + operationID string, + apiVersion string, +) (*GenerationJob, error) { + path := pathDataGenerationJobs + "/" + url.PathEscape(operationID) + return doRequestTyped[GenerationJob](c, ctx, http.MethodGet, path, nil, nil, apiVersion) +} + +// CreateEvaluatorGenerationJob starts an evaluator generation job for eval onboarding. +func (c *EvalClient) CreateEvaluatorGenerationJob( + ctx context.Context, + request *EvaluatorGenerationJobRequest, + apiVersion string, +) (*GenerationJob, error) { + return doRequestTyped[GenerationJob](c, ctx, http.MethodPost, pathEvaluatorGenerationJobs, nil, request, apiVersion) +} + +// GetEvaluatorGenerationJob gets the current state of an evaluator generation job. +func (c *EvalClient) GetEvaluatorGenerationJob( + ctx context.Context, + operationID string, + apiVersion string, +) (*GenerationJob, error) { + path := pathEvaluatorGenerationJobs + "/" + url.PathEscape(operationID) + return doRequestTyped[GenerationJob](c, ctx, http.MethodGet, path, nil, nil, apiVersion) +} + +// CreateEvaluatorVersion creates a new version of a named evaluator. +// The body should be the full evaluator JSON with the definition field updated. +func (c *EvalClient) CreateEvaluatorVersion( + ctx context.Context, + name string, + body json.RawMessage, + apiVersion string, +) (*EvaluatorVersion, error) { + path := pathEvaluators + "/" + url.PathEscape(name) + "/versions" + return doRequestTyped[EvaluatorVersion](c, ctx, http.MethodPost, path, nil, body, apiVersion) +} + +// GetEvaluatorRaw gets an evaluator by name and version as raw JSON. +// If version is empty, the latest version is fetched. +func (c *EvalClient) GetEvaluatorRaw( + ctx context.Context, + name string, + version string, + apiVersion string, +) (json.RawMessage, error) { + path := pathEvaluators + "/" + url.PathEscape(name) + if version != "" { + path += "/versions/" + url.PathEscape(version) + } + return c.doRequest(ctx, http.MethodGet, path, nil, nil, apiVersion) +} + +// CreateOpenAIEval creates an OpenAI eval definition. +func (c *EvalClient) CreateOpenAIEval( + ctx context.Context, + request *CreateOpenAIEvalRequest, +) (*OpenAIEval, error) { + return doRequestTyped[OpenAIEval](c, ctx, http.MethodPost, pathOpenAIEvals, nil, request, "") +} + +// ListOpenAIEvals lists OpenAI eval definitions. +func (c *EvalClient) ListOpenAIEvals(ctx context.Context, limit int) (*OpenAIEvalList, error) { + query := map[string]string{} + if limit > 0 { + query["limit"] = strconv.Itoa(limit) + } + + return doRequestTyped[OpenAIEvalList](c, ctx, http.MethodGet, pathOpenAIEvals, query, nil, "") +} + +// GetOpenAIEval gets an OpenAI eval definition. +func (c *EvalClient) GetOpenAIEval(ctx context.Context, evalID string) (*OpenAIEval, error) { + path := pathOpenAIEvals + "/" + url.PathEscape(evalID) + return doRequestTyped[OpenAIEval](c, ctx, http.MethodGet, path, nil, nil, "") +} + +// CreateOpenAIEvalRun starts a run for an OpenAI eval definition. +func (c *EvalClient) CreateOpenAIEvalRun( + ctx context.Context, + evalID string, + request *CreateOpenAIEvalRunRequest, +) (*OpenAIEvalRun, error) { + path := fmt.Sprintf("%s/%s/runs", pathOpenAIEvals, url.PathEscape(evalID)) + return doRequestTyped[OpenAIEvalRun](c, ctx, http.MethodPost, path, nil, request, "") +} + +// ListOpenAIEvalRuns lists runs for an OpenAI eval definition. +func (c *EvalClient) ListOpenAIEvalRuns( + ctx context.Context, + evalID string, + limit int, +) (*OpenAIEvalRunList, error) { + query := map[string]string{} + if limit > 0 { + query["limit"] = strconv.Itoa(limit) + } + + path := fmt.Sprintf("%s/%s/runs", pathOpenAIEvals, url.PathEscape(evalID)) + return doRequestTyped[OpenAIEvalRunList](c, ctx, http.MethodGet, path, query, nil, "") +} + +// GetOpenAIEvalRun gets a run for an OpenAI eval definition. +func (c *EvalClient) GetOpenAIEvalRun( + ctx context.Context, + evalID string, + runID string, +) (*OpenAIEvalRun, error) { + path := fmt.Sprintf("%s/%s/runs/%s", pathOpenAIEvals, url.PathEscape(evalID), url.PathEscape(runID)) + return doRequestTyped[OpenAIEvalRun](c, ctx, http.MethodGet, path, nil, nil, "") +} + +func (c *EvalClient) doRequest( + ctx context.Context, + method string, + path string, + query map[string]string, + body any, + apiVersion string, +) ([]byte, error) { + u, err := url.Parse(c.endpoint) + if err != nil { + return nil, fmt.Errorf("invalid endpoint URL: %w", err) + } + + u.Path += path + q := u.Query() + if apiVersion != "" { + q.Set("api-version", apiVersion) + } + for k, v := range query { + q.Set(k, v) + } + u.RawQuery = q.Encode() + + req, err := runtime.NewRequest(ctx, method, u.String()) + if err != nil { + return nil, fmt.Errorf("failed to create request: %w", err) + } + + log.Printf("[eval_api] %s %s", method, u.Redacted()) + + if body != nil { + payload, err := json.Marshal(body) + if err != nil { + return nil, fmt.Errorf("failed to marshal request: %w", err) + } + if err := req.SetBody(streaming.NopCloser(bytes.NewReader(payload)), "application/json"); err != nil { + return nil, fmt.Errorf("failed to set request body: %w", err) + } + } + + resp, err := c.pipeline.Do(req) + if err != nil { + return nil, fmt.Errorf("HTTP request failed: %w", err) + } + defer resp.Body.Close() + + respBody, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("failed to read response body: %w", err) + } + + log.Printf("[eval_api] response status: %d", resp.StatusCode) + + if !runtime.HasStatusCode(resp, http.StatusOK, http.StatusCreated, http.StatusAccepted) { + // Restore the body so runtime.NewResponseError can read it. + resp.Body = io.NopCloser(bytes.NewReader(respBody)) + return nil, runtime.NewResponseError(resp) + } + + return respBody, nil +} + +// doRequestTyped performs an HTTP request and unmarshals the response into T. +func doRequestTyped[T any]( + c *EvalClient, + ctx context.Context, + method string, + path string, + query map[string]string, + body any, + apiVersion string, +) (*T, error) { + respBody, err := c.doRequest(ctx, method, path, query, body, apiVersion) + if err != nil { + return nil, err + } + + if len(respBody) == 0 { + return new(T), nil + } + + var result T + if err := json.Unmarshal(respBody, &result); err != nil { + return nil, fmt.Errorf("failed to parse response: %w", err) + } + + return &result, nil +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/poller.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/poller.go new file mode 100644 index 00000000000..4976248d87a --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/poller.go @@ -0,0 +1,195 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package eval_api + +import ( + "context" + "fmt" + "log" + "strings" + "time" + + "azureaieval/internal/pkg/evalcore" +) + +// --------------------------------------------------------------------------- +// JobStatus — typed status with terminal/failed semantics +// --------------------------------------------------------------------------- + +// JobStatus represents the normalized status of a generation job. +type JobStatus string + +const ( + JobStatusRunning JobStatus = "running" + JobStatusCompleted JobStatus = "completed" + JobStatusSucceeded JobStatus = "succeeded" + JobStatusFailed JobStatus = "failed" + JobStatusCancelled JobStatus = "cancelled" + JobStatusCanceled JobStatus = "canceled" +) + +// ParseJobStatus normalizes a raw status string into a JobStatus. +// An empty string is treated as "running". +func ParseJobStatus(s string) JobStatus { + if s == "" { + return JobStatusRunning + } + return JobStatus(strings.ToLower(s)) +} + +// IsTerminal returns true when the status represents a final state. +func (s JobStatus) IsTerminal() bool { + switch s { + case JobStatusCompleted, JobStatusSucceeded, JobStatusFailed, JobStatusCancelled, JobStatusCanceled: + return true + } + return false +} + +// IsFailed returns true when the status represents a failure or cancellation. +func (s JobStatus) IsFailed() bool { + switch s { + case JobStatusFailed, JobStatusCancelled, JobStatusCanceled: + return true + } + return false +} + +// String returns the status as a plain string. +func (s JobStatus) String() string { + return string(s) +} + +// --------------------------------------------------------------------------- +// JobFailedError — returned when a polled job reaches a failed state +// --------------------------------------------------------------------------- + +// JobFailedError is returned when a generation job reaches a failed terminal state. +type JobFailedError struct { + Job *GenerationJob + Status JobStatus +} + +func (e *JobFailedError) Error() string { + if e.Job != nil && e.Job.Error != nil && e.Job.Error.Message != "" { + return fmt.Sprintf("job failed with status %q: %s", e.Status, e.Job.Error.Message) + } + return fmt.Sprintf("job failed with status %q", e.Status) +} + +// --------------------------------------------------------------------------- +// PollerTimeoutError — returned when polling exhausts all attempts +// --------------------------------------------------------------------------- + +// PollerTimeoutError is returned when a generation job has not reached a +// terminal state within the configured number of polling attempts. +type PollerTimeoutError struct { + OperationID string + Attempts int +} + +func (e *PollerTimeoutError) Error() string { + return fmt.Sprintf( + "operation %s did not complete within %d attempts", + e.OperationID, e.Attempts, + ) +} + +// --------------------------------------------------------------------------- +// GetJobFunc — callback type for fetching job state +// --------------------------------------------------------------------------- + +// GetJobFunc fetches the current state of a generation job by operation ID. +type GetJobFunc func(ctx context.Context, operationID, apiVersion string) (*GenerationJob, error) + +// --------------------------------------------------------------------------- +// PollerOptions — configurable polling behavior +// --------------------------------------------------------------------------- + +// PollerOptions configures the polling interval and attempt limit. +type PollerOptions struct { + Interval time.Duration + MaxAttempts int +} + +// DefaultPollerOptions returns sensible defaults: 2 s interval, 300 attempts (~10 min). +func DefaultPollerOptions() PollerOptions { + return PollerOptions{ + Interval: 2 * time.Second, + MaxAttempts: 300, + } +} + +// --------------------------------------------------------------------------- +// Poller — polls a generation job until it reaches a terminal state +// --------------------------------------------------------------------------- + +// Poller polls a GenerationJob until it reaches a terminal status. +type Poller struct { + OperationID string + APIVersion string + GetJob GetJobFunc + Options PollerOptions + // OnPoll is called after each successful poll with the latest status. + // Callers can use this for progress reporting (e.g. debug logging). + OnPoll func(status JobStatus) +} + +// NewPoller creates a Poller with default options. +func NewPoller(operationID, apiVersion string, getJob GetJobFunc) *Poller { + return &Poller{ + OperationID: operationID, + APIVersion: apiVersion, + GetJob: getJob, + Options: DefaultPollerOptions(), + } +} + +// Poll blocks until the job reaches a terminal state, the context is +// cancelled, or the maximum number of attempts is exhausted. +// +// On success it returns the completed GenerationJob. +// On failure it returns a *JobFailedError (which wraps the job for inspection). +// On timeout it returns a plain error. +func (p *Poller) Poll(ctx context.Context) (*GenerationJob, error) { + if p.OperationID == "" { + return nil, fmt.Errorf("operation ID is empty") + } + + for range p.Options.MaxAttempts { + select { + case <-ctx.Done(): + return nil, ctx.Err() + case <-time.After(p.Options.Interval): + } + + job, err := p.GetJob(ctx, p.OperationID, p.APIVersion) + if err != nil { + if evalcore.IsTransientError(err) { + log.Printf("[poller] transient error polling %s, will retry: %v", p.OperationID, err) + continue + } + return nil, err + } + + status := ParseJobStatus(job.Status) + log.Printf("[poller] operationID=%s status=%s", p.OperationID, status) + + if p.OnPoll != nil { + p.OnPoll(status) + } + + if status.IsTerminal() { + if status.IsFailed() { + return nil, &JobFailedError{Job: job, Status: status} + } + return job, nil + } + } + + return nil, &PollerTimeoutError{ + OperationID: p.OperationID, + Attempts: p.Options.MaxAttempts, + } +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/portal_urls.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/portal_urls.go new file mode 100644 index 00000000000..8b1ccd0fd5d --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/portal_urls.go @@ -0,0 +1,79 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package eval_api + +import ( + "encoding/base64" + "fmt" + "strings" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore/arm" + "github.com/google/uuid" +) + +// PortalPrefix holds the parsed project context needed to construct Foundry portal URLs. +type PortalPrefix struct { + prefix string // e.g. "https://ai.azure.com/nextgen/r/,,,," +} + +// NewPortalPrefix parses an ARM project resource ID and returns a PortalPrefix +// that can be reused to build multiple portal URLs. +// Returns an error if the resource ID is invalid or not a Foundry project. +func NewPortalPrefix(projectResourceID string) (*PortalPrefix, error) { + resourceID, err := arm.ParseResourceID(projectResourceID) + if err != nil { + return nil, fmt.Errorf("failed to parse project resource ID: %w", err) + } + + encodedSub, err := encodeSubscriptionForURL(resourceID.SubscriptionID) + if err != nil { + return nil, fmt.Errorf("failed to encode subscription ID: %w", err) + } + + if resourceID.Parent == nil || + !strings.Contains(string(resourceID.ResourceType.Type), "/") { + return nil, fmt.Errorf( + "resource ID does not represent a Foundry project (missing parent account): %s", + projectResourceID, + ) + } + + prefix := fmt.Sprintf( + "https://ai.azure.com/nextgen/r/%s,%s,,%s,%s", + encodedSub, resourceID.ResourceGroupName, + resourceID.Parent.Name, resourceID.Name, + ) + return &PortalPrefix{prefix: prefix}, nil +} + +// EvalRunURL returns the portal URL for an eval run report. +func (p *PortalPrefix) EvalRunURL(evalID, runID string) string { + return fmt.Sprintf("%s/build/evaluations/%s/run/%s", p.prefix, evalID, runID) +} + +// EvaluatorURL returns the portal URL for a generated evaluator. +func (p *PortalPrefix) EvaluatorURL(evaluatorName, version string) string { + return fmt.Sprintf("%s/build/evaluations/catalog/%s/%s", p.prefix, evaluatorName, version) +} + +// DatasetURL returns the portal URL for a dataset. +func (p *PortalPrefix) DatasetURL(datasetName, version string) string { + return fmt.Sprintf("%s/build/data/datasets/%s/%s", p.prefix, datasetName, version) +} + +// OptimizationURL returns the portal URL for an optimization job. +func (p *PortalPrefix) OptimizationURL(agentName, operationID string) string { + return fmt.Sprintf("%s/build/agents/%s/optimization/%s", + p.prefix, agentName, operationID) +} + +// encodeSubscriptionForURL encodes a subscription ID GUID as base64 without padding. +func encodeSubscriptionForURL(subscriptionID string) (string, error) { + guid, err := uuid.Parse(subscriptionID) + if err != nil { + return "", fmt.Errorf("invalid subscription ID format: %w", err) + } + guidBytes, _ := guid.MarshalBinary() + return strings.TrimRight(base64.URLEncoding.EncodeToString(guidBytes), "="), nil +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/evaluator.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/evaluator.go new file mode 100644 index 00000000000..0a4847ad4bd --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/evaluator.go @@ -0,0 +1,89 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package evalcore + +import ( + "fmt" + "strings" + + "go.yaml.in/yaml/v3" +) + +// BuiltinPrefix marks an evaluator provided by the platform. The prefix is +// stripped before the name is sent as testing_criteria[].evaluator_name. +const BuiltinPrefix = "builtin." + +// EvaluatorRef references an evaluator from an eval group. It accepts either a +// bare string or a mapping carrying a pass threshold: +// +// evaluators: +// - builtin.task_adherence +// - { name: support-quality, threshold: 4.0 } +type EvaluatorRef struct { + Name string `yaml:"name" json:"name"` + Version string `yaml:"version,omitempty" json:"version,omitempty"` + // Threshold maps to testing_criteria[].initialization_parameters.threshold. + Threshold *float64 `yaml:"threshold,omitempty" json:"threshold,omitempty"` +} + +// IsBuiltin reports whether the reference names a platform evaluator, which +// needs no declaration and is never uploaded. +func (e EvaluatorRef) IsBuiltin() bool { + return strings.HasPrefix(e.Name, BuiltinPrefix) +} + +// APIName is the name the service expects, with the builtin prefix removed. +func (e EvaluatorRef) APIName() string { + return strings.TrimPrefix(e.Name, BuiltinPrefix) +} + +// EvaluatorList is a sequence of EvaluatorRef supporting mixed string and +// mapping entries. +type EvaluatorList []EvaluatorRef + +func (el *EvaluatorList) UnmarshalYAML(value *yaml.Node) error { + if value.Kind != yaml.SequenceNode { + return fmt.Errorf("evaluators must be a sequence, got %v", value.Kind) + } + + result := make([]EvaluatorRef, 0, len(value.Content)) + for _, node := range value.Content { + switch node.Kind { + case yaml.ScalarNode: + var name string + if err := node.Decode(&name); err != nil { + return fmt.Errorf("decoding evaluator name: %w", err) + } + result = append(result, EvaluatorRef{Name: name}) + case yaml.MappingNode: + var ref EvaluatorRef + if err := node.Decode(&ref); err != nil { + return fmt.Errorf("decoding evaluator: %w", err) + } + if ref.Name == "" { + return fmt.Errorf("evaluator entry is missing 'name'") + } + result = append(result, ref) + default: + return fmt.Errorf("evaluator entry must be a string or a mapping, got %v", node.Kind) + } + } + + *el = result + return nil +} + +// MarshalYAML emits the compact string form when an entry carries nothing but a +// name, so round-tripping a hand-written config does not rewrite it. +func (el EvaluatorList) MarshalYAML() (any, error) { + out := make([]any, 0, len(el)) + for _, ref := range el { + if ref.Threshold == nil && ref.Version == "" { + out = append(out, ref.Name) + continue + } + out = append(out, ref) + } + return out, nil +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/transient.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/transient.go new file mode 100644 index 00000000000..029af83ffc0 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/transient.go @@ -0,0 +1,29 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package evalcore + +import ( + "errors" + "strings" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore" +) + +// IsTransientError reports whether err is worth retrying: throttling, a server +// fault, or a dropped connection. +func IsTransientError(err error) bool { + if err == nil { + return false + } + + var respErr *azcore.ResponseError + if errors.As(err, &respErr) { + return respErr.StatusCode == 429 || respErr.StatusCode >= 500 + } + + msg := err.Error() + return strings.Contains(msg, "connection reset") || + strings.Contains(msg, "connection refused") || + strings.Contains(msg, "EOF") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/version/version.go b/cli/azd/extensions/azure.ai.evaluations/internal/version/version.go new file mode 100644 index 00000000000..e7279d11fba --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/version/version.go @@ -0,0 +1,11 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package version + +var ( + // Populated at build time. + Version = "dev" + Commit = "none" + BuildDate = "unknown" +) From 2a8dbed08b945a270530af0360a198f5907a5d8d Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 28 Jul 2026 05:02:58 +0300 Subject: [PATCH 003/320] feat(azure.ai.evaluations): config model, client context, and dataset commands - EvalConfig and GenerateConfig types with validation covering cross-references, duplicate names, unsupported target types, and evaluation levels - ResolveGroup picks the only group or errors with the available names - ArtifactPath accepts a directory or an explicit file path for local_dir - evalContext resolves the project endpoint (flag, azd env, host env) and builds both clients against the azd developer CLI credential - dataset create/update/list/show/delete with -o json - Tier-0 tests for parsing, validation, group resolution, and path handling --- .../internal/cmd/apiversions.go | 17 ++ .../internal/cmd/context.go | 145 ++++++++++ .../internal/cmd/dataset.go | 253 ++++++++++++++++++ .../internal/cmd/output.go | 72 +++++ .../azure.ai.evaluations/internal/cmd/root.go | 2 + .../internal/pkg/dataset_api/list.go | 91 +++++++ .../internal/project/eval_config.go | 222 +++++++++++++++ .../internal/project/eval_config_test.go | 235 ++++++++++++++++ .../internal/project/generate_config.go | 163 +++++++++++ 9 files changed, 1200 insertions(+) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/apiversions.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/context.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/output.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/list.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_test.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/project/generate_config.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/apiversions.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/apiversions.go new file mode 100644 index 00000000000..89b1d125f34 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/apiversions.go @@ -0,0 +1,17 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +// API versions used by the Foundry data plane. +const ( + // ProjectEndpointAPIVersion covers datasets, evaluators, and evaluator + // generation jobs on the project endpoint. + ProjectEndpointAPIVersion = "2025-11-15-preview" + + // DataGenerationAPIVersion covers dataset generation jobs. + DataGenerationAPIVersion = "v1" + + // OpenAI-compatible eval group and run calls send no api-version, so there + // is deliberately no constant for them. +) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/context.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/context.go new file mode 100644 index 00000000000..f9ba6d0ec8d --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/context.go @@ -0,0 +1,145 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "context" + "fmt" + "os" + "strings" + + "azureaieval/internal/pkg/dataset_api" + "azureaieval/internal/pkg/eval_api" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore" + "github.com/Azure/azure-sdk-for-go/sdk/azidentity" + "github.com/azure/azure-dev/cli/azd/pkg/azdext" +) + +// projectEndpointEnvKey is the azd environment key holding the Foundry project +// endpoint the data-plane clients target. +const projectEndpointEnvKey = "FOUNDRY_PROJECT_ENDPOINT" + +// evalContext carries everything the commands need to reach the data plane. +type evalContext struct { + azdClient *azdext.AzdClient + endpoint string + envName string + cred azcore.TokenCredential + + evalClient *eval_api.EvalClient + datasetClient *dataset_api.DatasetClient +} + +// newEvalContext resolves the project endpoint and builds the data-plane +// clients. Endpoint resolution order: +// +// 1. --project-endpoint +// 2. the active azd environment's FOUNDRY_PROJECT_ENDPOINT +// 3. the host environment variable of the same name +func newEvalContext(ctx context.Context, endpointFlag string) (*evalContext, error) { + azdClient, err := azdext.NewAzdClient() + if err != nil { + return nil, fmt.Errorf("connecting to azd: %w", err) + } + + ec := &evalContext{azdClient: azdClient} + + if endpointFlag != "" { + ec.endpoint = endpointFlag + } else { + ec.endpoint, ec.envName = lookupEndpointFromAzd(ctx, azdClient) + } + if ec.endpoint == "" { + ec.endpoint = os.Getenv(projectEndpointEnvKey) + } + if ec.endpoint == "" { + return nil, fmt.Errorf( + "no Foundry project endpoint found; pass --project-endpoint or set %s "+ + "in the azd environment (azd env set %s )", + projectEndpointEnvKey, projectEndpointEnvKey) + } + ec.endpoint = strings.TrimSuffix(ec.endpoint, "/") + + cred, err := azidentity.NewAzureDeveloperCLICredential( + &azidentity.AzureDeveloperCLICredentialOptions{}, + ) + if err != nil { + return nil, fmt.Errorf("creating Azure credential: %w", err) + } + ec.cred = cred + + ec.evalClient = eval_api.NewEvalClient(ec.endpoint, cred) + ec.datasetClient = dataset_api.NewDatasetClient(ec.endpoint, cred) + + return ec, nil +} + +// lookupEndpointFromAzd reads the endpoint from the active azd environment, +// returning empty strings when azd has no current environment. +func lookupEndpointFromAzd(ctx context.Context, azdClient *azdext.AzdClient) (endpoint, envName string) { + envResp, err := azdClient.Environment().GetCurrent(ctx, &azdext.EmptyRequest{}) + if err != nil || envResp == nil || envResp.Environment == nil { + return "", "" + } + val, err := azdClient.Environment().GetValue(ctx, &azdext.GetEnvRequest{ + EnvName: envResp.Environment.Name, + Key: projectEndpointEnvKey, + }) + if err != nil || val == nil || val.Value == "" { + return "", envResp.Environment.Name + } + return val.Value, envResp.Environment.Name +} + +// setEnvValue persists a value into the active azd environment. azd itself +// writes none of these keys — the extension owns them. +func (ec *evalContext) setEnvValue(ctx context.Context, key, value string) error { + if ec.envName == "" { + envResp, err := ec.azdClient.Environment().GetCurrent(ctx, &azdext.EmptyRequest{}) + if err != nil || envResp == nil || envResp.Environment == nil { + return fmt.Errorf("no active azd environment to write %s into", key) + } + ec.envName = envResp.Environment.Name + } + _, err := ec.azdClient.Environment().SetValue(ctx, &azdext.SetEnvRequest{ + EnvName: ec.envName, + Key: key, + Value: value, + }) + if err != nil { + return fmt.Errorf("writing %s to the azd environment: %w", key, err) + } + return nil +} + +// getEnvValue reads a value from the active azd environment, returning empty +// when it is unset. +func (ec *evalContext) getEnvValue(ctx context.Context, key string) string { + if ec.envName == "" { + return "" + } + val, err := ec.azdClient.Environment().GetValue(ctx, &azdext.GetEnvRequest{ + EnvName: ec.envName, + Key: key, + }) + if err != nil || val == nil { + return "" + } + return val.Value +} + +func (ec *evalContext) Close() { + if ec.azdClient != nil { + ec.azdClient.Close() + } +} + +// azd environment keys written by this extension. +const ( + envKeyEvalGroupID = "EVAL_GROUP_ID" + envKeyEvalRunID = "EVAL_RUN_ID" + envKeyDatasetVersion = "EVAL_DATASET_VERSION" + envKeyFingerprintPrefix = "EVAL_FINGERPRINT_" +) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset.go new file mode 100644 index 00000000000..41685ae7e18 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset.go @@ -0,0 +1,253 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "fmt" + "os" + "path/filepath" + "strings" + + "azureaieval/internal/pkg/dataset_api" + + "github.com/spf13/cobra" +) + +func newDatasetCommand() *cobra.Command { + cmd := &cobra.Command{ + Use: "dataset", + Short: "Manage evaluation datasets.", + } + cmd.AddCommand( + newDatasetCreateCommand(false), + newDatasetCreateCommand(true), + newDatasetListCommand(), + newDatasetShowCommand(), + newDatasetDeleteCommand(), + ) + return cmd +} + +// newDatasetCreateCommand builds `dataset create` and `dataset update`. Both +// publish a new immutable version; the server auto-increments. +func newDatasetCreateCommand(update bool) *cobra.Command { + var ( + name string + file string + version string + endpointFlg string + ) + + use, short := "create", "Register a dataset, creating its first version." + if update { + use, short = "update", "Publish a new version of an existing dataset." + } + + cmd := &cobra.Command{ + Use: use, + Short: short, + RunE: func(cmd *cobra.Command, args []string) error { + if name == "" { + return requireFlag("name") + } + if file == "" { + return requireFlag("file") + } + + info, err := os.Stat(file) + if err != nil { + return fmt.Errorf("reading --file %q: %w", file, err) + } + // The upload helper scans a directory for the first .jsonl, so pass + // the containing directory when given a file path. + localDir := file + if !info.IsDir() { + if !strings.EqualFold(filepath.Ext(file), ".jsonl") { + return fmt.Errorf("--file must be a .jsonl file or a directory containing one, got %q", file) + } + localDir = filepath.Dir(file) + } + + ctx := cmd.Context() + ec, err := newEvalContext(ctx, endpointFlg) + if err != nil { + return err + } + defer ec.Close() + + ds, err := ec.datasetClient.UploadNewVersion( + ctx, name, version, localDir, ProjectEndpointAPIVersion, + ) + if err != nil { + return fmt.Errorf("registering dataset %q: %w", name, err) + } + + if err := ec.setEnvValue(ctx, envKeyDatasetVersion, ds.Version); err != nil { + // Persisting is a convenience; do not fail the command over it. + fmt.Fprintf(cmd.ErrOrStderr(), "warning: %v\n", err) + } + + if isJSON(cmd) { + return emitJSON(cmd.OutOrStdout(), ds) + } + fmt.Fprintf(cmd.OutOrStdout(), "Registered dataset %s version %s\n", ds.Name, ds.Version) + return nil + }, + } + + cmd.Flags().StringVar(&name, "name", "", "Name of the dataset.") + cmd.Flags().StringVar(&file, "file", "", "Path to a .jsonl file, or a directory containing one.") + cmd.Flags().StringVar(&version, "version", "", + "Current version to increment from. Omit to let the server assign the next version.") + cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") + return cmd +} + +func newDatasetListCommand() *cobra.Command { + var ( + name string + endpointFlg string + ) + + cmd := &cobra.Command{ + Use: "list", + Short: "List registered datasets, or the versions of one dataset.", + RunE: func(cmd *cobra.Command, args []string) error { + ctx := cmd.Context() + ec, err := newEvalContext(ctx, endpointFlg) + if err != nil { + return err + } + defer ec.Close() + + var list *dataset_api.DatasetList + if name != "" { + list, err = ec.datasetClient.ListDatasetVersions(ctx, name, ProjectEndpointAPIVersion) + } else { + list, err = ec.datasetClient.ListDatasets(ctx, ProjectEndpointAPIVersion) + } + if err != nil { + return fmt.Errorf("listing datasets: %w", err) + } + + if isJSON(cmd) { + return emitJSON(cmd.OutOrStdout(), list) + } + rows := make([][]string, 0, len(list.Value)) + for _, d := range list.Value { + rows = append(rows, []string{d.Name, d.Version, d.Format}) + } + if len(rows) == 0 { + fmt.Fprintln(cmd.OutOrStdout(), "No datasets found.") + return nil + } + return emitTable(cmd.OutOrStdout(), []string{"NAME", "VERSION", "FORMAT"}, rows) + }, + } + + cmd.Flags().StringVar(&name, "name", "", "Limit the listing to versions of this dataset.") + cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") + return cmd +} + +func newDatasetShowCommand() *cobra.Command { + var ( + name string + version string + endpointFlg string + ) + + cmd := &cobra.Command{ + Use: "show", + Short: "Show a dataset version.", + RunE: func(cmd *cobra.Command, args []string) error { + if name == "" { + return requireFlag("name") + } + + ctx := cmd.Context() + ec, err := newEvalContext(ctx, endpointFlg) + if err != nil { + return err + } + defer ec.Close() + + if version == "" { + list, err := ec.datasetClient.ListDatasetVersions(ctx, name, ProjectEndpointAPIVersion) + if err != nil { + return fmt.Errorf("resolving the latest version of %q: %w", name, err) + } + if len(list.Value) == 0 { + return fmt.Errorf("dataset %q has no versions", name) + } + version = dataset_api.LatestVersion(list.Value) + } + + ds, err := ec.datasetClient.GetDataset(ctx, name, version, ProjectEndpointAPIVersion) + if err != nil { + return fmt.Errorf("reading dataset %q version %q: %w", name, version, err) + } + + if isJSON(cmd) { + return emitJSON(cmd.OutOrStdout(), ds) + } + return emitTable(cmd.OutOrStdout(), + []string{"NAME", "VERSION", "FORMAT", "URI"}, + [][]string{{ds.Name, ds.Version, ds.Format, ds.ResolvedBlobURI()}}, + ) + }, + } + + cmd.Flags().StringVar(&name, "name", "", "Name of the dataset.") + cmd.Flags().StringVar(&version, "version", "", "Version to show. Omit for the latest.") + cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") + return cmd +} + +func newDatasetDeleteCommand() *cobra.Command { + var ( + name string + version string + endpointFlg string + ) + + cmd := &cobra.Command{ + Use: "delete", + Short: "Delete a dataset version.", + RunE: func(cmd *cobra.Command, args []string) error { + if name == "" { + return requireFlag("name") + } + if version == "" { + return requireFlag("version") + } + + ctx := cmd.Context() + ec, err := newEvalContext(ctx, endpointFlg) + if err != nil { + return err + } + defer ec.Close() + + if err := ec.datasetClient.DeleteDatasetVersion( + ctx, name, version, ProjectEndpointAPIVersion, + ); err != nil { + return fmt.Errorf("deleting dataset %q version %q: %w", name, version, err) + } + + if isJSON(cmd) { + return emitJSON(cmd.OutOrStdout(), map[string]string{ + "name": name, "version": version, "status": "deleted", + }) + } + fmt.Fprintf(cmd.OutOrStdout(), "Deleted dataset %s version %s\n", name, version) + return nil + }, + } + + cmd.Flags().StringVar(&name, "name", "", "Name of the dataset.") + cmd.Flags().StringVar(&version, "version", "", "Version to delete.") + cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") + return cmd +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/output.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/output.go new file mode 100644 index 00000000000..ee6a1928fdb --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/output.go @@ -0,0 +1,72 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "encoding/json" + "fmt" + "io" + "strings" + "text/tabwriter" + + "github.com/spf13/cobra" +) + +const outputJSON = "json" + +// outputFormat reads the inherited -o/--output flag. +func outputFormat(cmd *cobra.Command) string { + if cmd == nil { + return "" + } + v, err := cmd.Flags().GetString("output") + if err != nil { + return "" + } + return strings.ToLower(v) +} + +// isJSON reports whether the command should emit machine-readable output. +func isJSON(cmd *cobra.Command) bool { + return outputFormat(cmd) == outputJSON +} + +// emitJSON writes v as indented JSON. +func emitJSON(w io.Writer, v any) error { + enc := json.NewEncoder(w) + enc.SetIndent("", " ") + return enc.Encode(v) +} + +// emitTable writes a simple aligned table. Rows must match the header width. +func emitTable(w io.Writer, headers []string, rows [][]string) error { + tw := tabwriter.NewWriter(w, 0, 0, 3, ' ', 0) + if _, err := fmt.Fprintln(tw, strings.Join(headers, "\t")); err != nil { + return err + } + for _, row := range rows { + if _, err := fmt.Fprintln(tw, strings.Join(row, "\t")); err != nil { + return err + } + } + return tw.Flush() +} + +// noPrompt reports whether the command must run without any interaction. +func noPrompt(cmd *cobra.Command) bool { + if cmd == nil { + return false + } + v, err := cmd.Flags().GetBool("no-prompt") + if err != nil { + return false + } + return v +} + +// requireFlag returns an error naming the missing flag, used when --no-prompt +// prevents asking for a required value. +func requireFlag(name string) error { + return fmt.Errorf("--%s is required (running with --no-prompt)", name) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/root.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/root.go index 6654435f055..9901d141836 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/root.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/root.go @@ -25,5 +25,7 @@ func NewRootCommand() *cobra.Command { rootCmd.SilenceErrors = true rootCmd.CompletionOptions.DisableDefaultCmd = true + rootCmd.AddCommand(newDatasetCommand()) + return rootCmd } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/list.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/list.go new file mode 100644 index 00000000000..f81f372131d --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/list.go @@ -0,0 +1,91 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package dataset_api + +import ( + "context" + "fmt" + "net/http" + "net/url" + "strconv" + "strings" +) + +// DatasetList is the paged response returned when listing datasets or the +// versions of one dataset. +type DatasetList struct { + Value []Dataset `json:"value"` + NextLink string `json:"nextLink,omitempty"` +} + +// ListDatasets returns the datasets registered on the project. +func (c *DatasetClient) ListDatasets(ctx context.Context, apiVersion string) (*DatasetList, error) { + return doRequestTyped[DatasetList](c, ctx, http.MethodGet, pathDatasets, nil, nil, apiVersion) +} + +// ListDatasetVersions returns every version of a single dataset. +func (c *DatasetClient) ListDatasetVersions( + ctx context.Context, + name string, + apiVersion string, +) (*DatasetList, error) { + path := fmt.Sprintf("%s/%s/versions", pathDatasets, url.PathEscape(name)) + return doRequestTyped[DatasetList](c, ctx, http.MethodGet, path, nil, nil, apiVersion) +} + +// DeleteDatasetVersion removes a single dataset version. +func (c *DatasetClient) DeleteDatasetVersion( + ctx context.Context, + name string, + version string, + apiVersion string, +) error { + path := fmt.Sprintf( + "%s/%s/versions/%s", + pathDatasets, url.PathEscape(name), url.PathEscape(version), + ) + _, err := c.doRequest(ctx, http.MethodDelete, path, nil, nil, apiVersion) + return err +} + +// VersionOrder returns a sortable value for a version string, matching the +// decimal convention NextVersion produces ("1.0", "2.0"). Unparseable versions +// sort lowest. +func VersionOrder(version string) float64 { + v := strings.TrimSpace(version) + if v == "" { + return -1 + } + if f, err := strconv.ParseFloat(v, 64); err == nil { + return f + } + // Fall back to trailing digits, e.g. "v3" -> 3. + i := len(v) + for i > 0 && v[i-1] >= '0' && v[i-1] <= '9' { + i-- + } + if i == len(v) { + return -1 + } + if n, err := strconv.Atoi(v[i:]); err == nil { + return float64(n) + } + return -1 +} + +// LatestVersion returns the highest version in the list, falling back to the +// last entry when none of the versions can be ordered. +func LatestVersion(datasets []Dataset) string { + best := "" + bestOrder := -2.0 + for _, d := range datasets { + if o := VersionOrder(d.Version); o > bestOrder { + bestOrder, best = o, d.Version + } + } + if best == "" && len(datasets) > 0 { + return datasets[len(datasets)-1].Version + } + return best +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go new file mode 100644 index 00000000000..37a0777448a --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go @@ -0,0 +1,222 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +// Package project models the eval configuration carried by the +// `host: azure.ai.eval` service entry in azure.yaml. +package project + +import ( + "fmt" + "os" + "strings" + + "azureaieval/internal/pkg/evalcore" + + "go.yaml.in/yaml/v3" +) + +// EvalConfig is the deployment spec — the body of the azure.ai.eval service +// entry, normally kept in evals/azure.yaml and pulled in with $ref. +type EvalConfig struct { + Evaluators []EvaluatorDecl `yaml:"evaluators,omitempty" json:"evaluators,omitempty"` + Datasets []DatasetDecl `yaml:"datasets,omitempty" json:"datasets,omitempty"` + EvalGroups []EvalGroup `yaml:"evalGroups,omitempty" json:"evalGroups,omitempty"` +} + +// DatasetDecl declares a dataset. A local Source is uploaded on deploy; without +// one the name must already resolve to a registered dataset. +type DatasetDecl struct { + Name string `yaml:"name" json:"name"` + Source string `yaml:"source,omitempty" json:"source,omitempty"` + Version string `yaml:"version,omitempty" json:"version,omitempty"` +} + +// EvaluatorDecl declares a custom evaluator. Built-ins are referenced directly +// from an eval group and never declared here. +type EvaluatorDecl struct { + Name string `yaml:"name" json:"name"` + Source string `yaml:"source,omitempty" json:"source,omitempty"` + Version string `yaml:"version,omitempty" json:"version,omitempty"` +} + +// EvalGroup is a run definition: evaluators plus options, bound to a dataset. +type EvalGroup struct { + Name string `yaml:"name" json:"name"` + ID string `yaml:"id,omitempty" json:"id,omitempty"` + Description string `yaml:"description,omitempty" json:"description,omitempty"` + Dataset string `yaml:"dataset,omitempty" json:"dataset,omitempty"` + Evaluators evalcore.EvaluatorList `yaml:"evaluators,omitempty" json:"evaluators,omitempty"` + Target *Target `yaml:"target,omitempty" json:"target,omitempty"` + Options *Options `yaml:"options,omitempty" json:"options,omitempty"` +} + +// Target names what the run invokes. Only type "agent" is supported today. +type Target struct { + Type string `yaml:"type" json:"type"` + Name string `yaml:"name" json:"name"` +} + +const TargetTypeAgent = "agent" + +// Options are run settings carried on the group. +type Options struct { + EvalModel string `yaml:"eval_model,omitempty" json:"eval_model,omitempty"` + MaxSamples int `yaml:"max_samples,omitempty" json:"max_samples,omitempty"` + EvaluationLevel string `yaml:"evaluation_level,omitempty" json:"evaluation_level,omitempty"` +} + +// Evaluation levels accepted by the service. The service default is turn. +const ( + EvaluationLevelTurn = "turn" + EvaluationLevelConversation = "conversation" +) + +// LoadEvalConfig reads a deployment spec from disk. The path is used verbatim, +// relative to the process working directory — never re-rooted. +func LoadEvalConfig(path string) (*EvalConfig, error) { + data, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("reading eval config %q: %w", path, err) + } + + var cfg EvalConfig + if err := yaml.Unmarshal(data, &cfg); err != nil { + return nil, fmt.Errorf("parsing eval config %q: %w", path, err) + } + return &cfg, nil +} + +// Validate checks the invariants the provider relies on before it calls the +// service, so failures surface as config errors rather than opaque 4xx. +func (c *EvalConfig) Validate() error { + datasets := map[string]bool{} + for i, d := range c.Datasets { + if d.Name == "" { + return fmt.Errorf("datasets[%d]: 'name' is required", i) + } + if datasets[d.Name] { + return fmt.Errorf("datasets[%d]: duplicate dataset name %q", i, d.Name) + } + datasets[d.Name] = true + } + + evaluators := map[string]bool{} + for i, e := range c.Evaluators { + if e.Name == "" { + return fmt.Errorf("evaluators[%d]: 'name' is required", i) + } + if strings.HasPrefix(e.Name, evalcore.BuiltinPrefix) { + return fmt.Errorf( + "evaluators[%d]: built-in evaluator %q must not be declared; "+ + "reference it directly from an eval group", i, e.Name) + } + if evaluators[e.Name] { + return fmt.Errorf("evaluators[%d]: duplicate evaluator name %q", i, e.Name) + } + evaluators[e.Name] = true + } + + groups := map[string]bool{} + for i, g := range c.EvalGroups { + if g.Name == "" { + return fmt.Errorf("evalGroups[%d]: 'name' is required", i) + } + if groups[g.Name] { + return fmt.Errorf("evalGroups[%d]: duplicate eval group name %q", i, g.Name) + } + groups[g.Name] = true + + if g.Dataset != "" && !datasets[g.Dataset] { + return fmt.Errorf( + "evalGroups[%d] (%s): dataset %q is not declared in datasets", + i, g.Name, g.Dataset) + } + if len(g.Evaluators) == 0 { + return fmt.Errorf("evalGroups[%d] (%s): at least one evaluator is required", i, g.Name) + } + for _, ref := range g.Evaluators { + if ref.IsBuiltin() { + continue + } + if !evaluators[ref.Name] { + return fmt.Errorf( + "evalGroups[%d] (%s): evaluator %q is not declared in evaluators "+ + "(built-ins need the %q prefix)", + i, g.Name, ref.Name, evalcore.BuiltinPrefix) + } + } + if g.Target != nil && g.Target.Type != "" && g.Target.Type != TargetTypeAgent { + return fmt.Errorf( + "evalGroups[%d] (%s): target.type %q is not supported; only %q is available today", + i, g.Name, g.Target.Type, TargetTypeAgent) + } + if g.Options != nil { + switch g.Options.EvaluationLevel { + case "", EvaluationLevelTurn, EvaluationLevelConversation: + default: + return fmt.Errorf( + "evalGroups[%d] (%s): evaluation_level %q is invalid; expected %q or %q", + i, g.Name, g.Options.EvaluationLevel, + EvaluationLevelTurn, EvaluationLevelConversation) + } + } + } + + return nil +} + +// Dataset returns the declaration with the given name. +func (c *EvalConfig) Dataset(name string) (*DatasetDecl, bool) { + for i := range c.Datasets { + if c.Datasets[i].Name == name { + return &c.Datasets[i], true + } + } + return nil, false +} + +// Evaluator returns the declaration with the given name. +func (c *EvalConfig) Evaluator(name string) (*EvaluatorDecl, bool) { + for i := range c.Evaluators { + if c.Evaluators[i].Name == name { + return &c.Evaluators[i], true + } + } + return nil, false +} + +// Group returns the eval group with the given name. +func (c *EvalConfig) Group(name string) (*EvalGroup, bool) { + for i := range c.EvalGroups { + if c.EvalGroups[i].Name == name { + return &c.EvalGroups[i], true + } + } + return nil, false +} + +// ResolveGroup picks the group to act on: the named one, or the only one when +// the config declares exactly one. +func (c *EvalConfig) ResolveGroup(name string) (*EvalGroup, error) { + if name != "" { + g, ok := c.Group(name) + if !ok { + return nil, fmt.Errorf("eval group %q is not declared in the config", name) + } + return g, nil + } + switch len(c.EvalGroups) { + case 0: + return nil, fmt.Errorf("no eval groups are declared in the config") + case 1: + return &c.EvalGroups[0], nil + default: + names := make([]string, 0, len(c.EvalGroups)) + for _, g := range c.EvalGroups { + names = append(names, g.Name) + } + return nil, fmt.Errorf( + "the config declares %d eval groups (%s); choose one with --eval-group", + len(c.EvalGroups), strings.Join(names, ", ")) + } +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_test.go new file mode 100644 index 00000000000..172154ce714 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_test.go @@ -0,0 +1,235 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package project + +import ( + "os" + "path/filepath" + "testing" + + "azureaieval/internal/pkg/evalcore" + + "github.com/stretchr/testify/require" + "go.yaml.in/yaml/v3" +) + +const sampleDeployConfig = ` +evaluators: + - name: support-quality + source: ./evaluators/support-quality/rubric_dimensions.json + - name: safety-check + source: ./evaluators/safety-check.json + +datasets: + - name: support-golden + source: ./datasets/support-golden.jsonl + version: "1" + +evalGroups: + - name: pr-gate + description: Quality gate for the support agent + dataset: support-golden + evaluators: + - builtin.task_adherence + - { name: support-quality, threshold: 4.0 } + - safety-check + target: + type: agent + name: support-agent + options: + eval_model: gpt-4.1-nano + max_samples: 100 + evaluation_level: conversation +` + +func loadFromString(t *testing.T, body string) *EvalConfig { + t.Helper() + path := filepath.Join(t.TempDir(), "azure.yaml") + require.NoError(t, os.WriteFile(path, []byte(body), 0o600)) + cfg, err := LoadEvalConfig(path) + require.NoError(t, err) + return cfg +} + +func TestLoadEvalConfig_ParsesAllSections(t *testing.T) { + cfg := loadFromString(t, sampleDeployConfig) + + require.Len(t, cfg.Evaluators, 2) + require.Len(t, cfg.Datasets, 1) + require.Len(t, cfg.EvalGroups, 1) + + ds, ok := cfg.Dataset("support-golden") + require.True(t, ok) + require.Equal(t, "./datasets/support-golden.jsonl", ds.Source) + require.Equal(t, "1", ds.Version) + + g, ok := cfg.Group("pr-gate") + require.True(t, ok) + require.Equal(t, "support-golden", g.Dataset) + require.Equal(t, TargetTypeAgent, g.Target.Type) + require.Equal(t, "support-agent", g.Target.Name) + require.Equal(t, EvaluationLevelConversation, g.Options.EvaluationLevel) +} + +// Evaluator entries accept a bare string or a mapping carrying a threshold. +func TestEvaluatorList_MixedForms(t *testing.T) { + cfg := loadFromString(t, sampleDeployConfig) + g, ok := cfg.Group("pr-gate") + require.True(t, ok) + require.Len(t, g.Evaluators, 3) + + require.Equal(t, "builtin.task_adherence", g.Evaluators[0].Name) + require.True(t, g.Evaluators[0].IsBuiltin()) + require.Equal(t, "task_adherence", g.Evaluators[0].APIName(), + "the builtin prefix must be stripped before it reaches the service") + require.Nil(t, g.Evaluators[0].Threshold) + + require.Equal(t, "support-quality", g.Evaluators[1].Name) + require.False(t, g.Evaluators[1].IsBuiltin()) + require.NotNil(t, g.Evaluators[1].Threshold) + require.InDelta(t, 4.0, *g.Evaluators[1].Threshold, 0.0001) + + require.Equal(t, "safety-check", g.Evaluators[2].Name) + require.Nil(t, g.Evaluators[2].Threshold) +} + +// Round-tripping must not rewrite bare names into mappings. +func TestEvaluatorList_RoundTripKeepsCompactForm(t *testing.T) { + threshold := 4.0 + list := evalcore.EvaluatorList{ + {Name: "builtin.relevance"}, + {Name: "support-quality", Threshold: &threshold}, + } + + out, err := yaml.Marshal(list) + require.NoError(t, err) + + var back evalcore.EvaluatorList + require.NoError(t, yaml.Unmarshal(out, &back)) + require.Len(t, back, 2) + require.Equal(t, "builtin.relevance", back[0].Name) + require.Nil(t, back[0].Threshold) + require.NotNil(t, back[1].Threshold) + require.Contains(t, string(out), "- builtin.relevance", + "an evaluator with only a name should stay a plain string") +} + +func TestValidate_Accepts(t *testing.T) { + require.NoError(t, loadFromString(t, sampleDeployConfig).Validate()) +} + +func TestValidate_Rejects(t *testing.T) { + cases := []struct { + name string + body string + wantErr string + }{ + { + name: "dataset referenced but not declared", + body: "evalGroups:\n - name: g\n dataset: missing\n evaluators: [builtin.relevance]\n", + wantErr: "is not declared in datasets", + }, + { + name: "custom evaluator referenced but not declared", + body: "datasets:\n - name: d\n" + + "evalGroups:\n - name: g\n dataset: d\n evaluators: [not-declared]\n", + wantErr: "is not declared in evaluators", + }, + { + name: "built-in declared as a custom evaluator", + body: "evaluators:\n - name: builtin.relevance\n", + wantErr: "must not be declared", + }, + { + name: "group without evaluators", + body: "evalGroups:\n - name: g\n evaluators: []\n", + wantErr: "at least one evaluator is required", + }, + { + name: "unsupported target type", + body: "evalGroups:\n - name: g\n evaluators: [builtin.relevance]\n target:\n type: model\n", + wantErr: "is not supported", + }, + { + name: "invalid evaluation level", + body: "evalGroups:\n - name: g\n evaluators: [builtin.relevance]\n" + + " options:\n evaluation_level: sentence\n", + wantErr: "evaluation_level", + }, + { + name: "duplicate dataset", + body: "datasets:\n - name: d\n - name: d\n", + wantErr: "duplicate dataset name", + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + err := loadFromString(t, tc.body).Validate() + require.Error(t, err) + require.Contains(t, err.Error(), tc.wantErr) + }) + } +} + +func TestResolveGroup(t *testing.T) { + single := loadFromString(t, sampleDeployConfig) + + t.Run("only group is used when unnamed", func(t *testing.T) { + g, err := single.ResolveGroup("") + require.NoError(t, err) + require.Equal(t, "pr-gate", g.Name) + }) + + t.Run("named group", func(t *testing.T) { + g, err := single.ResolveGroup("pr-gate") + require.NoError(t, err) + require.Equal(t, "pr-gate", g.Name) + }) + + t.Run("unknown name is an error", func(t *testing.T) { + _, err := single.ResolveGroup("nope") + require.ErrorContains(t, err, "is not declared") + }) + + t.Run("ambiguous without a name", func(t *testing.T) { + multi := loadFromString(t, + "evalGroups:\n - name: pr-gate\n evaluators: [builtin.relevance]\n"+ + " - name: nightly\n evaluators: [builtin.relevance]\n") + _, err := multi.ResolveGroup("") + require.ErrorContains(t, err, "--eval-group") + require.ErrorContains(t, err, "nightly") + }) + + t.Run("empty config", func(t *testing.T) { + _, err := (&EvalConfig{}).ResolveGroup("") + require.ErrorContains(t, err, "no eval groups") + }) +} + +// local_dir accepts a directory or an explicit file path. +func TestArtifactPath(t *testing.T) { + cases := []struct { + name string + localDir string + resource string + ext string + want string + }{ + {"directory derives the file name", "datasets", "support-golden", ".jsonl", + filepath.Join("base", "datasets", "support-golden.jsonl")}, + {"explicit file path is used as-is", "generated/datasets/support-golden.jsonl", "ignored", ".jsonl", + filepath.Join("base", "generated", "datasets", "support-golden.jsonl")}, + {"empty local_dir falls back to the base", "", "support-quality", ".json", + filepath.Join("base", "support-quality.json")}, + {"yaml rubric file path", "generated/rubrics/quality.yaml", "ignored", ".json", + filepath.Join("base", "generated", "rubrics", "quality.yaml")}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + require.Equal(t, tc.want, ArtifactPath("base", tc.localDir, tc.resource, tc.ext)) + }) + } +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/generate_config.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/generate_config.go new file mode 100644 index 00000000000..4e1d647387d --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/generate_config.go @@ -0,0 +1,163 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package project + +import ( + "fmt" + "os" + "path/filepath" + "strings" + + "go.yaml.in/yaml/v3" +) + +// Conventional locations. Both are relative to the working directory and are +// used verbatim — never re-rooted under the agent or project directory. +const ( + DefaultEvalDir = "evals" + DefaultGenerateConfig = "evals/eval_generate.yaml" + DefaultDeployConfig = "evals/azure.yaml" + DefaultDatasetsDir = "datasets" + DefaultEvaluatorsDir = "evaluators" +) + +// GenerateConfig is the generation spec — input to `azd ai eval generate`. It +// is never deployed. +type GenerateConfig struct { + Agent AgentSpec `yaml:"agent" json:"agent"` + Generate GenerateSpec `yaml:"generate" json:"generate"` +} + +// AgentSpec identifies the agent and the context the generator reads. +type AgentSpec struct { + Name string `yaml:"name" json:"name"` + Context AgentContext `yaml:"context,omitempty" json:"context,omitempty"` +} + +// AgentContext points at the material used to synthesize a rubric and dataset. +type AgentContext struct { + Instructions string `yaml:"instructions,omitempty" json:"instructions,omitempty"` + Tools string `yaml:"tools,omitempty" json:"tools,omitempty"` + Traces *TraceSpec `yaml:"traces,omitempty" json:"traces,omitempty"` +} + +// TraceSpec seeds rubric generation from recent traces. Traces are a generation +// input only; they cannot be a run's data source. +type TraceSpec struct { + Source string `yaml:"source,omitempty" json:"source,omitempty"` + Window string `yaml:"window,omitempty" json:"window,omitempty"` + Sample int `yaml:"sample,omitempty" json:"sample,omitempty"` +} + +// GenerateSpec configures what gets produced. +type GenerateSpec struct { + Rubric *RubricSpec `yaml:"rubric,omitempty" json:"rubric,omitempty"` + Dataset *DatasetSpec `yaml:"dataset,omitempty" json:"dataset,omitempty"` +} + +// RubricSpec configures rubric (LLM-graded evaluator) generation. +type RubricSpec struct { + Name string `yaml:"name" json:"name"` + Model string `yaml:"model,omitempty" json:"model,omitempty"` + LocalDir string `yaml:"local_dir,omitempty" json:"local_dir,omitempty"` +} + +// DatasetSpec configures synthetic dataset generation. +type DatasetSpec struct { + Name string `yaml:"name" json:"name"` + Strategy string `yaml:"strategy,omitempty" json:"strategy,omitempty"` + SampleSize int `yaml:"sampleSize,omitempty" json:"sampleSize,omitempty"` + LocalDir string `yaml:"local_dir,omitempty" json:"local_dir,omitempty"` +} + +// Generation strategies. +const ( + StrategySynthetic = "synthetic" + StrategyFromTraces = "from-traces" +) + +// Sample-count bounds enforced by the generation service. +const ( + MinSampleSize = 15 + MaxSampleSize = 1000 + DefaultSampleSize = 15 +) + +// LoadGenerateConfig reads a generation spec from disk. +func LoadGenerateConfig(path string) (*GenerateConfig, error) { + data, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("reading generation config %q: %w", path, err) + } + + var cfg GenerateConfig + if err := yaml.Unmarshal(data, &cfg); err != nil { + return nil, fmt.Errorf("parsing generation config %q: %w", path, err) + } + return &cfg, nil +} + +// Validate reports configuration errors before any generation job is submitted. +func (c *GenerateConfig) Validate() error { + if c.Agent.Name == "" { + return fmt.Errorf("agent.name is required") + } + if c.Generate.Rubric == nil && c.Generate.Dataset == nil { + return fmt.Errorf("generate must declare a rubric, a dataset, or both") + } + if r := c.Generate.Rubric; r != nil && r.Name == "" { + return fmt.Errorf("generate.rubric.name is required") + } + if d := c.Generate.Dataset; d != nil { + if d.Name == "" { + return fmt.Errorf("generate.dataset.name is required") + } + switch d.Strategy { + case "", StrategySynthetic, StrategyFromTraces: + default: + return fmt.Errorf( + "generate.dataset.strategy %q is invalid; expected %q or %q", + d.Strategy, StrategySynthetic, StrategyFromTraces) + } + if d.SampleSize != 0 && (d.SampleSize < MinSampleSize || d.SampleSize > MaxSampleSize) { + return fmt.Errorf( + "generate.dataset.sampleSize must be between %d and %d, got %d", + MinSampleSize, MaxSampleSize, d.SampleSize) + } + } + return nil +} + +// ArtifactPath resolves a local_dir value against baseDir. The value may be a +// directory, in which case the file name is derived from resourceName and ext, +// or an explicit file path, which is used as-is. +func ArtifactPath(baseDir, localDir, resourceName, ext string) string { + if localDir == "" { + return filepath.Join(baseDir, resourceName+ext) + } + candidate := localDir + if !filepath.IsAbs(candidate) { + candidate = filepath.Join(baseDir, candidate) + } + if looksLikeFile(localDir, ext) { + return candidate + } + return filepath.Join(candidate, resourceName+ext) +} + +// looksLikeFile treats a trailing recognised extension as an explicit file path. +func looksLikeFile(p, ext string) bool { + got := strings.ToLower(filepath.Ext(p)) + if got == "" { + return false + } + if got == strings.ToLower(ext) { + return true + } + switch got { + case ".json", ".jsonl", ".yaml", ".yml": + return true + } + return false +} From 125cd7a9b2462f722621fda773fd2edbc4fbf9cb Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 28 Jul 2026 05:05:40 +0300 Subject: [PATCH 004/320] feat(azure.ai.evaluations): eval group builder and run command - buildEvalGroupRequest maps evaluators to testing criteria, keeping the builtin prefix on evaluator_name while stripping it from name, and carries per-evaluator thresholds in initialization_parameters - run resolves the group from --eval-id, a pinned id, or the azd environment, creating it when absent, then binds the dataset to the run since the group has no dataset binding today - Local datasets are sent inline with optional truncation; registered datasets are referenced by id --- .../internal/cmd/build.go | 83 +++++ .../azure.ai.evaluations/internal/cmd/root.go | 5 +- .../azure.ai.evaluations/internal/cmd/run.go | 313 ++++++++++++++++++ 3 files changed, 400 insertions(+), 1 deletion(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/build.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build.go new file mode 100644 index 00000000000..77a3ef8918c --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build.go @@ -0,0 +1,83 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "azureaieval/internal/pkg/eval_api" + "azureaieval/internal/project" +) + +// dataMapping is the template binding the service uses to feed each evaluator. +// It pairs with the item schema below, which declares a single `query` field. +func dataMapping() map[string]string { + return map[string]string{ + "query": "{{item.query}}", + "response": "{{sample.output_items}}", + "tool_calls": "{{sample.tool_calls}}", + "tool_definitions": "{{sample.tool_definitions}}", + } +} + +// agentItemSchema mirrors the shape the agent-target runner expects. It is a +// fixed schema, not inferred from the dataset. +func agentItemSchema() map[string]any { + return map[string]any{ + "type": "object", + "properties": map[string]any{ + "query": map[string]any{"type": "string"}, + }, + } +} + +// buildEvalGroupRequest converts an eval group declaration into the create +// request. Evaluators become testing criteria; a per-evaluator threshold is +// carried in initialization_parameters alongside the judge model. +func buildEvalGroupRequest(group *project.EvalGroup) *eval_api.CreateOpenAIEvalRequest { + metadata := map[string]string{} + if group.Target != nil && group.Target.Name != "" { + metadata["azd_agent"] = group.Target.Name + } + metadata["azd_eval_group"] = group.Name + + req := &eval_api.CreateOpenAIEvalRequest{ + Name: group.Name, + Metadata: metadata, + DataSourceConfig: &eval_api.DataSourceConfig{ + Type: "custom", + IncludeSampleSchema: true, + ItemSchema: agentItemSchema(), + }, + } + + evalModel := "" + if group.Options != nil { + evalModel = group.Options.EvalModel + } + + for _, ref := range group.Evaluators { + criterion := eval_api.TestingCriterion{ + Type: "azure_ai_evaluator", + // Name drops the builtin prefix; EvaluatorName keeps it. + Name: ref.APIName(), + EvaluatorName: ref.Name, + DataMapping: dataMapping(), + } + + params := map[string]any{} + if evalModel != "" { + params["model"] = evalModel + params["deployment_name"] = evalModel + } + if ref.Threshold != nil { + params["threshold"] = *ref.Threshold + } + if len(params) > 0 { + criterion.InitializationParameters = params + } + + req.TestingCriteria = append(req.TestingCriteria, criterion) + } + + return req +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/root.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/root.go index 9901d141836..40f666b2f90 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/root.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/root.go @@ -25,7 +25,10 @@ func NewRootCommand() *cobra.Command { rootCmd.SilenceErrors = true rootCmd.CompletionOptions.DisableDefaultCmd = true - rootCmd.AddCommand(newDatasetCommand()) + rootCmd.AddCommand( + newDatasetCommand(), + newRunCommand(), + ) return rootCmd } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go new file mode 100644 index 00000000000..7085d5b7ad7 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go @@ -0,0 +1,313 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "bufio" + "context" + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + "time" + + "azureaieval/internal/pkg/eval_api" + "azureaieval/internal/project" + + "github.com/spf13/cobra" +) + +// Terminal run states reported by the service. +var terminalRunStates = map[string]bool{ + "completed": true, + "failed": true, + "canceled": true, + "cancelled": true, + "error": true, +} + +func newRunCommand() *cobra.Command { + var ( + configPath string + groupName string + evalID string + runName string + level string + maxSamples int + wait bool + endpointFlg string + ) + + cmd := &cobra.Command{ + Use: "run", + Short: "Run an evaluation, creating the eval group if it does not exist yet.", + RunE: func(cmd *cobra.Command, args []string) error { + ctx := cmd.Context() + out := cmd.OutOrStdout() + + ec, err := newEvalContext(ctx, endpointFlg) + if err != nil { + return err + } + defer ec.Close() + + // --eval-id bypasses the config entirely. + var group *project.EvalGroup + if evalID == "" { + cfg, err := project.LoadEvalConfig(configPath) + if err != nil { + return err + } + if err := cfg.Validate(); err != nil { + return err + } + group, err = cfg.ResolveGroup(groupName) + if err != nil { + return err + } + + evalID, err = ec.resolveEvalGroupID(ctx, group, out, isJSON(cmd)) + if err != nil { + return err + } + } + + dataSource, err := buildRunDataSource(group, configPath, maxSamples) + if err != nil { + return err + } + + if runName == "" { + base := "eval" + if group != nil { + base = group.Name + } + runName = fmt.Sprintf("%s-%s", base, time.Now().UTC().Format("20060102-150405")) + } + + metadata := map[string]string{} + if lvl := resolveLevel(level, group); lvl != "" { + metadata["evaluation_level"] = lvl + } + + run, err := ec.evalClient.CreateOpenAIEvalRun(ctx, evalID, &eval_api.CreateOpenAIEvalRunRequest{ + Name: runName, + DataSource: dataSource, + Metadata: metadata, + }) + if err != nil { + return fmt.Errorf("starting the evaluation run: %w", err) + } + + if err := ec.setEnvValue(ctx, envKeyEvalRunID, run.ID); err != nil { + fmt.Fprintf(cmd.ErrOrStderr(), "warning: %v\n", err) + } + + if !wait { + if isJSON(cmd) { + return emitJSON(out, run) + } + fmt.Fprintf(out, "Started run %s (status: %s)\n", run.ID, run.Status) + fmt.Fprintf(out, "Check progress with: azd ai eval results show %s --run-id %s\n", evalID, run.ID) + return nil + } + + final, err := ec.pollRun(ctx, evalID, run.ID, out, isJSON(cmd)) + if err != nil { + return err + } + + if isJSON(cmd) { + return emitJSON(out, final) + } + return renderRun(out, final) + }, + } + + cmd.Flags().StringVar(&configPath, "config", project.DefaultDeployConfig, + "Path to the eval deployment config.") + cmd.Flags().StringVar(&groupName, "eval-group", "", + "Which evalGroups entry to run. Defaults to the only one.") + cmd.Flags().StringVar(&evalID, "eval-id", "", + "Run against an existing eval group by id, ignoring the config.") + cmd.Flags().StringVar(&runName, "name", "", "Name for this run. Defaults to the group name plus a timestamp.") + cmd.Flags().StringVar(&level, "level", "", + "Scoring granularity: turn or conversation. Defaults to the service default (turn).") + cmd.Flags().IntVar(&maxSamples, "max-samples", 0, + "Cap the rows sent from a local dataset file. Ignored for registered datasets.") + cmd.Flags().BoolVar(&wait, "wait", true, "Block until the run reaches a terminal state.") + cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") + return cmd +} + +// resolveEvalGroupID finds the eval group to run against, creating it when it +// has never been deployed. Resolution order: an id pinned on the group, then +// the azd environment, then create. +func (ec *evalContext) resolveEvalGroupID( + ctx context.Context, + group *project.EvalGroup, + out interface{ Write([]byte) (int, error) }, + jsonMode bool, +) (string, error) { + if group.ID != "" { + return group.ID, nil + } + + if cached := ec.getEnvValue(ctx, envKeyEvalGroupID); cached != "" { + // Confirm it still exists; a deleted group should fall through to create. + if _, err := ec.evalClient.GetOpenAIEval(ctx, cached); err == nil { + return cached, nil + } + } + + if !jsonMode { + fmt.Fprintf(out, "Creating eval group %q...\n", group.Name) + } + created, err := ec.evalClient.CreateOpenAIEval(ctx, buildEvalGroupRequest(group)) + if err != nil { + return "", fmt.Errorf("creating eval group %q: %w", group.Name, err) + } + if err := ec.setEnvValue(ctx, envKeyEvalGroupID, created.ID); err != nil { + fmt.Fprintf(out, "warning: %v\n", err) + } + return created.ID, nil +} + +// buildRunDataSource binds the dataset to the run. The eval group carries no +// dataset today, so it is supplied here. +func buildRunDataSource( + group *project.EvalGroup, + configPath string, + maxSamples int, +) (*eval_api.EvalRunDataSource, error) { + if group == nil || group.Target == nil { + return nil, fmt.Errorf( + "the eval group must declare target.type: agent so the run knows what to invoke") + } + + ds := eval_api.NewAgentTargetDataSource(group.Target.Name, nil) + + if group.Dataset == "" { + return nil, fmt.Errorf("eval group %q does not reference a dataset", group.Name) + } + + // A local source is sent inline; anything else is a registered dataset. + localPath := localDatasetPath(configPath, group) + if localPath == "" { + ds.SetFileID(group.Dataset) + return ds, nil + } + + items, err := readJSONL(localPath, maxSamples) + if err != nil { + return nil, err + } + if len(items) == 0 { + return nil, fmt.Errorf("dataset file %q has no rows", localPath) + } + ds.SetFileContent(items) + return ds, nil +} + +// localDatasetPath resolves the dataset's local source relative to the config +// file, returning empty when the dataset is registered rather than local. +func localDatasetPath(configPath string, group *project.EvalGroup) string { + cfg, err := project.LoadEvalConfig(configPath) + if err != nil { + return "" + } + decl, ok := cfg.Dataset(group.Dataset) + if !ok || decl.Source == "" { + return "" + } + if filepath.IsAbs(decl.Source) { + return decl.Source + } + return filepath.Join(filepath.Dir(configPath), decl.Source) +} + +// readJSONL reads newline-delimited JSON, optionally truncating to limit rows. +func readJSONL(path string, limit int) ([]map[string]any, error) { + f, err := os.Open(path) + if err != nil { + return nil, fmt.Errorf("reading dataset %q: %w", path, err) + } + defer f.Close() + + var items []map[string]any + scanner := bufio.NewScanner(f) + scanner.Buffer(make([]byte, 0, 64*1024), 8*1024*1024) + line := 0 + for scanner.Scan() { + line++ + text := strings.TrimSpace(scanner.Text()) + if text == "" { + continue + } + var row map[string]any + if err := json.Unmarshal([]byte(text), &row); err != nil { + return nil, fmt.Errorf("%s line %d is not valid JSON: %w", path, line, err) + } + items = append(items, row) + if limit > 0 && len(items) >= limit { + break + } + } + if err := scanner.Err(); err != nil { + return nil, fmt.Errorf("reading dataset %q: %w", path, err) + } + return items, nil +} + +// resolveLevel prefers the flag, then the group's options. +func resolveLevel(flag string, group *project.EvalGroup) string { + if flag != "" { + return flag + } + if group != nil && group.Options != nil { + return group.Options.EvaluationLevel + } + return "" +} + +// pollRun waits for the run to reach a terminal state, reporting status changes. +func (ec *evalContext) pollRun( + ctx context.Context, + evalID, runID string, + out interface{ Write([]byte) (int, error) }, + jsonMode bool, +) (*eval_api.OpenAIEvalRun, error) { + const interval = 5 * time.Second + lastStatus := "" + + for { + run, err := ec.evalClient.GetOpenAIEvalRun(ctx, evalID, runID) + if err != nil { + return nil, fmt.Errorf("polling run %s: %w", runID, err) + } + if run.Status != lastStatus { + lastStatus = run.Status + if !jsonMode { + fmt.Fprintf(out, " status: %s\n", run.Status) + } + } + if terminalRunStates[strings.ToLower(run.Status)] { + return run, nil + } + select { + case <-ctx.Done(): + return nil, ctx.Err() + case <-time.After(interval): + } + } +} + +func renderRun(out interface{ Write([]byte) (int, error) }, run *eval_api.OpenAIEvalRun) error { + fmt.Fprintf(out, "\nRun %s finished with status %s\n", run.ID, run.Status) + if run.ReportURL != "" { + fmt.Fprintf(out, "Report: %s\n", run.ReportURL) + } + return nil +} From 035f6bb85fe4b3db522a2bce5c133b4c5ddcd97d Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 28 Jul 2026 05:11:15 +0300 Subject: [PATCH 005/320] feat(azure.ai.evaluations): evaluator and results commands - evaluator upload/update/list/show/builtins/delete; rubric evaluators only in M1, code evaluators deferred to M2 with the folder walk and RBAC they require - normalizeRubricBody accepts a bare definition or a full document - results show/export with per-criteria pass and fail counts, --failed-only, and JSON or CSV output, replacing the counts-only view - Added ListEvaluators, ListEvaluatorVersions, DeleteEvaluatorVersion, and CancelOpenAIEvalRun to the eval client --- .../internal/cmd/evaluator.go | 298 ++++++++++++++++++ .../internal/cmd/results.go | 239 ++++++++++++++ .../azure.ai.evaluations/internal/cmd/root.go | 2 + .../internal/pkg/eval_api/evaluators.go | 84 +++++ 4 files changed, 623 insertions(+) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/results.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/evaluators.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator.go new file mode 100644 index 00000000000..e374079dd9e --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator.go @@ -0,0 +1,298 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "encoding/json" + "fmt" + "os" + + "azureaieval/internal/pkg/eval_api" + + "github.com/spf13/cobra" +) + +func newEvaluatorCommand() *cobra.Command { + cmd := &cobra.Command{ + Use: "evaluator", + Short: "Manage custom evaluators.", + } + cmd.AddCommand( + newEvaluatorUploadCommand(false), + newEvaluatorUploadCommand(true), + newEvaluatorListCommand(), + newEvaluatorShowCommand(), + newEvaluatorBuiltinsCommand(), + newEvaluatorDeleteCommand(), + ) + return cmd +} + +// newEvaluatorUploadCommand builds `evaluator upload` and `evaluator update`. +// Both publish a new immutable version. +// +// M1 supports rubric evaluators only. Code evaluators need a folder walk, +// multi-blob upload, and the Azure AI User role assignment, so they land in M2. +func newEvaluatorUploadCommand(update bool) *cobra.Command { + var ( + name string + rubric string + endpointFlg string + ) + + use, short := "upload", "Register a rubric evaluator, creating its first version." + if update { + use, short = "update", "Publish a new version of an existing rubric evaluator." + } + + cmd := &cobra.Command{ + Use: use, + Short: short, + RunE: func(cmd *cobra.Command, args []string) error { + if name == "" { + return requireFlag("name") + } + if rubric == "" { + return requireFlag("rubric") + } + + raw, err := os.ReadFile(rubric) + if err != nil { + return fmt.Errorf("reading rubric %q: %w", rubric, err) + } + + body, err := normalizeRubricBody(name, raw) + if err != nil { + return fmt.Errorf("rubric %q: %w", rubric, err) + } + + ctx := cmd.Context() + ec, err := newEvalContext(ctx, endpointFlg) + if err != nil { + return err + } + defer ec.Close() + + created, err := ec.evalClient.CreateEvaluatorVersion( + ctx, name, body, ProjectEndpointAPIVersion, + ) + if err != nil { + return fmt.Errorf("registering evaluator %q: %w", name, err) + } + + if isJSON(cmd) { + return emitJSON(cmd.OutOrStdout(), created) + } + fmt.Fprintf(cmd.OutOrStdout(), + "Registered evaluator %s version %s\n", created.Name, created.Version) + return nil + }, + } + + cmd.Flags().StringVar(&name, "name", "", "Name of the evaluator.") + cmd.Flags().StringVar(&rubric, "rubric", "", "Path to the rubric JSON file.") + cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") + return cmd +} + +// normalizeRubricBody accepts either a bare definition ({type, dimensions}) or +// a full evaluator document ({name, definition}) and returns the request body. +func normalizeRubricBody(name string, raw []byte) (json.RawMessage, error) { + var probe map[string]json.RawMessage + if err := json.Unmarshal(raw, &probe); err != nil { + return nil, fmt.Errorf("not valid JSON: %w", err) + } + + if _, hasDefinition := probe["definition"]; hasDefinition { + // Already a full document; make sure the name matches the flag. + probe["name"] = json.RawMessage(fmt.Sprintf("%q", name)) + out, err := json.Marshal(probe) + if err != nil { + return nil, err + } + return out, nil + } + + if _, hasDimensions := probe["dimensions"]; !hasDimensions { + return nil, fmt.Errorf( + "expected a rubric definition with 'dimensions', or a document with 'definition'") + } + + doc := map[string]any{ + "name": name, + "definition": json.RawMessage(raw), + } + out, err := json.Marshal(doc) + if err != nil { + return nil, err + } + return out, nil +} + +func newEvaluatorListCommand() *cobra.Command { + var ( + name string + endpointFlg string + ) + + cmd := &cobra.Command{ + Use: "list", + Short: "List evaluators, or the versions of one evaluator.", + RunE: func(cmd *cobra.Command, args []string) error { + ctx := cmd.Context() + ec, err := newEvalContext(ctx, endpointFlg) + if err != nil { + return err + } + defer ec.Close() + + var list *eval_api.EvaluatorListResponse + if name != "" { + list, err = ec.evalClient.ListEvaluatorVersions(ctx, name, ProjectEndpointAPIVersion) + } else { + list, err = ec.evalClient.ListEvaluators(ctx, "", ProjectEndpointAPIVersion) + } + if err != nil { + return fmt.Errorf("listing evaluators: %w", err) + } + return renderEvaluators(cmd, list) + }, + } + + cmd.Flags().StringVar(&name, "name", "", "Limit the listing to versions of this evaluator.") + cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") + return cmd +} + +func newEvaluatorBuiltinsCommand() *cobra.Command { + var endpointFlg string + + cmd := &cobra.Command{ + Use: "builtins", + Short: "List the platform's built-in evaluators.", + RunE: func(cmd *cobra.Command, args []string) error { + ctx := cmd.Context() + ec, err := newEvalContext(ctx, endpointFlg) + if err != nil { + return err + } + defer ec.Close() + + list, err := ec.evalClient.ListEvaluators( + ctx, eval_api.EvaluatorTypeBuiltin, ProjectEndpointAPIVersion, + ) + if err != nil { + return fmt.Errorf("listing built-in evaluators: %w", err) + } + return renderEvaluators(cmd, list) + }, + } + + cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") + return cmd +} + +func renderEvaluators(cmd *cobra.Command, list *eval_api.EvaluatorListResponse) error { + if isJSON(cmd) { + return emitJSON(cmd.OutOrStdout(), list) + } + if len(list.Value) == 0 { + fmt.Fprintln(cmd.OutOrStdout(), "No evaluators found.") + return nil + } + rows := make([][]string, 0, len(list.Value)) + for _, e := range list.Value { + rows = append(rows, []string{e.Name, e.Version, e.Type}) + } + return emitTable(cmd.OutOrStdout(), []string{"NAME", "VERSION", "TYPE"}, rows) +} + +func newEvaluatorShowCommand() *cobra.Command { + var ( + name string + version string + endpointFlg string + ) + + cmd := &cobra.Command{ + Use: "show", + Short: "Show an evaluator definition.", + RunE: func(cmd *cobra.Command, args []string) error { + if name == "" { + return requireFlag("name") + } + + ctx := cmd.Context() + ec, err := newEvalContext(ctx, endpointFlg) + if err != nil { + return err + } + defer ec.Close() + + raw, err := ec.evalClient.GetEvaluatorRaw(ctx, name, version, ProjectEndpointAPIVersion) + if err != nil { + return fmt.Errorf("reading evaluator %q: %w", name, err) + } + + var pretty any + if err := json.Unmarshal(raw, &pretty); err != nil { + fmt.Fprintln(cmd.OutOrStdout(), string(raw)) + return nil + } + return emitJSON(cmd.OutOrStdout(), pretty) + }, + } + + cmd.Flags().StringVar(&name, "name", "", "Name of the evaluator.") + cmd.Flags().StringVar(&version, "version", "", "Version to show. Omit for the latest.") + cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") + return cmd +} + +func newEvaluatorDeleteCommand() *cobra.Command { + var ( + name string + version string + endpointFlg string + ) + + cmd := &cobra.Command{ + Use: "delete", + Short: "Delete an evaluator version.", + RunE: func(cmd *cobra.Command, args []string) error { + if name == "" { + return requireFlag("name") + } + if version == "" { + return requireFlag("version") + } + + ctx := cmd.Context() + ec, err := newEvalContext(ctx, endpointFlg) + if err != nil { + return err + } + defer ec.Close() + + if err := ec.evalClient.DeleteEvaluatorVersion( + ctx, name, version, ProjectEndpointAPIVersion, + ); err != nil { + return fmt.Errorf("deleting evaluator %q version %q: %w", name, version, err) + } + + if isJSON(cmd) { + return emitJSON(cmd.OutOrStdout(), map[string]string{ + "name": name, "version": version, "status": "deleted", + }) + } + fmt.Fprintf(cmd.OutOrStdout(), "Deleted evaluator %s version %s\n", name, version) + return nil + }, + } + + cmd.Flags().StringVar(&name, "name", "", "Name of the evaluator.") + cmd.Flags().StringVar(&version, "version", "", "Version to delete.") + cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") + return cmd +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/results.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/results.go new file mode 100644 index 00000000000..3eb79a1bc22 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/results.go @@ -0,0 +1,239 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "encoding/csv" + "fmt" + "io" + "os" + "strconv" + "strings" + + "azureaieval/internal/pkg/eval_api" + + "github.com/spf13/cobra" +) + +func newResultsCommand() *cobra.Command { + cmd := &cobra.Command{ + Use: "results", + Short: "Inspect evaluation results.", + } + cmd.AddCommand(newResultsShowCommand(), newResultsExportCommand()) + return cmd +} + +func newResultsShowCommand() *cobra.Command { + var ( + runID string + failedOnly bool + outFile string + endpointFlg string + ) + + cmd := &cobra.Command{ + Use: "show ", + Short: "Show per-sample results for a run.", + Args: cobra.MaximumNArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + ctx := cmd.Context() + ec, err := newEvalContext(ctx, endpointFlg) + if err != nil { + return err + } + defer ec.Close() + + evalID, err := resolveEvalID(cmd, ec, args) + if err != nil { + return err + } + + run, err := ec.latestOrNamedRun(cmd, evalID, runID) + if err != nil { + return err + } + + if outFile != "" { + f, err := os.Create(outFile) + if err != nil { + return fmt.Errorf("creating %q: %w", outFile, err) + } + defer f.Close() + return emitJSON(f, run) + } + if isJSON(cmd) { + return emitJSON(cmd.OutOrStdout(), run) + } + return renderResults(cmd.OutOrStdout(), run, failedOnly) + }, + } + + cmd.Flags().StringVar(&runID, "run-id", "", "Run to show. Defaults to the most recent run.") + cmd.Flags().BoolVar(&failedOnly, "failed-only", false, "Show only criteria with failures.") + cmd.Flags().StringVarP(&outFile, "out-file", "O", "", "Write JSON results to this path.") + cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") + return cmd +} + +func newResultsExportCommand() *cobra.Command { + var ( + runID string + format string + outFile string + endpointFlg string + ) + + cmd := &cobra.Command{ + Use: "export ", + Short: "Export run results as JSON or CSV.", + Args: cobra.MaximumNArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + format = strings.ToLower(format) + if format != "json" && format != "csv" { + return fmt.Errorf("--format must be json or csv, got %q", format) + } + + ctx := cmd.Context() + ec, err := newEvalContext(ctx, endpointFlg) + if err != nil { + return err + } + defer ec.Close() + + evalID, err := resolveEvalID(cmd, ec, args) + if err != nil { + return err + } + + run, err := ec.latestOrNamedRun(cmd, evalID, runID) + if err != nil { + return err + } + + var w io.Writer = cmd.OutOrStdout() + if outFile != "" { + f, err := os.Create(outFile) + if err != nil { + return fmt.Errorf("creating %q: %w", outFile, err) + } + defer f.Close() + w = f + } + + if format == "json" { + return emitJSON(w, run) + } + return writeResultsCSV(w, run) + }, + } + + cmd.Flags().StringVar(&runID, "run-id", "", "Run to export. Defaults to the most recent run.") + cmd.Flags().StringVar(&format, "format", "json", "Output format: json or csv.") + cmd.Flags().StringVarP(&outFile, "out-file", "O", "", "Write to this path instead of stdout.") + cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") + return cmd +} + +// resolveEvalID takes the eval group id from the argument, falling back to the +// id cached in the azd environment. +func resolveEvalID(cmd *cobra.Command, ec *evalContext, args []string) (string, error) { + if len(args) > 0 && args[0] != "" { + return args[0], nil + } + if cached := ec.getEnvValue(cmd.Context(), envKeyEvalGroupID); cached != "" { + return cached, nil + } + return "", fmt.Errorf( + "no eval group id given; pass it as an argument or set %s in the azd environment", + envKeyEvalGroupID) +} + +// latestOrNamedRun returns the named run, or the most recent one for the group. +func (ec *evalContext) latestOrNamedRun( + cmd *cobra.Command, + evalID, runID string, +) (*eval_api.OpenAIEvalRun, error) { + ctx := cmd.Context() + + if runID == "" { + if cached := ec.getEnvValue(ctx, envKeyEvalRunID); cached != "" { + runID = cached + } + } + if runID != "" { + run, err := ec.evalClient.GetOpenAIEvalRun(ctx, evalID, runID) + if err != nil { + return nil, fmt.Errorf("reading run %s: %w", runID, err) + } + return run, nil + } + + list, err := ec.evalClient.ListOpenAIEvalRuns(ctx, evalID, 1) + if err != nil { + return nil, fmt.Errorf("listing runs for eval group %s: %w", evalID, err) + } + if len(list.Data) == 0 { + return nil, fmt.Errorf("eval group %s has no runs yet", evalID) + } + return &list.Data[0], nil +} + +func renderResults(w io.Writer, run *eval_api.OpenAIEvalRun, failedOnly bool) error { + fmt.Fprintf(w, "Run %s status: %s\n", run.ID, run.Status) + + if c := run.ResultCounts; c != nil { + fmt.Fprintf(w, "Totals: %d passed, %d failed, %d errored\n\n", + c.Passed, c.Failed, c.Errored) + } + + if len(run.PerTestingCriteria) == 0 { + fmt.Fprintln(w, "No per-criteria results are available yet.") + return nil + } + + rows := make([][]string, 0, len(run.PerTestingCriteria)) + for _, cr := range run.PerTestingCriteria { + if failedOnly && cr.Failed == 0 { + continue + } + rows = append(rows, []string{ + cr.TestingCriteria, + strconv.Itoa(cr.Passed), + strconv.Itoa(cr.Failed), + }) + } + if len(rows) == 0 { + fmt.Fprintln(w, "No failing criteria.") + return nil + } + if err := emitTable(w, []string{"CRITERION", "PASSED", "FAILED"}, rows); err != nil { + return err + } + if run.ReportURL != "" { + fmt.Fprintf(w, "\nReport: %s\n", run.ReportURL) + } + return nil +} + +func writeResultsCSV(w io.Writer, run *eval_api.OpenAIEvalRun) error { + cw := csv.NewWriter(w) + defer cw.Flush() + + if err := cw.Write([]string{"run_id", "status", "criterion", "passed", "failed"}); err != nil { + return err + } + if len(run.PerTestingCriteria) == 0 { + return cw.Write([]string{run.ID, run.Status, "", "", ""}) + } + for _, cr := range run.PerTestingCriteria { + if err := cw.Write([]string{ + run.ID, run.Status, cr.TestingCriteria, + strconv.Itoa(cr.Passed), strconv.Itoa(cr.Failed), + }); err != nil { + return err + } + } + return nil +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/root.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/root.go index 40f666b2f90..64c6d63c568 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/root.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/root.go @@ -28,6 +28,8 @@ func NewRootCommand() *cobra.Command { rootCmd.AddCommand( newDatasetCommand(), newRunCommand(), + newEvaluatorCommand(), + newResultsCommand(), ) return rootCmd diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/evaluators.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/evaluators.go new file mode 100644 index 00000000000..771f894b03d --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/evaluators.go @@ -0,0 +1,84 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package eval_api + +import ( + "context" + "fmt" + "net/http" + "net/url" +) + +// EvaluatorTypeBuiltin selects the platform-provided evaluators. +const EvaluatorTypeBuiltin = "Builtin" + +// EvaluatorSummary is a single entry in an evaluator listing. +type EvaluatorSummary struct { + Name string `json:"name"` + Version string `json:"version,omitempty"` + Type string `json:"type,omitempty"` + Description string `json:"description,omitempty"` +} + +// EvaluatorListResponse is the paged response for an evaluator listing. +type EvaluatorListResponse struct { + Value []EvaluatorSummary `json:"value"` + NextLink string `json:"nextLink,omitempty"` +} + +// ListEvaluators returns the evaluators visible to the project. Pass +// EvaluatorTypeBuiltin to list only the platform's built-ins. +func (c *EvalClient) ListEvaluators( + ctx context.Context, + evaluatorType string, + apiVersion string, +) (*EvaluatorListResponse, error) { + var query map[string]string + if evaluatorType != "" { + query = map[string]string{"type": evaluatorType} + } + return doRequestTyped[EvaluatorListResponse]( + c, ctx, http.MethodGet, pathEvaluators, query, nil, apiVersion, + ) +} + +// ListEvaluatorVersions returns every version of one evaluator. +func (c *EvalClient) ListEvaluatorVersions( + ctx context.Context, + name string, + apiVersion string, +) (*EvaluatorListResponse, error) { + path := pathEvaluators + "/" + url.PathEscape(name) + "/versions" + return doRequestTyped[EvaluatorListResponse]( + c, ctx, http.MethodGet, path, nil, nil, apiVersion, + ) +} + +// DeleteEvaluatorVersion removes a single evaluator version. +func (c *EvalClient) DeleteEvaluatorVersion( + ctx context.Context, + name string, + version string, + apiVersion string, +) error { + path := fmt.Sprintf( + "%s/%s/versions/%s", + pathEvaluators, url.PathEscape(name), url.PathEscape(version), + ) + _, err := c.doRequest(ctx, http.MethodDelete, path, nil, nil, apiVersion) + return err +} + +// CancelOpenAIEvalRun stops an in-flight run. +func (c *EvalClient) CancelOpenAIEvalRun( + ctx context.Context, + evalID string, + runID string, +) (*OpenAIEvalRun, error) { + path := fmt.Sprintf( + "%s/%s/runs/%s/cancel", + pathOpenAIEvals, url.PathEscape(evalID), url.PathEscape(runID), + ) + return doRequestTyped[OpenAIEvalRun](c, ctx, http.MethodPost, path, nil, nil, "") +} From 4f8a4eb60a641ec32f0e493fd709c9a249882baa Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 28 Jul 2026 05:13:10 +0300 Subject: [PATCH 006/320] feat(azure.ai.evaluations): offline init command with round-trip tests init scaffolds both YAML files and the artifact directories without any service call, so it works offline and unauthenticated. Built-ins are referenced from the group but never declared as custom evaluators. A dataset flag containing a path becomes a local source; a bare name references a registered dataset. Tests assert the scaffold loads and validates, and that paths are used verbatim rather than re-rooted. --- .../azure.ai.evaluations/internal/cmd/init.go | 218 ++++++++++++++++++ .../internal/cmd/init_test.go | 143 ++++++++++++ .../azure.ai.evaluations/internal/cmd/root.go | 1 + 3 files changed, 362 insertions(+) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go new file mode 100644 index 00000000000..047257edd2e --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go @@ -0,0 +1,218 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "fmt" + "os" + "path/filepath" + "strings" + + "azureaieval/internal/pkg/evalcore" + "azureaieval/internal/project" + + "github.com/spf13/cobra" + "go.yaml.in/yaml/v3" +) + +// newInitCommand scaffolds the eval configuration. It makes no service calls at +// all, so it works offline and unauthenticated. +func newInitCommand() *cobra.Command { + var ( + target string + dataset string + evaluators []string + evalModel string + outDir string + force bool + ) + + cmd := &cobra.Command{ + Use: "init", + Short: "Scaffold evaluation config for an agent. Makes no service calls.", + RunE: func(cmd *cobra.Command, args []string) error { + out := cmd.OutOrStdout() + + if target == "" { + return requireFlag("target") + } + if outDir == "" { + outDir = project.DefaultEvalDir + } + + genPath := filepath.Join(outDir, "eval_generate.yaml") + depPath := filepath.Join(outDir, "azure.yaml") + + for _, p := range []string{genPath, depPath} { + if _, err := os.Stat(p); err == nil && !force { + return fmt.Errorf("%s already exists; pass --force to overwrite", p) + } + } + + if err := os.MkdirAll(filepath.Join(outDir, project.DefaultDatasetsDir), 0o750); err != nil { + return fmt.Errorf("creating the datasets directory: %w", err) + } + if err := os.MkdirAll(filepath.Join(outDir, project.DefaultEvaluatorsDir), 0o750); err != nil { + return fmt.Errorf("creating the evaluators directory: %w", err) + } + + rubricName := fmt.Sprintf("%s-quality", target) + + genCfg := buildGenerateScaffold(target, rubricName, evalModel) + if err := writeYAML(genPath, genCfg); err != nil { + return err + } + + depCfg := buildDeployScaffold(target, rubricName, dataset, evaluators, evalModel) + if err := writeYAML(depPath, depCfg); err != nil { + return err + } + + if isJSON(cmd) { + return emitJSON(out, map[string]any{ + "generateConfig": genPath, + "deployConfig": depPath, + "datasetsDir": filepath.Join(outDir, project.DefaultDatasetsDir), + "evaluatorsDir": filepath.Join(outDir, project.DefaultEvaluatorsDir), + }) + } + + fmt.Fprintf(out, "Wrote %s\n", genPath) + fmt.Fprintf(out, "Wrote %s\n", depPath) + fmt.Fprintln(out, "\nNext:") + fmt.Fprintf(out, " 1. Reference %s from your root azure.yaml:\n", depPath) + fmt.Fprintln(out, " services:") + fmt.Fprintln(out, " evals:") + fmt.Fprintln(out, " host: azure.ai.eval") + fmt.Fprintln(out, " uses: [ai-project]") + fmt.Fprintf(out, " $ref: ./%s\n", filepath.ToSlash(depPath)) + fmt.Fprintln(out, " 2. azd ai eval generate (or supply your own dataset)") + fmt.Fprintln(out, " 3. azd up") + fmt.Fprintln(out, " 4. azd ai eval run") + return nil + }, + } + + cmd.Flags().StringVar(&target, "target", "", "Name of the agent to evaluate.") + cmd.Flags().StringVar(&dataset, "dataset", "", "Path to a local .jsonl, or the name of a registered dataset.") + cmd.Flags().StringArrayVar(&evaluators, "evaluator", nil, + "Evaluator reference, repeatable. Use builtin. for a built-in.") + cmd.Flags().StringVar(&evalModel, "eval-model", "", "Model deployment used as the LLM judge.") + cmd.Flags().StringVar(&outDir, "out-dir", project.DefaultEvalDir, + "Directory to write the config into. Used verbatim, never re-rooted.") + cmd.Flags().BoolVar(&force, "force", false, "Overwrite existing files.") + return cmd +} + +func buildGenerateScaffold(target, rubricName, evalModel string) *project.GenerateConfig { + return &project.GenerateConfig{ + Agent: project.AgentSpec{ + Name: target, + Context: project.AgentContext{ + Instructions: "./agent/instructions.md", + Tools: "./agent/tools.json", + }, + }, + Generate: project.GenerateSpec{ + Rubric: &project.RubricSpec{ + Name: rubricName, + Model: evalModel, + LocalDir: "./" + project.DefaultEvaluatorsDir, + }, + Dataset: &project.DatasetSpec{ + Name: fmt.Sprintf("%s-golden", target), + Strategy: project.StrategySynthetic, + SampleSize: project.DefaultSampleSize, + LocalDir: "./" + project.DefaultDatasetsDir, + }, + }, + } +} + +func buildDeployScaffold( + target, rubricName, dataset string, + evaluators []string, + evalModel string, +) *project.EvalConfig { + cfg := &project.EvalConfig{} + + datasetName := fmt.Sprintf("%s-golden", target) + datasetSource := "" + if dataset != "" { + if looksLikeLocalDataset(dataset) { + datasetSource = dataset + datasetName = strings.TrimSuffix(filepath.Base(dataset), filepath.Ext(dataset)) + } else { + // A bare name references an already-registered dataset. + datasetName = dataset + } + } else { + datasetSource = fmt.Sprintf("./%s/%s.jsonl", project.DefaultDatasetsDir, datasetName) + } + cfg.Datasets = append(cfg.Datasets, project.DatasetDecl{ + Name: datasetName, + Source: datasetSource, + }) + + // Evaluators supplied on the command line win; otherwise scaffold the + // generated rubric so `generate` has somewhere to write its reference. + refs := evalcore.EvaluatorList{} + if len(evaluators) == 0 { + cfg.Evaluators = append(cfg.Evaluators, project.EvaluatorDecl{ + Name: rubricName, + Source: fmt.Sprintf("./%s/%s.json", project.DefaultEvaluatorsDir, rubricName), + }) + refs = append(refs, evalcore.EvaluatorRef{Name: rubricName}) + } else { + for _, e := range evaluators { + ref := evalcore.EvaluatorRef{Name: e} + if !ref.IsBuiltin() { + cfg.Evaluators = append(cfg.Evaluators, project.EvaluatorDecl{ + Name: e, + Source: fmt.Sprintf("./%s/%s.json", project.DefaultEvaluatorsDir, e), + }) + } + refs = append(refs, ref) + } + } + + group := project.EvalGroup{ + Name: fmt.Sprintf("%s-quality", target), + Description: fmt.Sprintf("Quality gate for %s", target), + Dataset: datasetName, + Evaluators: refs, + Target: &project.Target{ + Type: project.TargetTypeAgent, + Name: target, + }, + } + if evalModel != "" { + group.Options = &project.Options{EvalModel: evalModel} + } + cfg.EvalGroups = append(cfg.EvalGroups, group) + + return cfg +} + +// looksLikeLocalDataset distinguishes a path from a registered dataset name. +func looksLikeLocalDataset(v string) bool { + if strings.ContainsAny(v, `/\`) { + return true + } + return strings.EqualFold(filepath.Ext(v), ".jsonl") +} + +func writeYAML(path string, v any) error { + if err := os.MkdirAll(filepath.Dir(path), 0o750); err != nil { + return fmt.Errorf("creating %q: %w", filepath.Dir(path), err) + } + data, err := yaml.Marshal(v) + if err != nil { + return fmt.Errorf("serializing %q: %w", path, err) + } + if err := os.WriteFile(path, data, 0o600); err != nil { + return fmt.Errorf("writing %q: %w", path, err) + } + return nil +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_test.go new file mode 100644 index 00000000000..55820051efb --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_test.go @@ -0,0 +1,143 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "os" + "path/filepath" + "testing" + + "azureaieval/internal/project" + + "github.com/stretchr/testify/require" +) + +// The scaffold must load and validate cleanly, otherwise `azd up` fails on a +// config the tool itself produced. +func TestScaffold_RoundTripsAndValidates(t *testing.T) { + dir := t.TempDir() + depPath := filepath.Join(dir, "azure.yaml") + + cfg := buildDeployScaffold("support-agent", "support-agent-quality", "", nil, "gpt-4.1-nano") + require.NoError(t, writeYAML(depPath, cfg)) + + loaded, err := project.LoadEvalConfig(depPath) + require.NoError(t, err) + require.NoError(t, loaded.Validate(), "the generated scaffold must be valid") + + g, err := loaded.ResolveGroup("") + require.NoError(t, err) + require.Equal(t, project.TargetTypeAgent, g.Target.Type) + require.Equal(t, "support-agent", g.Target.Name) + require.Equal(t, "gpt-4.1-nano", g.Options.EvalModel) + require.Len(t, g.Evaluators, 1) +} + +func TestGenerateScaffold_RoundTripsAndValidates(t *testing.T) { + dir := t.TempDir() + genPath := filepath.Join(dir, "eval_generate.yaml") + + cfg := buildGenerateScaffold("support-agent", "support-agent-quality", "gpt-4.1-nano") + require.NoError(t, writeYAML(genPath, cfg)) + + loaded, err := project.LoadGenerateConfig(genPath) + require.NoError(t, err) + require.NoError(t, loaded.Validate()) + require.Equal(t, "support-agent", loaded.Agent.Name) + require.Equal(t, project.StrategySynthetic, loaded.Generate.Dataset.Strategy) + require.Equal(t, project.DefaultSampleSize, loaded.Generate.Dataset.SampleSize) +} + +// Built-ins are referenced from the group but never declared as custom +// evaluators; declaring one is a validation error. +func TestScaffold_BuiltinEvaluatorsAreNotDeclared(t *testing.T) { + cfg := buildDeployScaffold( + "support-agent", "unused", "", + []string{"builtin.task_adherence", "my-custom"}, "", + ) + + require.Len(t, cfg.Evaluators, 1, "only the custom evaluator should be declared") + require.Equal(t, "my-custom", cfg.Evaluators[0].Name) + + require.Len(t, cfg.EvalGroups[0].Evaluators, 2) + require.True(t, cfg.EvalGroups[0].Evaluators[0].IsBuiltin()) + require.False(t, cfg.EvalGroups[0].Evaluators[1].IsBuiltin()) + + path := filepath.Join(t.TempDir(), "azure.yaml") + require.NoError(t, writeYAML(path, cfg)) + loaded, err := project.LoadEvalConfig(path) + require.NoError(t, err) + require.NoError(t, loaded.Validate()) +} + +// A bare name means an already-registered dataset; a path means a local file. +func TestScaffold_DatasetReferenceForms(t *testing.T) { + t.Run("local path becomes a source", func(t *testing.T) { + cfg := buildDeployScaffold("a", "r", "./tests/golden.jsonl", nil, "") + require.Equal(t, "./tests/golden.jsonl", cfg.Datasets[0].Source) + require.Equal(t, "golden", cfg.Datasets[0].Name) + }) + + t.Run("bare name references a registered dataset", func(t *testing.T) { + cfg := buildDeployScaffold("a", "r", "prod-sample", nil, "") + require.Equal(t, "prod-sample", cfg.Datasets[0].Name) + require.Empty(t, cfg.Datasets[0].Source, + "a registered dataset must not get a local source") + }) + + t.Run("no dataset flag scaffolds a local path", func(t *testing.T) { + cfg := buildDeployScaffold("support-agent", "r", "", nil, "") + require.Contains(t, cfg.Datasets[0].Source, "support-agent-golden.jsonl") + }) +} + +func TestLooksLikeLocalDataset(t *testing.T) { + require.True(t, looksLikeLocalDataset("./data/golden.jsonl")) + require.True(t, looksLikeLocalDataset("golden.jsonl")) + require.True(t, looksLikeLocalDataset(`data\golden.jsonl`)) + require.False(t, looksLikeLocalDataset("prod-sample")) +} + +// Paths are used verbatim relative to the working directory; the doubling bug +// in the agent-scoped command must not reappear. +func TestWriteYAML_UsesPathVerbatim(t *testing.T) { + dir := t.TempDir() + nested := filepath.Join(dir, "evals", "azure.yaml") + + require.NoError(t, writeYAML(nested, &project.EvalConfig{})) + _, err := os.Stat(nested) + require.NoError(t, err, "the file must land exactly at the requested path") + + doubled := filepath.Join(dir, "evals", "evals", "azure.yaml") + _, err = os.Stat(doubled) + require.Error(t, err, "the path must not be re-rooted under itself") +} + +// normalizeRubricBody accepts a bare definition or a full document. +func TestNormalizeRubricBody(t *testing.T) { + t.Run("bare definition is wrapped", func(t *testing.T) { + body, err := normalizeRubricBody("quality", + []byte(`{"type":"rubric","dimensions":[{"id":"q","weight":10}]}`)) + require.NoError(t, err) + require.Contains(t, string(body), `"name":"quality"`) + require.Contains(t, string(body), `"definition"`) + }) + + t.Run("full document keeps its definition and takes the flag name", func(t *testing.T) { + body, err := normalizeRubricBody("renamed", + []byte(`{"name":"old","definition":{"type":"rubric","dimensions":[]}}`)) + require.NoError(t, err) + require.Contains(t, string(body), `"name":"renamed"`) + }) + + t.Run("rejects a document with neither", func(t *testing.T) { + _, err := normalizeRubricBody("x", []byte(`{"unrelated":true}`)) + require.ErrorContains(t, err, "dimensions") + }) + + t.Run("rejects invalid JSON", func(t *testing.T) { + _, err := normalizeRubricBody("x", []byte(`not json`)) + require.ErrorContains(t, err, "not valid JSON") + }) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/root.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/root.go index 64c6d63c568..bb606a49303 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/root.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/root.go @@ -26,6 +26,7 @@ func NewRootCommand() *cobra.Command { rootCmd.CompletionOptions.DisableDefaultCmd = true rootCmd.AddCommand( + newInitCommand(), newDatasetCommand(), newRunCommand(), newEvaluatorCommand(), From 6b576e3581e623d4e5396c0d395c976e4453ff98 Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 28 Jul 2026 05:17:43 +0300 Subject: [PATCH 007/320] feat(azure.ai.evaluations): service-target provider with change detection Registers the azure.ai.eval service target so azd up and azd deploy reach this extension; the extension ships no deploy command of its own. - Reads the eval config from the service entry's inline properties, the same AdditionalProperties channel the agents extension uses - Deploy reconciles datasets, then evaluators, then eval groups, since a group references the versions the first two resolve to; it fails fast and the next deploy resumes - Datasets are change-detected with a local SHA-256 digest kept in the azd environment, because the dataset API returns no content hash and comparing against the service would mean downloading the blob every deploy - Evaluator definitions come back inline, so those are compared directly - Package and Publish are no-ops; eval artifacts are plain files already on disk --- .../extensions/azure.ai.evaluations/go.mod | 6 +- .../internal/cmd/listen.go | 34 +++ .../internal/cmd/reconciler.go | 216 ++++++++++++++ .../azure.ai.evaluations/internal/cmd/root.go | 1 + .../internal/project/service_target_eval.go | 281 ++++++++++++++++++ 5 files changed, 535 insertions(+), 3 deletions(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/listen.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go diff --git a/cli/azd/extensions/azure.ai.evaluations/go.mod b/cli/azd/extensions/azure.ai.evaluations/go.mod index c63d3067dfb..ab57a735e7f 100644 --- a/cli/azd/extensions/azure.ai.evaluations/go.mod +++ b/cli/azd/extensions/azure.ai.evaluations/go.mod @@ -4,17 +4,19 @@ go 1.26.4 require ( github.com/Azure/azure-sdk-for-go/sdk/azcore v1.21.0 + github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.14.0-beta.3 github.com/azure/azure-dev/cli/azd v1.28.0 github.com/fatih/color v1.18.0 github.com/google/uuid v1.6.0 github.com/spf13/cobra v1.10.1 + github.com/stretchr/testify v1.11.1 go.yaml.in/yaml/v3 v3.0.4 + google.golang.org/protobuf v1.36.11 ) require ( dario.cat/mergo v1.0.2 // indirect github.com/AlecAivazis/survey/v2 v2.3.7 // indirect - github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.14.0-beta.3 // indirect github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2 // indirect github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/appservice/armappservice/v2 v2.3.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/keyvault/armkeyvault v1.5.0 // indirect @@ -77,7 +79,6 @@ require ( github.com/sethvargo/go-retry v0.3.0 // indirect github.com/spf13/cast v1.10.0 // indirect github.com/spf13/pflag v1.0.10 // indirect - github.com/stretchr/testify v1.11.1 // indirect github.com/theckman/yacspin v0.13.12 // indirect github.com/wk8/go-ordered-map/v2 v2.1.8 // indirect github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect @@ -101,6 +102,5 @@ require ( golang.org/x/time v0.9.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20260401024825-9d38bb4040a9 // indirect google.golang.org/grpc v1.80.0 // indirect - google.golang.org/protobuf v1.36.11 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/listen.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/listen.go new file mode 100644 index 00000000000..9cef816cec2 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/listen.go @@ -0,0 +1,34 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "context" + + "azureaieval/internal/project" + + "github.com/azure/azure-dev/cli/azd/pkg/azdext" + "github.com/spf13/cobra" +) + +// newListenCommand registers the service-target provider with azd. It is hidden +// and invoked by azd itself, not by users. +func newListenCommand() *cobra.Command { + return azdext.NewListenCommand(configureExtensionHost) +} + +// configureExtensionHost wires the azure.ai.eval service target so `azd up` and +// `azd deploy` reach this extension. The provider name must match the manifest. +func configureExtensionHost(host *azdext.ExtensionHost) { + azdClient := host.Client() + + host.WithServiceTarget(project.EvalHost, func() azdext.ServiceTargetProvider { + return project.NewEvalServiceTargetProvider( + azdClient, + func(ctx context.Context) (project.Reconciler, error) { + return newEvalReconciler(ctx) + }, + ) + }) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go new file mode 100644 index 00000000000..e2f433fe84f --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go @@ -0,0 +1,216 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "context" + "encoding/json" + "fmt" + "os" + "path/filepath" + + "azureaieval/internal/pkg/dataset_api" + "azureaieval/internal/project" +) + +// evalReconciler applies the eval configuration to the data plane. It is the +// deploy half of the provider; the provider owns ordering, this owns the calls. +type evalReconciler struct { + ec *evalContext +} + +var _ project.Reconciler = (*evalReconciler)(nil) + +func newEvalReconciler(ctx context.Context) (project.Reconciler, error) { + ec, err := newEvalContext(ctx, "") + if err != nil { + return nil, err + } + return &evalReconciler{ec: ec}, nil +} + +// EnsureDataset registers a new version only when the local content changed. +// +// The dataset API exposes no content hash, so comparing against the service +// would mean downloading the blob on every deploy. Instead the local file is +// hashed and the digest kept in the azd environment. +func (r *evalReconciler) EnsureDataset( + ctx context.Context, + decl project.DatasetDecl, + localPath string, +) (string, bool, error) { + // No local source means the dataset is already registered; just confirm it. + if localPath == "" { + version := decl.Version + if version == "" { + list, err := r.ec.datasetClient.ListDatasetVersions( + ctx, decl.Name, ProjectEndpointAPIVersion, + ) + if err != nil { + return "", false, fmt.Errorf( + "dataset %q has no local source and could not be found on the project: %w", + decl.Name, err) + } + if len(list.Value) == 0 { + return "", false, fmt.Errorf( + "dataset %q has no local source and is not registered on the project", decl.Name) + } + version = dataset_api.LatestVersion(list.Value) + } + return version, false, nil + } + + if _, err := os.Stat(localPath); err != nil { + return "", false, fmt.Errorf("dataset source %q: %w", localPath, err) + } + + digest, err := project.Fingerprint(localPath) + if err != nil { + return "", false, err + } + + key := project.FingerprintKey("dataset", decl.Name) + if prior := r.ec.getEnvValue(ctx, key); prior == digest { + // Unchanged since the last deploy; reuse the recorded version. + if version := r.ec.getEnvValue(ctx, versionKey("dataset", decl.Name)); version != "" { + return version, false, nil + } + } + + // The upload helper scans a directory for the first .jsonl. + dir := localPath + if info, err := os.Stat(localPath); err == nil && !info.IsDir() { + dir = filepath.Dir(localPath) + } + + ds, err := r.ec.datasetClient.UploadNewVersion( + ctx, decl.Name, decl.Version, dir, ProjectEndpointAPIVersion, + ) + if err != nil { + return "", false, err + } + + _ = r.ec.setEnvValue(ctx, key, digest) + _ = r.ec.setEnvValue(ctx, versionKey("dataset", decl.Name), ds.Version) + _ = r.ec.setEnvValue(ctx, envKeyDatasetVersion, ds.Version) + + return ds.Version, true, nil +} + +// EnsureEvaluator publishes a new version when the local definition differs +// from what the service holds. Evaluator definitions come back inline, so this +// compares content directly rather than relying on a cached digest. +func (r *evalReconciler) EnsureEvaluator( + ctx context.Context, + decl project.EvaluatorDecl, + localPath string, +) (string, bool, error) { + if localPath == "" { + raw, err := r.ec.evalClient.GetEvaluatorRaw( + ctx, decl.Name, decl.Version, ProjectEndpointAPIVersion, + ) + if err != nil { + return "", false, fmt.Errorf( + "evaluator %q has no local source and could not be found on the project: %w", + decl.Name, err) + } + return versionFromRaw(raw, decl.Version), false, nil + } + + raw, err := os.ReadFile(localPath) + if err != nil { + return "", false, fmt.Errorf("evaluator source %q: %w", localPath, err) + } + + body, err := normalizeRubricBody(decl.Name, raw) + if err != nil { + return "", false, fmt.Errorf("evaluator %q: %w", decl.Name, err) + } + + // Compare against the definition already on the service. + if existing, err := r.ec.evalClient.GetEvaluatorRaw( + ctx, decl.Name, "", ProjectEndpointAPIVersion, + ); err == nil { + if sameDefinition(existing, body) { + return versionFromRaw(existing, decl.Version), false, nil + } + } + + created, err := r.ec.evalClient.CreateEvaluatorVersion( + ctx, decl.Name, body, ProjectEndpointAPIVersion, + ) + if err != nil { + return "", false, err + } + return created.Version, true, nil +} + +// EnsureEvalGroup creates the group when it has never been deployed, or when an +// upstream artifact changed. Groups are immutable, so a change means a new +// group and a new id. +func (r *evalReconciler) EnsureEvalGroup( + ctx context.Context, + group project.EvalGroup, + recreate bool, +) (string, error) { + if group.ID != "" { + return group.ID, nil + } + + cached := r.ec.getEnvValue(ctx, envKeyEvalGroupID) + if cached != "" && !recreate { + if _, err := r.ec.evalClient.GetOpenAIEval(ctx, cached); err == nil { + return cached, nil + } + } + + created, err := r.ec.evalClient.CreateOpenAIEval(ctx, buildEvalGroupRequest(&group)) + if err != nil { + return "", err + } + _ = r.ec.setEnvValue(ctx, envKeyEvalGroupID, created.ID) + return created.ID, nil +} + +// sameDefinition compares only the definition body, ignoring server-assigned +// fields such as version and timestamps. +func sameDefinition(existing, candidate []byte) bool { + extract := func(raw []byte) string { + var doc map[string]json.RawMessage + if err := json.Unmarshal(raw, &doc); err != nil { + return "" + } + def, ok := doc["definition"] + if !ok { + return "" + } + var normalized any + if err := json.Unmarshal(def, &normalized); err != nil { + return "" + } + out, err := json.Marshal(normalized) + if err != nil { + return "" + } + return string(out) + } + + a, b := extract(existing), extract(candidate) + return a != "" && a == b +} + +func versionFromRaw(raw []byte, fallback string) string { + var doc struct { + Version string `json:"version"` + } + if err := json.Unmarshal(raw, &doc); err == nil && doc.Version != "" { + return doc.Version + } + return fallback +} + +// versionKey holds the version resolved for an artifact at the last deploy. +func versionKey(kind, name string) string { + return project.FingerprintKey(kind, name) + "_VERSION" +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/root.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/root.go index bb606a49303..671c10fe0ad 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/root.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/root.go @@ -31,6 +31,7 @@ func NewRootCommand() *cobra.Command { newRunCommand(), newEvaluatorCommand(), newResultsCommand(), + newListenCommand(), ) return rootCmd diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go new file mode 100644 index 00000000000..ce8a1439b5d --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go @@ -0,0 +1,281 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package project + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/azure/azure-dev/cli/azd/pkg/azdext" + "google.golang.org/protobuf/types/known/structpb" +) + +// EvalHost is the azure.yaml host this provider serves. +const EvalHost = "azure.ai.eval" + +// azd environment keys owned by this extension. +const ( + EnvKeyEvalGroupID = "EVAL_GROUP_ID" + EnvKeyDatasetVersion = "EVAL_DATASET_VERSION" + EnvKeyFingerprintPrefix = "EVAL_FINGERPRINT_" +) + +// Reconciler applies the eval configuration to the service. It is satisfied by +// the command layer, which owns the data-plane clients. +type Reconciler interface { + // EnsureDataset registers a new dataset version when the local content + // changed, returning the resolved version and whether anything was written. + EnsureDataset(ctx context.Context, decl DatasetDecl, localPath string) (version string, changed bool, err error) + // EnsureEvaluator registers a new evaluator version when the definition + // differs from what the service already holds. + EnsureEvaluator(ctx context.Context, decl EvaluatorDecl, localPath string) (version string, changed bool, err error) + // EnsureEvalGroup creates the group when it is absent or its resolved + // evaluators or options changed, returning its id. + EnsureEvalGroup(ctx context.Context, group EvalGroup, recreate bool) (id string, err error) +} + +// EvalServiceTargetProvider deploys eval resources during `azd up`. azd owns +// ordering across services through `uses:`; this provider owns only the order +// within the eval service itself. +type EvalServiceTargetProvider struct { + azdClient *azdext.AzdClient + newReconciler func(ctx context.Context) (Reconciler, error) + + serviceConfig *azdext.ServiceConfig + envName string +} + +// NewEvalServiceTargetProvider builds the provider. The reconciler is supplied +// lazily so the data-plane clients are only created when a deploy actually runs. +func NewEvalServiceTargetProvider( + azdClient *azdext.AzdClient, + newReconciler func(ctx context.Context) (Reconciler, error), +) *EvalServiceTargetProvider { + return &EvalServiceTargetProvider{azdClient: azdClient, newReconciler: newReconciler} +} + +func (p *EvalServiceTargetProvider) Initialize( + ctx context.Context, + serviceConfig *azdext.ServiceConfig, +) error { + p.serviceConfig = serviceConfig + return nil +} + +// Endpoints reports no endpoints: eval resources are not addressable. +func (p *EvalServiceTargetProvider) Endpoints( + ctx context.Context, + serviceConfig *azdext.ServiceConfig, + targetResource *azdext.TargetResource, +) ([]string, error) { + return nil, nil +} + +func (p *EvalServiceTargetProvider) GetTargetResource( + ctx context.Context, + subscriptionId string, + serviceConfig *azdext.ServiceConfig, + defaultResolver func() (*azdext.TargetResource, error), +) (*azdext.TargetResource, error) { + if defaultResolver != nil { + if target, err := defaultResolver(); err == nil { + return target, nil + } + } + // Eval resources live on the project data plane, so there is no ARM + // resource of our own to resolve. + return &azdext.TargetResource{SubscriptionId: subscriptionId}, nil +} + +// Package is a no-op: eval artifacts are plain files already on disk. +func (p *EvalServiceTargetProvider) Package( + ctx context.Context, + serviceConfig *azdext.ServiceConfig, + serviceContext *azdext.ServiceContext, + progress azdext.ProgressReporter, +) (*azdext.ServicePackageResult, error) { + return &azdext.ServicePackageResult{}, nil +} + +// Publish is a no-op: there is no artifact registry step for eval resources. +func (p *EvalServiceTargetProvider) Publish( + ctx context.Context, + serviceConfig *azdext.ServiceConfig, + serviceContext *azdext.ServiceContext, + targetResource *azdext.TargetResource, + publishOptions *azdext.PublishOptions, + progress azdext.ProgressReporter, +) (*azdext.ServicePublishResult, error) { + return &azdext.ServicePublishResult{}, nil +} + +// Deploy reconciles the eval configuration in a fixed order — datasets, then +// evaluators, then eval groups — because a group references the versions the +// first two resolve to. It fails fast; the next `azd up` resumes from wherever +// it stopped. +func (p *EvalServiceTargetProvider) Deploy( + ctx context.Context, + serviceConfig *azdext.ServiceConfig, + serviceContext *azdext.ServiceContext, + targetResource *azdext.TargetResource, + progress azdext.ProgressReporter, +) (*azdext.ServiceDeployResult, error) { + cfg, err := EvalConfigFromService(serviceConfig) + if err != nil { + return nil, err + } + if err := cfg.Validate(); err != nil { + return nil, fmt.Errorf("eval config is invalid: %w", err) + } + + reconciler, err := p.newReconciler(ctx) + if err != nil { + return nil, err + } + + baseDir := serviceRelativeDir(serviceConfig) + + // 1. Datasets. + anyChanged := false + for _, decl := range cfg.Datasets { + report(progress, fmt.Sprintf("Reconciling dataset %s", decl.Name)) + localPath := resolveSource(baseDir, decl.Source) + version, changed, err := reconciler.EnsureDataset(ctx, decl, localPath) + if err != nil { + return nil, fmt.Errorf("dataset %q: %w", decl.Name, err) + } + anyChanged = anyChanged || changed + report(progress, describeResult("dataset", decl.Name, version, changed)) + } + + // 2. Evaluators. + for _, decl := range cfg.Evaluators { + report(progress, fmt.Sprintf("Reconciling evaluator %s", decl.Name)) + localPath := resolveSource(baseDir, decl.Source) + version, changed, err := reconciler.EnsureEvaluator(ctx, decl, localPath) + if err != nil { + return nil, fmt.Errorf("evaluator %q: %w", decl.Name, err) + } + anyChanged = anyChanged || changed + report(progress, describeResult("evaluator", decl.Name, version, changed)) + } + + // 3. Eval groups. Groups are immutable, so a change upstream means a new + // group must be created and the stored id replaced. + for _, group := range cfg.EvalGroups { + report(progress, fmt.Sprintf("Reconciling eval group %s", group.Name)) + id, err := reconciler.EnsureEvalGroup(ctx, group, anyChanged) + if err != nil { + return nil, fmt.Errorf("eval group %q: %w", group.Name, err) + } + report(progress, fmt.Sprintf("Eval group %s is %s", group.Name, id)) + } + + return &azdext.ServiceDeployResult{}, nil +} + +// describeResult reports whether a version was published or reused, so a +// no-op deploy is visibly a no-op. +func describeResult(kind, name, version string, changed bool) string { + if changed { + return fmt.Sprintf("Published %s %s version %s", kind, name, version) + } + return fmt.Sprintf("%s %s is unchanged at version %s", strings.ToUpper(kind[:1])+kind[1:], name, version) +} + +func report(progress azdext.ProgressReporter, message string) { + if progress != nil { + progress(message) + } +} + +// EvalConfigFromService reads the eval configuration carried inline on the +// service entry. azd captures unknown keys into AdditionalProperties and hands +// them to the extension untouched. +func EvalConfigFromService(svc *azdext.ServiceConfig) (*EvalConfig, error) { + props := serviceProps(svc) + if props == nil || len(props.GetFields()) == 0 { + return nil, fmt.Errorf( + "service %q carries no eval configuration; expected evaluators, datasets, or evalGroups", + svc.GetName()) + } + + raw, err := props.MarshalJSON() + if err != nil { + return nil, fmt.Errorf("reading the eval service configuration: %w", err) + } + + var cfg EvalConfig + if err := json.Unmarshal(raw, &cfg); err != nil { + return nil, fmt.Errorf("parsing the eval service configuration: %w", err) + } + return &cfg, nil +} + +// serviceProps prefers the inline properties, falling back to the nested +// config block. +func serviceProps(svc *azdext.ServiceConfig) *structpb.Struct { + if s := svc.GetAdditionalProperties(); s != nil && len(s.GetFields()) > 0 { + return s + } + return svc.GetConfig() +} + +// serviceRelativeDir returns the directory that `source:` paths resolve against. +func serviceRelativeDir(svc *azdext.ServiceConfig) string { + if svc == nil { + return "." + } + if p := svc.GetRelativePath(); p != "" { + return p + } + return "." +} + +// resolveSource joins a declared source against the service directory, leaving +// absolute paths and empty values alone. +func resolveSource(baseDir, source string) string { + if source == "" { + return "" + } + if filepath.IsAbs(source) { + return source + } + return filepath.Join(baseDir, source) +} + +// Fingerprint hashes a local artifact so a later deploy can tell whether the +// content changed without downloading anything from the service. +// +// The dataset API returns no content hash or etag, so comparing against the +// service would mean downloading the blob on every deploy. +func Fingerprint(path string) (string, error) { + data, err := os.ReadFile(path) + if err != nil { + return "", fmt.Errorf("hashing %q: %w", path, err) + } + sum := sha256.Sum256(data) + return hex.EncodeToString(sum[:]), nil +} + +// FingerprintKey is the azd environment key holding an artifact's fingerprint. +func FingerprintKey(kind, name string) string { + safe := strings.Map(func(r rune) rune { + switch { + case r >= 'A' && r <= 'Z', r >= '0' && r <= '9': + return r + case r >= 'a' && r <= 'z': + return r - 32 + default: + return '_' + } + }, kind+"_"+name) + return EnvKeyFingerprintPrefix + safe +} From 065e84f442077aebb22a50ac0325c1fe0cdb7a2c Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 28 Jul 2026 05:23:27 +0300 Subject: [PATCH 008/320] feat(azure.ai.evaluations): generate command with YAML write-back - generate submits the rubric and dataset generation jobs, downloads the artifacts locally, and writes source references into the deployment spec - MergeArtifactRefs edits through the yaml Node API so comments, key order, and hand-edited sibling keys survive; matching is by name and merging is idempotent - Raised the client poll budget from 2s x 300 to 5s x 720. The old 10 minute limit gave up while the service was still working, which is the timeout that forced a second command - A supplied --evaluator or a local --dataset is honored and its generation is skipped - Tests cover comment preservation, section creation, idempotence, and fingerprinting --- .../internal/cmd/generate.go | 360 ++++++++++++++++++ .../azure.ai.evaluations/internal/cmd/root.go | 1 + .../internal/project/merge.go | 171 +++++++++ .../internal/project/merge_test.go | 142 +++++++ 4 files changed, 674 insertions(+) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/project/merge.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/project/merge_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go new file mode 100644 index 00000000000..3ce03cbfd65 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go @@ -0,0 +1,360 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "context" + "encoding/json" + "fmt" + "io" + "os" + "path/filepath" + "strconv" + "strings" + "time" + + "azureaieval/internal/pkg/eval_api" + "azureaieval/internal/project" + + "github.com/spf13/cobra" +) + +// generatePollBudget replaces the inherited 2s x 300 (10 minute) client budget. +// The generation job is not gateway-capped; the old limit simply gave up while +// the service was still working, forcing a second command. +var generatePollBudget = eval_api.PollerOptions{ + Interval: 5 * time.Second, + MaxAttempts: 720, // one hour +} + +func newGenerateCommand() *cobra.Command { + var ( + configPath string + deployPath string + target string + instruction string + datasetFlag string + evaluators []string + maxSamples int + traceDays int + evalModel string + noWait bool + endpointFlg string + ) + + cmd := &cobra.Command{ + Use: "generate", + Short: "Generate a rubric and dataset, download them, and reference them from the deployment spec.", + RunE: func(cmd *cobra.Command, args []string) error { + ctx := cmd.Context() + out := cmd.OutOrStdout() + + cfg, err := resolveGenerateConfig( + configPath, target, evalModel, datasetFlag, maxSamples, traceDays, + ) + if err != nil { + return err + } + if err := cfg.Validate(); err != nil { + return err + } + + ec, err := newEvalContext(ctx, endpointFlg) + if err != nil { + return err + } + defer ec.Close() + + baseDir := filepath.Dir(deployPath) + var datasetRefs, evaluatorRefs []project.ArtifactRef + + // Supplied evaluators are honored: their generation is skipped. + if len(evaluators) > 0 { + fmt.Fprintf(out, "Using the supplied evaluators; skipping rubric generation.\n") + } else if cfg.Generate.Rubric != nil { + ref, err := ec.generateRubric(ctx, cfg, instruction, baseDir, out, noWait) + if err != nil { + return err + } + if ref != nil { + evaluatorRefs = append(evaluatorRefs, *ref) + } + } + + if datasetFlag != "" && looksLikeLocalDataset(datasetFlag) { + fmt.Fprintf(out, "Using the supplied dataset; skipping data generation.\n") + } else if cfg.Generate.Dataset != nil { + ref, err := ec.generateDataset(ctx, cfg, instruction, baseDir, out, noWait) + if err != nil { + return err + } + if ref != nil { + datasetRefs = append(datasetRefs, *ref) + } + } + + if len(datasetRefs) == 0 && len(evaluatorRefs) == 0 { + fmt.Fprintln(out, "Nothing was generated.") + return nil + } + + if err := project.MergeArtifactRefs(deployPath, datasetRefs, evaluatorRefs); err != nil { + return err + } + fmt.Fprintf(out, "\nUpdated %s\n", deployPath) + fmt.Fprintln(out, "Review the generated artifacts, then run: azd up && azd ai eval run") + return nil + }, + } + + cmd.Flags().StringVar(&configPath, "config", project.DefaultGenerateConfig, + "Path to the generation spec. Optional; flags alone are sufficient.") + cmd.Flags().StringVar(&deployPath, "deploy-config", project.DefaultDeployConfig, + "Deployment spec to write source references into.") + cmd.Flags().StringVar(&target, "target", "", "Agent whose context seeds generation.") + cmd.Flags().StringVar(&instruction, "gen-instruction", "", + "What the agent does and what to test.") + cmd.Flags().StringVar(&datasetFlag, "dataset", "", + "Use this dataset instead of generating one.") + cmd.Flags().StringArrayVar(&evaluators, "evaluator", nil, + "Use these evaluators instead of generating a rubric; repeatable.") + cmd.Flags().IntVar(&maxSamples, "max-samples", 0, + fmt.Sprintf("Rows to synthesize (%d-%d).", project.MinSampleSize, project.MaxSampleSize)) + cmd.Flags().IntVar(&traceDays, "trace-days", 0, + "Days of traces to seed rubric generation. 0 disables.") + cmd.Flags().StringVar(&evalModel, "eval-model", "", "Model deployment used for generation.") + cmd.Flags().BoolVar(&noWait, "no-wait", false, "Submit the jobs and return without polling.") + cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") + return cmd +} + +// resolveGenerateConfig loads the spec when present, then layers flags on top. +// A missing file is not an error: flags alone are sufficient. +func resolveGenerateConfig( + path, target, evalModel, datasetFlag string, + maxSamples, traceDays int, +) (*project.GenerateConfig, error) { + cfg := &project.GenerateConfig{} + + if _, err := os.Stat(path); err == nil { + loaded, err := project.LoadGenerateConfig(path) + if err != nil { + return nil, err + } + cfg = loaded + } + + if target != "" { + cfg.Agent.Name = target + } + if cfg.Agent.Name == "" { + return nil, requireFlag("target") + } + + if cfg.Generate.Rubric == nil { + cfg.Generate.Rubric = &project.RubricSpec{ + Name: cfg.Agent.Name + "-quality", + LocalDir: "./" + project.DefaultEvaluatorsDir, + } + } + if cfg.Generate.Dataset == nil && !looksLikeLocalDataset(datasetFlag) { + cfg.Generate.Dataset = &project.DatasetSpec{ + Name: cfg.Agent.Name + "-golden", + Strategy: project.StrategySynthetic, + SampleSize: project.DefaultSampleSize, + LocalDir: "./" + project.DefaultDatasetsDir, + } + } + + if evalModel != "" { + cfg.Generate.Rubric.Model = evalModel + } + if maxSamples > 0 && cfg.Generate.Dataset != nil { + cfg.Generate.Dataset.SampleSize = maxSamples + } + if cfg.Generate.Dataset != nil && cfg.Generate.Dataset.SampleSize == 0 { + cfg.Generate.Dataset.SampleSize = project.DefaultSampleSize + } + if traceDays > 0 { + if cfg.Agent.Context.Traces == nil { + cfg.Agent.Context.Traces = &project.TraceSpec{} + } + cfg.Agent.Context.Traces.Window = fmt.Sprintf("%dd", traceDays) + } + + return cfg, nil +} + +// generateRubric submits the evaluator generation job and saves the rubric. +func (ec *evalContext) generateRubric( + ctx context.Context, + cfg *project.GenerateConfig, + instruction, baseDir string, + out io.Writer, + noWait bool, +) (*project.ArtifactRef, error) { + spec := cfg.Generate.Rubric + fmt.Fprintf(out, "Generating rubric %s...\n", spec.Name) + + sources := eval_api.BuildGenerationSources( + "agent", cfg.Agent.Name, "", instruction, traceOptions(cfg), + ) + req := eval_api.NewEvaluatorGenerationJobRequest(spec.Name, spec.Model, sources) + + job, err := ec.evalClient.CreateEvaluatorGenerationJob(ctx, req, ProjectEndpointAPIVersion) + if err != nil { + return nil, fmt.Errorf("submitting the rubric generation job: %w", err) + } + if noWait { + fmt.Fprintf(out, " submitted job %s\n", job.ID) + return nil, nil + } + + completed, err := ec.pollGeneration(ctx, job.ID, ProjectEndpointAPIVersion, + ec.evalClient.GetEvaluatorGenerationJob) + if err != nil { + return nil, fmt.Errorf("rubric generation: %w", err) + } + + path := project.ArtifactPath(baseDir, spec.LocalDir, spec.Name, ".json") + if err := writeRubric(path, completed.Result); err != nil { + return nil, err + } + fmt.Fprintf(out, " wrote %s\n", path) + + return &project.ArtifactRef{Name: spec.Name, Source: relativeSource(baseDir, path)}, nil +} + +// generateDataset submits the data generation job and downloads the result. +func (ec *evalContext) generateDataset( + ctx context.Context, + cfg *project.GenerateConfig, + instruction, baseDir string, + out io.Writer, + noWait bool, +) (*project.ArtifactRef, error) { + spec := cfg.Generate.Dataset + fmt.Fprintf(out, "Generating dataset %s (%d samples)...\n", spec.Name, spec.SampleSize) + + sources := eval_api.BuildGenerationSources( + "agent", cfg.Agent.Name, "", instruction, traceOptions(cfg), + ) + model := "" + if cfg.Generate.Rubric != nil { + model = cfg.Generate.Rubric.Model + } + req := eval_api.NewDataGenerationJobRequest(spec.Name, model, spec.SampleSize, sources) + + job, err := ec.evalClient.CreateDataGenerationJob(ctx, req, DataGenerationAPIVersion) + if err != nil { + return nil, fmt.Errorf("submitting the data generation job: %w", err) + } + if noWait { + fmt.Fprintf(out, " submitted job %s\n", job.ID) + return nil, nil + } + + completed, err := ec.pollGeneration(ctx, job.ID, DataGenerationAPIVersion, + ec.evalClient.GetDataGenerationJob) + if err != nil { + return nil, fmt.Errorf("data generation: %w", err) + } + + name, version := completed.ResolvedNameVersion() + if name == "" { + return nil, fmt.Errorf("the data generation job returned no dataset reference") + } + + ds, err := ec.datasetClient.GetDataset(ctx, name, version, ProjectEndpointAPIVersion) + if err != nil { + return nil, fmt.Errorf("reading the generated dataset %q: %w", name, err) + } + content, err := ec.datasetClient.DownloadDataset(ctx, ds.ResolvedBlobURI()) + if err != nil { + return nil, fmt.Errorf("downloading the generated dataset %q: %w", name, err) + } + + path := project.ArtifactPath(baseDir, spec.LocalDir, spec.Name, ".jsonl") + if err := os.MkdirAll(filepath.Dir(path), 0o750); err != nil { + return nil, fmt.Errorf("creating %q: %w", filepath.Dir(path), err) + } + if err := os.WriteFile(path, content, 0o600); err != nil { + return nil, fmt.Errorf("writing %q: %w", path, err) + } + fmt.Fprintf(out, " wrote %s\n", path) + + return &project.ArtifactRef{Name: spec.Name, Source: relativeSource(baseDir, path)}, nil +} + +// pollGeneration waits for a generation job using the raised budget. +func (ec *evalContext) pollGeneration( + ctx context.Context, + operationID, apiVersion string, + get eval_api.GetJobFunc, +) (*eval_api.GenerationJob, error) { + poller := eval_api.NewPoller(operationID, apiVersion, get) + poller.Options = generatePollBudget + return poller.Poll(ctx) +} + +// traceOptions converts the config's trace window into the generation client's +// day count. Traces seed rubric generation only; they are never a run's data +// source. +func traceOptions(cfg *project.GenerateConfig) *eval_api.TraceOptions { + t := cfg.Agent.Context.Traces + if t == nil { + return nil + } + days := parseWindowDays(t.Window) + if days <= 0 { + return nil + } + return &eval_api.TraceOptions{Days: days} +} + +// parseWindowDays reads a window such as "30d" or a bare day count. +func parseWindowDays(window string) int { + w := strings.TrimSpace(strings.ToLower(window)) + if w == "" { + return 0 + } + w = strings.TrimSuffix(w, "d") + days, err := strconv.Atoi(w) + if err != nil { + return 0 + } + return days +} + +// writeRubric persists only the rubric dimensions so the developer can edit +// weights and descriptions and publish a new version. +func writeRubric(path string, result json.RawMessage) error { + if len(result) == 0 { + return fmt.Errorf("the rubric generation job returned no result") + } + if err := os.MkdirAll(filepath.Dir(path), 0o750); err != nil { + return fmt.Errorf("creating %q: %w", filepath.Dir(path), err) + } + + var parsed eval_api.EvaluatorResult + if err := json.Unmarshal(result, &parsed); err == nil && len(parsed.Definition.Dimensions) > 0 { + body, err := json.MarshalIndent(parsed.Definition, "", " ") + if err != nil { + return err + } + return os.WriteFile(path, body, 0o600) + } + + // Fall back to the raw payload rather than losing the result. + return os.WriteFile(path, result, 0o600) +} + +// relativeSource expresses an artifact path relative to the deployment spec. +func relativeSource(baseDir, path string) string { + rel, err := filepath.Rel(baseDir, path) + if err != nil { + return filepath.ToSlash(path) + } + return "./" + filepath.ToSlash(rel) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/root.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/root.go index 671c10fe0ad..3a289d6347d 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/root.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/root.go @@ -27,6 +27,7 @@ func NewRootCommand() *cobra.Command { rootCmd.AddCommand( newInitCommand(), + newGenerateCommand(), newDatasetCommand(), newRunCommand(), newEvaluatorCommand(), diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/merge.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/merge.go new file mode 100644 index 00000000000..c8329356c23 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/merge.go @@ -0,0 +1,171 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package project + +import ( + "fmt" + "os" + + "go.yaml.in/yaml/v3" +) + +// ArtifactRef is a name/source pair written back into the deployment spec after +// generation. +type ArtifactRef struct { + Name string + Source string +} + +// MergeArtifactRefs writes `source:` references for generated artifacts into +// the deployment spec, matching entries by name and appending when absent. +// +// It edits the document through the yaml Node API rather than round-tripping +// through structs, so comments, key order, and formatting survive. Only the +// `source` key of a matched entry is touched; anything the developer hand-edited +// is left alone. +func MergeArtifactRefs(path string, datasets, evaluators []ArtifactRef) error { + data, err := os.ReadFile(path) + if err != nil { + return fmt.Errorf("reading %q: %w", path, err) + } + + var doc yaml.Node + if err := yaml.Unmarshal(data, &doc); err != nil { + return fmt.Errorf("parsing %q: %w", path, err) + } + + root := documentRoot(&doc) + if root == nil { + return fmt.Errorf("%q is not a YAML mapping", path) + } + + if err := mergeSection(root, "datasets", datasets); err != nil { + return fmt.Errorf("%q: %w", path, err) + } + if err := mergeSection(root, "evaluators", evaluators); err != nil { + return fmt.Errorf("%q: %w", path, err) + } + + out, err := yaml.Marshal(&doc) + if err != nil { + return fmt.Errorf("serializing %q: %w", path, err) + } + if err := os.WriteFile(path, out, 0o600); err != nil { + return fmt.Errorf("writing %q: %w", path, err) + } + return nil +} + +// documentRoot unwraps the document node to the top-level mapping. +func documentRoot(doc *yaml.Node) *yaml.Node { + if doc.Kind == yaml.DocumentNode && len(doc.Content) > 0 { + doc = doc.Content[0] + } + if doc.Kind != yaml.MappingNode { + return nil + } + return doc +} + +// mergeSection updates or appends entries in a top-level sequence. +func mergeSection(root *yaml.Node, key string, refs []ArtifactRef) error { + if len(refs) == 0 { + return nil + } + + seq := findOrCreateSequence(root, key) + if seq == nil { + return fmt.Errorf("%q is present but is not a sequence", key) + } + + for _, ref := range refs { + if entry := findEntryByName(seq, ref.Name); entry != nil { + setMappingValue(entry, "source", ref.Source) + continue + } + seq.Content = append(seq.Content, newArtifactNode(ref)) + } + return nil +} + +// findOrCreateSequence returns the sequence node for key, creating an empty one +// when the key is absent. +func findOrCreateSequence(root *yaml.Node, key string) *yaml.Node { + for i := 0; i+1 < len(root.Content); i += 2 { + if root.Content[i].Value != key { + continue + } + value := root.Content[i+1] + if value.Kind == yaml.SequenceNode { + return value + } + // An explicit null is treated as an empty sequence. + if value.Tag == "!!null" { + value.Kind = yaml.SequenceNode + value.Tag = "!!seq" + value.Value = "" + return value + } + return nil + } + + keyNode := &yaml.Node{Kind: yaml.ScalarNode, Tag: "!!str", Value: key} + seqNode := &yaml.Node{Kind: yaml.SequenceNode, Tag: "!!seq"} + root.Content = append(root.Content, keyNode, seqNode) + return seqNode +} + +// findEntryByName locates a mapping entry whose `name` matches. +func findEntryByName(seq *yaml.Node, name string) *yaml.Node { + for _, item := range seq.Content { + if item.Kind != yaml.MappingNode { + continue + } + if mappingValue(item, "name") == name { + return item + } + } + return nil +} + +// mappingValue reads a scalar value from a mapping node. +func mappingValue(node *yaml.Node, key string) string { + for i := 0; i+1 < len(node.Content); i += 2 { + if node.Content[i].Value == key { + return node.Content[i+1].Value + } + } + return "" +} + +// setMappingValue updates a scalar in place, or appends it when absent. Only +// the targeted key is touched. +func setMappingValue(node *yaml.Node, key, value string) { + for i := 0; i+1 < len(node.Content); i += 2 { + if node.Content[i].Value == key { + node.Content[i+1].Kind = yaml.ScalarNode + node.Content[i+1].Tag = "!!str" + node.Content[i+1].Value = value + return + } + } + node.Content = append(node.Content, + &yaml.Node{Kind: yaml.ScalarNode, Tag: "!!str", Value: key}, + &yaml.Node{Kind: yaml.ScalarNode, Tag: "!!str", Value: value}, + ) +} + +// newArtifactNode builds a fresh `{name, source}` entry. +func newArtifactNode(ref ArtifactRef) *yaml.Node { + return &yaml.Node{ + Kind: yaml.MappingNode, + Tag: "!!map", + Content: []*yaml.Node{ + {Kind: yaml.ScalarNode, Tag: "!!str", Value: "name"}, + {Kind: yaml.ScalarNode, Tag: "!!str", Value: ref.Name}, + {Kind: yaml.ScalarNode, Tag: "!!str", Value: "source"}, + {Kind: yaml.ScalarNode, Tag: "!!str", Value: ref.Source}, + }, + } +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/merge_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/merge_test.go new file mode 100644 index 00000000000..b253c8ad1f2 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/merge_test.go @@ -0,0 +1,142 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package project + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" +) + +const handAuthored = `# Eval deployment spec +# Edited by hand - comments must survive generate. +evaluators: + - name: safety-check # hand-authored + source: ./evaluators/safety-check.json + +datasets: + - name: support-golden + source: ./datasets/old.jsonl + version: "3" + +evalGroups: + - name: pr-gate + dataset: support-golden + evaluators: + - builtin.task_adherence + - { name: safety-check, threshold: 4.0 } +` + +func writeTemp(t *testing.T, body string) string { + t.Helper() + path := filepath.Join(t.TempDir(), "azure.yaml") + require.NoError(t, os.WriteFile(path, []byte(body), 0o600)) + return path +} + +// Regenerating must not destroy a hand-edited file. +func TestMergeArtifactRefs_PreservesCommentsAndSiblings(t *testing.T) { + path := writeTemp(t, handAuthored) + + require.NoError(t, MergeArtifactRefs(path, + []ArtifactRef{{Name: "support-golden", Source: "./datasets/new.jsonl"}}, + []ArtifactRef{{Name: "support-quality", Source: "./evaluators/support-quality.json"}}, + )) + + out, err := os.ReadFile(path) + require.NoError(t, err) + text := string(out) + + require.Contains(t, text, "# Eval deployment spec", "leading comments must survive") + require.Contains(t, text, "# hand-authored", "inline comments must survive") + require.Contains(t, text, "./datasets/new.jsonl", "the matched source must be updated") + require.NotContains(t, text, "./datasets/old.jsonl", "the old source must be replaced") + require.Contains(t, text, "support-quality", "a new evaluator must be appended") + require.Contains(t, text, "safety-check", "existing entries must be kept") + + cfg, err := LoadEvalConfig(path) + require.NoError(t, err) + require.NoError(t, cfg.Validate()) + + ds, ok := cfg.Dataset("support-golden") + require.True(t, ok) + require.Equal(t, "./datasets/new.jsonl", ds.Source) + require.Equal(t, "3", ds.Version, "sibling keys must not be disturbed") + require.Len(t, cfg.Evaluators, 2) +} + +// The eval group's evaluator list must be left exactly as written. +func TestMergeArtifactRefs_DoesNotTouchEvalGroups(t *testing.T) { + path := writeTemp(t, handAuthored) + require.NoError(t, MergeArtifactRefs(path, nil, + []ArtifactRef{{Name: "support-quality", Source: "./evaluators/q.json"}})) + + cfg, err := LoadEvalConfig(path) + require.NoError(t, err) + g, ok := cfg.Group("pr-gate") + require.True(t, ok) + require.Len(t, g.Evaluators, 2) + require.Equal(t, "builtin.task_adherence", g.Evaluators[0].Name) + require.NotNil(t, g.Evaluators[1].Threshold) +} + +// Sections absent from the file are created rather than erroring. +func TestMergeArtifactRefs_CreatesMissingSections(t *testing.T) { + path := writeTemp(t, "evalGroups:\n - name: pr-gate\n evaluators: [builtin.relevance]\n") + + require.NoError(t, MergeArtifactRefs(path, + []ArtifactRef{{Name: "d1", Source: "./datasets/d1.jsonl"}}, + []ArtifactRef{{Name: "e1", Source: "./evaluators/e1.json"}}, + )) + + cfg, err := LoadEvalConfig(path) + require.NoError(t, err) + require.Len(t, cfg.Datasets, 1) + require.Len(t, cfg.Evaluators, 1) + require.Equal(t, "d1", cfg.Datasets[0].Name) +} + +// Running generate twice must be idempotent. +func TestMergeArtifactRefs_IsIdempotent(t *testing.T) { + path := writeTemp(t, handAuthored) + refs := []ArtifactRef{{Name: "support-golden", Source: "./datasets/new.jsonl"}} + + require.NoError(t, MergeArtifactRefs(path, refs, nil)) + first, err := os.ReadFile(path) + require.NoError(t, err) + + require.NoError(t, MergeArtifactRefs(path, refs, nil)) + second, err := os.ReadFile(path) + require.NoError(t, err) + + require.Equal(t, string(first), string(second), + "merging the same references twice must not change the file") +} + +func TestFingerprint_DetectsChange(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "data.jsonl") + + require.NoError(t, os.WriteFile(path, []byte(`{"query":"a"}`), 0o600)) + first, err := Fingerprint(path) + require.NoError(t, err) + + again, err := Fingerprint(path) + require.NoError(t, err) + require.Equal(t, first, again, "unchanged content must hash the same") + + require.NoError(t, os.WriteFile(path, []byte(`{"query":"b"}`), 0o600)) + changed, err := Fingerprint(path) + require.NoError(t, err) + require.NotEqual(t, first, changed, "changed content must hash differently") +} + +func TestFingerprintKey_IsEnvSafe(t *testing.T) { + require.Equal(t, "EVAL_FINGERPRINT_DATASET_SUPPORT_GOLDEN", + FingerprintKey("dataset", "support-golden")) + require.Equal(t, "EVAL_FINGERPRINT_EVALUATOR_MY_EVAL_1", + FingerprintKey("evaluator", "my.eval-1")) +} From 38fb856537621e85fc3657b8817ed41922c5ecb2 Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 28 Jul 2026 05:25:03 +0300 Subject: [PATCH 009/320] build(azure.ai.evaluations): release pipeline and foundry bundle entry Adds the templated release pipeline for the new extension and lists it as a dependency of the microsoft.foundry meta-package. The registry.json artifact entries are generated by the release, so they are not hand-authored here. --- .../microsoft.foundry/extension.yaml | 2 + .../release-ext-azure-ai-evaluations.yml | 46 +++++++++++++++++++ 2 files changed, 48 insertions(+) create mode 100644 eng/pipelines/release-ext-azure-ai-evaluations.yml diff --git a/cli/azd/extensions/microsoft.foundry/extension.yaml b/cli/azd/extensions/microsoft.foundry/extension.yaml index 336bc749643..7c96b52bd9f 100644 --- a/cli/azd/extensions/microsoft.foundry/extension.yaml +++ b/cli/azd/extensions/microsoft.foundry/extension.yaml @@ -12,6 +12,8 @@ dependencies: version: "~1.0.0-beta.7" - id: azure.ai.connections version: "~1.0.0-beta.1" + - id: azure.ai.evaluations + version: "~1.0.0-beta.1" - id: azure.ai.inspector version: "~1.0.0-beta.1" - id: azure.ai.projects diff --git a/eng/pipelines/release-ext-azure-ai-evaluations.yml b/eng/pipelines/release-ext-azure-ai-evaluations.yml new file mode 100644 index 00000000000..d8b60935909 --- /dev/null +++ b/eng/pipelines/release-ext-azure-ai-evaluations.yml @@ -0,0 +1,46 @@ +# Continuous deployment trigger +trigger: + branches: + include: + - main + paths: + include: + - go.mod + - cli/azd/extensions/azure.ai.evaluations + - eng/pipelines/release-azd-extension.yml + - /eng/pipelines/templates/jobs/build-azd-extension.yml + - /eng/pipelines/templates/jobs/cross-build-azd-extension.yml + - /eng/pipelines/templates/variables/image.yml + +pr: + paths: + include: + - cli/azd/extensions/azure.ai.evaluations + - eng/pipelines/release-ext-azure-ai-evaluations.yml + - eng/pipelines/release-azd-extension.yml + - eng/pipelines/templates/steps/publish-cli.yml + exclude: + - cli/azd/docs/** + +parameters: + - name: PublishToRegistry + displayName: Publish to registry + type: string + # Scheduled (nightly) runs override this in the shared templates; the runtime + # parameter default must be a literal because it renders before variables exist. + default: stable + values: + - stable + - dev + - nightly + +extends: + template: /eng/pipelines/templates/stages/1es-redirect.yml + parameters: + stages: + - template: /eng/pipelines/templates/stages/release-azd-extension.yml + parameters: + AzdExtensionId: azure.ai.evaluations + SanitizedExtensionId: azure-ai-evaluations + AzdExtensionDirectory: cli/azd/extensions/azure.ai.evaluations + PublishToRegistry: ${{ parameters.PublishToRegistry }} From 3b9cb935fad16ac33ddfc96c95904c3792797e99 Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 28 Jul 2026 05:29:44 +0300 Subject: [PATCH 010/320] fix(azure.ai.evaluations): bind dataset URIs and add live integration tests Live testing against a real project found two issues. 1. The dataset model only bound snake_case URIs (data_uri, blob_uri), but the project endpoint returns camelCase (dataUri). ResolvedBlobURI therefore returned empty, which would have failed the generate download much later with no useful error. Both spellings are now accepted. 2. Built-in evaluators do not share one input contract. builtin.ifeval requires instruction_id_list and is rejected under the agent-target data mapping with MissingRequiredDataMapping. The live tests now select an evaluator whose inputs match, and the helper documents why. Live tests are gated behind the live build tag and AZURE_AI_EVAL_E2E_LIVE, and clean up every resource they create. Verified: builtin listing, the full dataset pending-upload lifecycle with version auto-increment, and eval group creation returning 201. --- .../internal/pkg/dataset_api/models.go | 46 ++- .../tests/live/live_test.go | 325 ++++++++++++++++++ 2 files changed, 357 insertions(+), 14 deletions(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/tests/live/live_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/models.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/models.go index cee6bbcd886..7e6c8633c09 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/models.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/models.go @@ -21,28 +21,46 @@ type CreateDatasetRequest struct { } // Dataset is the response for dataset operations. -// Note: The GET /datasets API returns snake_case field names (data_uri, blob_uri, -// content_uri), while the POST /finalize API accepts camelCase (dataUri). -// Both conventions are correct for their respective endpoints. +// +// The field spelling is not consistent across the surface: the live +// project-endpoint GET returns camelCase (dataUri, isSingleFile), while other +// paths have used snake_case (data_uri, blob_uri, content_uri). Both spellings +// are accepted here because binding only one silently yields an empty URI, +// which then fails much later at download time. type Dataset struct { - Name string `json:"name"` - Version string `json:"version"` + ID string `json:"id,omitempty"` + Name string `json:"name"` + Version string `json:"version"` + Type string `json:"type,omitempty"` + Format string `json:"format,omitempty"` + + // camelCase spellings (project endpoint). + DataURICamel string `json:"dataUri,omitempty"` + BlobURICamel string `json:"blobUri,omitempty"` + ContentURICamel string `json:"contentUri,omitempty"` + IsSingleFile bool `json:"isSingleFile,omitempty"` + ConnectionName string `json:"connectionName,omitempty"` + + // snake_case spellings. BlobURI string `json:"blob_uri,omitempty"` - Format string `json:"format,omitempty"` DataURI string `json:"data_uri,omitempty"` ContentURI string `json:"content_uri,omitempty"` } -// ResolvedBlobURI returns the best available blob URI. Prefers blob_uri, -// falls back to data_uri, then content_uri. +// ResolvedBlobURI returns the first URI the service supplied, across both +// spellings. An empty result means the dataset carries no downloadable URI and +// the caller must fetch a credential instead. func (d *Dataset) ResolvedBlobURI() string { - if d.BlobURI != "" { - return d.BlobURI - } - if d.DataURI != "" { - return d.DataURI + for _, candidate := range []string{ + d.BlobURI, d.BlobURICamel, + d.DataURI, d.DataURICamel, + d.ContentURI, d.ContentURICamel, + } { + if candidate != "" { + return candidate + } } - return d.ContentURI + return "" } // DatasetCredential is the response for dataset credential (SAS token) requests. diff --git a/cli/azd/extensions/azure.ai.evaluations/tests/live/live_test.go b/cli/azd/extensions/azure.ai.evaluations/tests/live/live_test.go new file mode 100644 index 00000000000..c04af2d47a6 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/tests/live/live_test.go @@ -0,0 +1,325 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +//go:build live + +// Package live holds integration tests that talk to a real Foundry project. +// They are excluded from the default build by the `live` tag and additionally +// gated on AZURE_AI_EVAL_E2E_LIVE so an accidental run cannot create resources. +// +// go test -tags live -v ./tests/live/... +// +// Required: +// +// AZURE_AI_EVAL_E2E_LIVE=1 +// FOUNDRY_PROJECT_ENDPOINT=https://.services.ai.azure.com/api/projects/ +// +// Optional: +// +// AZURE_AI_EVAL_MODEL= (default gpt-4.1-nano) +// AZURE_AI_EVAL_AGENT= (enables the run phase) +package live + +import ( + "context" + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "azureaieval/internal/pkg/dataset_api" + "azureaieval/internal/pkg/eval_api" + + "github.com/Azure/azure-sdk-for-go/sdk/azidentity" + "github.com/stretchr/testify/require" +) + +const ( + projectAPIVersion = "2025-11-15-preview" + defaultJudgeModel = "gpt-4.1-nano" + sampleDatasetContent = `{"query":"How do I reset my password?"} +{"query":"What is the refund window?"} +{"query":"Can I change my shipping address after ordering?"} +` +) + +type liveEnv struct { + endpoint string + judgeModel string + agentName string + evalClient *eval_api.EvalClient + datasetClient *dataset_api.DatasetClient +} + +func setup(t *testing.T) *liveEnv { + t.Helper() + + if os.Getenv("AZURE_AI_EVAL_E2E_LIVE") != "1" { + t.Skip("set AZURE_AI_EVAL_E2E_LIVE=1 to run live tests") + } + endpoint := strings.TrimSuffix(os.Getenv("FOUNDRY_PROJECT_ENDPOINT"), "/") + if endpoint == "" { + t.Fatal("FOUNDRY_PROJECT_ENDPOINT is required") + } + + // The azd developer CLI credential works non-interactively when azd already + // holds a refresh token, which is what makes an unattended run possible. + cred, err := azidentity.NewAzureDeveloperCLICredential( + &azidentity.AzureDeveloperCLICredentialOptions{}, + ) + require.NoError(t, err, "acquiring an azd credential") + + judge := os.Getenv("AZURE_AI_EVAL_MODEL") + if judge == "" { + judge = defaultJudgeModel + } + + return &liveEnv{ + endpoint: endpoint, + judgeModel: judge, + agentName: os.Getenv("AZURE_AI_EVAL_AGENT"), + evalClient: eval_api.NewEvalClient(endpoint, cred), + datasetClient: dataset_api.NewDatasetClient(endpoint, cred), + } +} + +func uniqueName(prefix string) string { + return fmt.Sprintf("%s-%d", prefix, time.Now().UTC().Unix()) +} + +// pickQualityEvaluator selects a built-in whose required inputs match the +// agent-target data mapping this extension sends. +// +// Built-ins do not share one input contract: builtin.ifeval, for example, +// requires an `instruction_id_list` field, and creating a group with it under +// the agent-target mapping fails with MissingRequiredDataMapping. The +// agent-target mapping supplies query, response, tool_calls and +// tool_definitions, so the evaluators below are the compatible set. +func pickQualityEvaluator(t *testing.T, available []eval_api.EvaluatorSummary) string { + t.Helper() + + preferred := []string{ + "builtin.task_adherence", + "builtin.task_completion", + "builtin.tool_call_accuracy", + } + present := map[string]bool{} + for _, e := range available { + present[e.Name] = true + } + for _, name := range preferred { + if present[name] { + return name + } + } + + names := make([]string, 0, len(available)) + for _, e := range available { + names = append(names, e.Name) + } + t.Skipf("no agent-target compatible evaluator found; available: %s", strings.Join(names, ", ")) + return "" +} + +// TestLiveBuiltinEvaluators is the cheapest reachability check: it proves the +// endpoint, credential, api-version, and auth scope are all correct without +// creating anything. +func TestLiveBuiltinEvaluators(t *testing.T) { + env := setup(t) + ctx := context.Background() + + list, err := env.evalClient.ListEvaluators( + ctx, eval_api.EvaluatorTypeBuiltin, projectAPIVersion, + ) + require.NoError(t, err, "listing built-in evaluators") + require.NotEmpty(t, list.Value, "the project should expose built-in evaluators") + + t.Logf("found %d built-in evaluators; first: %s", len(list.Value), list.Value[0].Name) +} + +// TestLiveDatasetLifecycle exercises the full pending-upload flow and asserts +// that re-registering the same name yields the next version rather than an error. +func TestLiveDatasetLifecycle(t *testing.T) { + env := setup(t) + ctx := context.Background() + + dir := t.TempDir() + require.NoError(t, + os.WriteFile(filepath.Join(dir, "golden.jsonl"), []byte(sampleDatasetContent), 0o600)) + + name := uniqueName("azd-eval-e2e") + + first, err := env.datasetClient.UploadNewVersion(ctx, name, "", dir, projectAPIVersion) + require.NoError(t, err, "registering the first dataset version") + require.Equal(t, name, first.Name) + require.NotEmpty(t, first.Version) + t.Logf("registered %s version %s", first.Name, first.Version) + + t.Cleanup(func() { + // Best effort: leave nothing behind even if the test fails midway. + _ = env.datasetClient.DeleteDatasetVersion( + context.Background(), name, first.Version, projectAPIVersion) + }) + + fetched, err := env.datasetClient.GetDataset(ctx, name, first.Version, projectAPIVersion) + require.NoError(t, err, "reading the dataset back") + t.Logf("dataset uri: %q (empty means a credential call is required)", fetched.ResolvedBlobURI()) + + versions, err := env.datasetClient.ListDatasetVersions(ctx, name, projectAPIVersion) + require.NoError(t, err, "listing dataset versions") + require.NotEmpty(t, versions.Value) + require.Equal(t, first.Version, dataset_api.LatestVersion(versions.Value)) + + // A second upload must advance the version, not conflict. + second, err := env.datasetClient.UploadNewVersion( + ctx, name, first.Version, dir, projectAPIVersion) + require.NoError(t, err, "registering a second dataset version") + require.NotEqual(t, first.Version, second.Version, + "re-registering the same name must produce the next version") + t.Cleanup(func() { + _ = env.datasetClient.DeleteDatasetVersion( + context.Background(), name, second.Version, projectAPIVersion) + }) +} + +// TestLiveEvalGroupLifecycle proves the create request this extension builds is +// accepted, which is the single most important contract to get right. +func TestLiveEvalGroupLifecycle(t *testing.T) { + env := setup(t) + ctx := context.Background() + + builtins, err := env.evalClient.ListEvaluators( + ctx, eval_api.EvaluatorTypeBuiltin, projectAPIVersion) + require.NoError(t, err) + require.NotEmpty(t, builtins.Value, "need at least one built-in evaluator") +evaluatorName := pickQualityEvaluator(t, builtins.Value) + + threshold := 3.0 + req := &eval_api.CreateOpenAIEvalRequest{ + Name: uniqueName("azd-eval-e2e-group"), + Metadata: map[string]string{"azd_source": "e2e"}, + DataSourceConfig: &eval_api.DataSourceConfig{ + Type: "custom", + IncludeSampleSchema: true, + ItemSchema: map[string]any{ + "type": "object", + "properties": map[string]any{"query": map[string]any{"type": "string"}}, + }, + }, + TestingCriteria: []eval_api.TestingCriterion{{ + Type: "azure_ai_evaluator", + Name: strings.TrimPrefix(evaluatorName, "builtin."), + EvaluatorName: evaluatorName, + DataMapping: map[string]string{ + "query": "{{item.query}}", + "response": "{{sample.output_items}}", + "tool_calls": "{{sample.tool_calls}}", + "tool_definitions": "{{sample.tool_definitions}}", + }, + InitializationParameters: map[string]any{ + "model": env.judgeModel, + "deployment_name": env.judgeModel, + "threshold": threshold, + }, + }}, + } + + group, err := env.evalClient.CreateOpenAIEval(ctx, req) + require.NoError(t, err, "creating the eval group") + require.NotEmpty(t, group.ID, "the service assigns the id; name is not unique") + t.Logf("created eval group %s (name %q)", group.ID, group.Name) + + fetched, err := env.evalClient.GetOpenAIEval(ctx, group.ID) + require.NoError(t, err, "reading the eval group back") + require.Equal(t, group.ID, fetched.ID) +} + +// TestLiveRun invokes a real agent, so it only runs when one is named. +func TestLiveRun(t *testing.T) { + env := setup(t) + if env.agentName == "" { + t.Skip("set AZURE_AI_EVAL_AGENT to a deployed agent to exercise the run phase") + } + ctx := context.Background() + + builtins, err := env.evalClient.ListEvaluators( + ctx, eval_api.EvaluatorTypeBuiltin, projectAPIVersion) + require.NoError(t, err) + require.NotEmpty(t, builtins.Value) +evaluatorName := pickQualityEvaluator(t, builtins.Value) + + group, err := env.evalClient.CreateOpenAIEval(ctx, &eval_api.CreateOpenAIEvalRequest{ + Name: uniqueName("azd-eval-e2e-run"), + DataSourceConfig: &eval_api.DataSourceConfig{ + Type: "custom", + IncludeSampleSchema: true, + ItemSchema: map[string]any{ + "type": "object", + "properties": map[string]any{"query": map[string]any{"type": "string"}}, + }, + }, + TestingCriteria: []eval_api.TestingCriterion{{ + Type: "azure_ai_evaluator", + Name: strings.TrimPrefix(evaluatorName, "builtin."), + EvaluatorName: evaluatorName, + DataMapping: map[string]string{ + "query": "{{item.query}}", + "response": "{{sample.output_items}}", + "tool_calls": "{{sample.tool_calls}}", + "tool_definitions": "{{sample.tool_definitions}}", + }, + InitializationParameters: map[string]any{ + "model": env.judgeModel, + "deployment_name": env.judgeModel, + }, + }}, + }) + require.NoError(t, err, "creating the eval group for the run") + + ds := eval_api.NewAgentTargetDataSource(env.agentName, nil) + ds.SetFileContent([]map[string]any{ + {"query": "How do I reset my password?"}, + }) + + run, err := env.evalClient.CreateOpenAIEvalRun(ctx, group.ID, &eval_api.CreateOpenAIEvalRunRequest{ + Name: uniqueName("run"), + DataSource: ds, + }) + require.NoError(t, err, "starting the run") + require.NotEmpty(t, run.ID) + t.Logf("started run %s (status %s)", run.ID, run.Status) + + t.Cleanup(func() { + _, _ = env.evalClient.CancelOpenAIEvalRun(context.Background(), group.ID, run.ID) + }) + + // A single sample is roughly 40 seconds; allow generous headroom. + deadline := time.Now().Add(10 * time.Minute) + terminal := map[string]bool{ + "completed": true, "failed": true, "canceled": true, "cancelled": true, "error": true, + } + for { + current, err := env.evalClient.GetOpenAIEvalRun(ctx, group.ID, run.ID) + require.NoError(t, err, "polling the run") + if terminal[strings.ToLower(current.Status)] { + t.Logf("run reached %s", current.Status) + if current.ResultCounts != nil { + t.Logf("counts: passed=%d failed=%d errored=%d", + current.ResultCounts.Passed, + current.ResultCounts.Failed, + current.ResultCounts.Errored) + } + body, _ := json.MarshalIndent(current.PerTestingCriteria, "", " ") + t.Logf("per-criteria results: %s", string(body)) + return + } + if time.Now().After(deadline) { + t.Fatalf("run %s did not finish within the deadline (last status %q)", + run.ID, current.Status) + } + time.Sleep(10 * time.Second) + } +} From b7aba9dae3cca902e6793320ae247e6cb5272841 Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 28 Jul 2026 05:45:03 +0300 Subject: [PATCH 011/320] fix(azure.ai.evaluations): build data mappings from each evaluator's published contract The builder sent one fixed data mapping and one fixed set of initialization parameters to every evaluator. That only suited agent-target quality evaluators, and the service rejected the rest. The evaluator listing publishes a full contract per evaluator: definition.data_schema (accepted and required inputs), definition.init_parameters, and supported_evaluation_levels. The builder now reads it and shapes each testing criterion accordingly. This fixes four concrete defects. - Required inputs were never honoured, so builtin.ifeval (instruction_id_list), builtin.similarity (ground_truth) and builtin.retrieval (context) all failed with MissingRequiredDataMapping. Fields not supplied by the agent target are now bound to dataset columns, and the item schema declares them. - Inputs an evaluator does not accept were sent anyway. - initialization_parameters always carried model, deployment_name and threshold. No evaluator accepts 'model', and builtin.ifeval accepts nothing at all. Parameters are now filtered to the declared properties, and a required one that is missing is reported locally. - evaluation_level was sent as run metadata, where it has no effect. It is an initialization parameter on the evaluators that declare it. It also encodes an exclusivity rule the service enforces: 'messages' and 'query'/'response' cannot both appear in a mapping, so the evaluation level selects between the conversation and turn shapes. A missing dataset column is now caught before the request is sent and names the column, rather than surfacing as a 400 pointing at testing_criteria[0].data_mapping. An evaluator with no published contract keeps the previous agent-target shape, so custom evaluators are unaffected. Verified against a live project: all ten built-ins are accepted, where two previously failed. The live test exercises the shipping builder rather than a hand-rolled request, so a regression in this logic fails the suite. Adds DeleteOpenAIEval so those tests clean up after themselves. --- .../internal/cmd/build.go | 320 +++++++++++++++--- .../internal/cmd/build_live_test.go | 142 ++++++++ .../internal/cmd/build_test.go | 277 +++++++++++++++ .../internal/cmd/reconciler.go | 11 +- .../azure.ai.evaluations/internal/cmd/run.go | 57 +++- .../internal/pkg/dataset_api/models.go | 10 +- .../internal/pkg/eval_api/evaluators.go | 90 +++++ .../internal/pkg/eval_api/operations.go | 7 + .../internal/project/service_target_eval.go | 12 +- .../tests/live/live_test.go | 4 +- 10 files changed, 876 insertions(+), 54 deletions(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/build_live_test.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/build_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build.go index 77a3ef8918c..7e880001488 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build.go @@ -4,80 +4,318 @@ package cmd import ( + "context" + "fmt" + "sort" + "strings" + "azureaieval/internal/pkg/eval_api" + "azureaieval/internal/pkg/evalcore" "azureaieval/internal/project" ) -// dataMapping is the template binding the service uses to feed each evaluator. -// It pairs with the item schema below, which declares a single `query` field. -func dataMapping() map[string]string { - return map[string]string{ - "query": "{{item.query}}", - "response": "{{sample.output_items}}", - "tool_calls": "{{sample.tool_calls}}", - "tool_definitions": "{{sample.tool_definitions}}", +// evaluatorSchemas indexes the published evaluator contracts by name. +// +// A failure is deliberately not fatal: without schemas the builder falls back +// to the agent-target shape, which is what it always used to send. +func (ec *evalContext) evaluatorSchemas(ctx context.Context) map[string]*eval_api.EvaluatorSummary { + list, err := ec.evalClient.ListEvaluators(ctx, "", ProjectEndpointAPIVersion) + if err != nil { + return nil } + return list.ByName() } -// agentItemSchema mirrors the shape the agent-target runner expects. It is a -// fixed schema, not inferred from the dataset. -func agentItemSchema() map[string]any { - return map[string]any{ - "type": "object", - "properties": map[string]any{ - "query": map[string]any{"type": "string"}, - }, +// sampleBindings are the fields an agent target produces at run time. Anything +// an evaluator accepts that is not in this set has to come from a dataset +// column instead. +var sampleBindings = map[string]string{ + "response": "{{sample.output_items}}", + "tool_calls": "{{sample.tool_calls}}", + "tool_definitions": "{{sample.tool_definitions}}", +} + +// legacyInputs is the mapping used when the service publishes no schema for an +// evaluator, which is the case for freshly uploaded custom evaluators. It +// matches the agent-target shape. +var legacyInputs = []string{"query", "response", "tool_calls", "tool_definitions"} + +// criterionPlan is the resolved binding for one evaluator. +type criterionPlan struct { + dataMapping map[string]string + initParams map[string]any + // itemFields are the fields sourced from dataset columns; they have to be + // declared in the item schema. + itemFields []string +} + +// conversationField carries a whole conversation. The service rejects a +// mapping that pairs it with the turn-level fields: +// +// Evaluator 'builtin.task_completion' has both 'messages' and +// 'query'/'response' in data_mapping. Use 'messages' for conversation-level +// evaluation or 'query'/'response' for turn-level evaluation, but not both. +const conversationField = "messages" + +// turnFields are the per-turn counterparts to conversationField. +var turnFields = []string{"query", "response"} + +// selectLevelFields resolves the conversation/turn exclusivity for evaluators +// that accept both shapes, keeping whichever matches the evaluation level. +// Required fields are never dropped, so a genuine conflict still surfaces as a +// missing-field error rather than being silently reshaped. +func selectLevelFields(accepted, required []string, level string) []string { + isRequired := make(map[string]bool, len(required)) + for _, name := range required { + isRequired[name] = true + } + + acceptsConversation := false + acceptsTurn := false + for _, field := range accepted { + if field == conversationField { + acceptsConversation = true + } + for _, turn := range turnFields { + if field == turn { + acceptsTurn = true + } + } + } + if !acceptsConversation || !acceptsTurn { + return accepted + } + + drop := map[string]bool{} + if strings.EqualFold(level, project.EvaluationLevelConversation) { + for _, turn := range turnFields { + drop[turn] = true + } + } else { + drop[conversationField] = true + } + + kept := make([]string, 0, len(accepted)) + for _, field := range accepted { + if drop[field] && !isRequired[field] { + continue + } + kept = append(kept, field) + } + return kept +} + +// planCriterion shapes one evaluator's bindings from its published contract. +// +// Evaluators do not share an input contract: builtin.similarity needs +// ground_truth, builtin.retrieval needs context, and builtin.ifeval needs +// instruction_id_list. Sending one fixed mapping to all of them earns a +// service-side MissingRequiredDataMapping rejection, so the mapping is derived +// per evaluator and anything unsatisfiable is reported before the request is +// sent. +func planCriterion( + ref evalcore.EvaluatorRef, + schema *eval_api.EvaluatorSummary, + hasTarget bool, + datasetColumns map[string]bool, + evalModel string, + level string, +) (*criterionPlan, error) { + accepted := legacyInputs + var required []string + // A published schema is authoritative even when it is empty: an empty + // property set means the evaluator accepts nothing, which is different from + // publishing no schema at all. + if dataSchema := schema.DataSchema(); dataSchema != nil { + accepted = dataSchema.PropertyNames() + required = dataSchema.Required + } + accepted = selectLevelFields(accepted, required, level) + + plan := &criterionPlan{ + dataMapping: map[string]string{}, + initParams: map[string]any{}, + } + + for _, field := range accepted { + if binding, ok := sampleBindings[field]; ok && hasTarget { + plan.dataMapping[field] = binding + continue + } + // Everything else comes from the dataset. When the columns are known, + // bind only the ones that exist so optional fields stay unbound rather + // than resolving to nothing at run time. + if datasetColumns != nil && !datasetColumns[field] { + continue + } + plan.dataMapping[field] = fmt.Sprintf("{{item.%s}}", field) + plan.itemFields = append(plan.itemFields, field) + } + + var missing []string + for _, field := range required { + if _, ok := plan.dataMapping[field]; !ok { + missing = append(missing, field) + } + } + if len(missing) > 0 { + return nil, fmt.Errorf( + "evaluator %q requires %s, which the dataset does not provide; "+ + "add %s to the dataset, or choose an evaluator that matches the data", + ref.Name, quoteList(missing), pluralColumns(missing), + ) + } + + if !schema.SupportsLevel(level) { + return nil, fmt.Errorf( + "evaluator %q does not support evaluation level %q; it supports %s", + ref.Name, level, quoteList(schema.SupportedEvaluationLevels), + ) + } + + initSchema := schema.InitSchema() + accepts := func(name string) bool { + // Only an absent schema falls back to the historical parameters; + // builtin.ifeval publishes an empty one and takes none. + if initSchema == nil { + return name == "deployment_name" || name == "threshold" + } + return initSchema.Accepts(name) + } + + if evalModel != "" && accepts("deployment_name") { + plan.initParams["deployment_name"] = evalModel + } + if ref.Threshold != nil && accepts("threshold") { + plan.initParams["threshold"] = *ref.Threshold + } + if level != "" && accepts("evaluation_level") { + plan.initParams["evaluation_level"] = level + } + + if initSchema != nil { + var missingInit []string + for _, name := range initSchema.Required { + if _, ok := plan.initParams[name]; !ok { + missingInit = append(missingInit, name) + } + } + if len(missingInit) > 0 { + return nil, fmt.Errorf( + "evaluator %q requires %s; set the judge model on the eval group", + ref.Name, quoteList(missingInit), + ) + } } + + return plan, nil } // buildEvalGroupRequest converts an eval group declaration into the create -// request. Evaluators become testing criteria; a per-evaluator threshold is -// carried in initialization_parameters alongside the judge model. -func buildEvalGroupRequest(group *project.EvalGroup) *eval_api.CreateOpenAIEvalRequest { +// request. Each evaluator becomes a testing criterion bound to its own +// contract, and the item schema declares every dataset column those bindings +// reference. +// +// schemas may be nil or partial; an evaluator with no published contract falls +// back to the agent-target shape. datasetColumns may be nil, meaning the +// columns are unknown and every accepted field is assumed present. +func buildEvalGroupRequest( + group *project.EvalGroup, + schemas map[string]*eval_api.EvaluatorSummary, + datasetColumns map[string]bool, +) (*eval_api.CreateOpenAIEvalRequest, error) { metadata := map[string]string{} - if group.Target != nil && group.Target.Name != "" { + hasTarget := group.Target != nil && group.Target.Name != "" + if hasTarget { metadata["azd_agent"] = group.Target.Name } metadata["azd_eval_group"] = group.Name + evalModel := "" + level := "" + if group.Options != nil { + evalModel = group.Options.EvalModel + level = group.Options.EvaluationLevel + } + req := &eval_api.CreateOpenAIEvalRequest{ Name: group.Name, Metadata: metadata, - DataSourceConfig: &eval_api.DataSourceConfig{ - Type: "custom", - IncludeSampleSchema: true, - ItemSchema: agentItemSchema(), - }, } - evalModel := "" - if group.Options != nil { - evalModel = group.Options.EvalModel - } + itemFields := map[string]bool{} for _, ref := range group.Evaluators { + schema := schemas[ref.Name] + if schema == nil { + schema = &eval_api.EvaluatorSummary{Name: ref.Name} + } + + plan, err := planCriterion(ref, schema, hasTarget, datasetColumns, evalModel, level) + if err != nil { + return nil, err + } + criterion := eval_api.TestingCriterion{ Type: "azure_ai_evaluator", // Name drops the builtin prefix; EvaluatorName keeps it. Name: ref.APIName(), EvaluatorName: ref.Name, - DataMapping: dataMapping(), - } - - params := map[string]any{} - if evalModel != "" { - params["model"] = evalModel - params["deployment_name"] = evalModel + DataMapping: plan.dataMapping, } - if ref.Threshold != nil { - params["threshold"] = *ref.Threshold + if len(plan.initParams) > 0 { + criterion.InitializationParameters = plan.initParams } - if len(params) > 0 { - criterion.InitializationParameters = params + for _, field := range plan.itemFields { + itemFields[field] = true } req.TestingCriteria = append(req.TestingCriteria, criterion) } - return req + req.DataSourceConfig = &eval_api.DataSourceConfig{ + Type: "custom", + IncludeSampleSchema: hasTarget, + ItemSchema: itemSchema(itemFields), + } + + return req, nil +} + +// itemSchema declares the dataset columns the criteria bind to. It always +// declares at least `query`, the column an agent target reads. +func itemSchema(fields map[string]bool) map[string]any { + if len(fields) == 0 { + fields = map[string]bool{"query": true} + } + properties := map[string]any{} + for field := range fields { + properties[field] = map[string]any{"type": "string"} + } + return map[string]any{ + "type": "object", + "properties": properties, + } +} + +func quoteList(values []string) string { + if len(values) == 0 { + return "nothing" + } + quoted := make([]string, 0, len(values)) + for _, value := range values { + quoted = append(quoted, fmt.Sprintf("%q", value)) + } + sort.Strings(quoted) + if len(quoted) == 1 { + return quoted[0] + } + return strings.Join(quoted[:len(quoted)-1], ", ") + " and " + quoted[len(quoted)-1] +} + +func pluralColumns(values []string) string { + if len(values) == 1 { + return "that column" + } + return "those columns" } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build_live_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build_live_test.go new file mode 100644 index 00000000000..0eefc25a34c --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build_live_test.go @@ -0,0 +1,142 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +//go:build live + +// This file proves the request buildEvalGroupRequest produces is accepted by +// the real service. It lives in the cmd package on purpose: the tests under +// tests/live can only hand-roll a request, which validates the API but not the +// code that ships. +// +// go test -tags live -v ./internal/cmd/ -run TestLiveBuild +// +// Required: AZURE_AI_EVAL_E2E_LIVE=1 and FOUNDRY_PROJECT_ENDPOINT. + +package cmd + +import ( + "context" + "fmt" + "os" + "strings" + "testing" + "time" + + "azureaieval/internal/pkg/eval_api" + "azureaieval/internal/pkg/evalcore" + "azureaieval/internal/project" + + "github.com/Azure/azure-sdk-for-go/sdk/azidentity" + "github.com/stretchr/testify/require" +) + +func liveEvalClient(t *testing.T) (*eval_api.EvalClient, string) { + t.Helper() + if os.Getenv("AZURE_AI_EVAL_E2E_LIVE") != "1" { + t.Skip("set AZURE_AI_EVAL_E2E_LIVE=1 to run live tests") + } + endpoint := strings.TrimSuffix(os.Getenv("FOUNDRY_PROJECT_ENDPOINT"), "/") + if endpoint == "" { + t.Fatal("FOUNDRY_PROJECT_ENDPOINT is required") + } + cred, err := azidentity.NewAzureDeveloperCLICredential( + &azidentity.AzureDeveloperCLICredentialOptions{}) + require.NoError(t, err) + + judge := os.Getenv("AZURE_AI_EVAL_MODEL") + if judge == "" { + judge = "gpt-4.1-nano" + } + return eval_api.NewEvalClient(endpoint, cred), judge +} + +// TestLiveBuildAcceptedForEveryBuiltin walks every built-in the project +// exposes, builds a group with the shipping builder, and posts it. +// +// Each evaluator declares a different input contract, so this is the test that +// would have caught the fixed data mapping: it previously produced a +// MissingRequiredDataMapping rejection for builtin.ifeval and would do so +// again for any evaluator whose contract the builder stops honouring. +func TestLiveBuildAcceptedForEveryBuiltin(t *testing.T) { + client, judge := liveEvalClient(t) + ctx := context.Background() + + listed, err := client.ListEvaluators(ctx, eval_api.EvaluatorTypeBuiltin, ProjectEndpointAPIVersion) + require.NoError(t, err) + require.NotEmpty(t, listed.Value) + schemas := listed.ByName() + + for _, summary := range listed.Value { + summary := summary + t.Run(summary.Name, func(t *testing.T) { + // Give the builder a dataset carrying every column the evaluator + // accepts, so a rejection means the request shape is wrong rather + // than the data being genuinely absent. + columns := map[string]bool{"query": true} + if ds := summary.DataSchema(); ds != nil { + for _, name := range ds.PropertyNames() { + columns[name] = true + } + } + + level := "" + if len(summary.SupportedEvaluationLevels) > 0 { + level = summary.SupportedEvaluationLevels[0] + } + + group := &project.EvalGroup{ + Name: fmt.Sprintf("azd-live-%d", time.Now().UTC().UnixNano()), + Dataset: "inline", + Target: &project.Target{Type: "agent", Name: "probe-agent"}, + Evaluators: []evalcore.EvaluatorRef{{Name: summary.Name}}, + Options: &project.Options{EvalModel: judge, EvaluationLevel: level}, + } + + req, err := buildEvalGroupRequest(group, schemas, columns) + require.NoError(t, err, "the builder must satisfy every published contract") + + created, err := client.CreateOpenAIEval(ctx, req) + require.NoError(t, err, + "the service rejected the request this extension builds for %s", summary.Name) + require.NotEmpty(t, created.ID) + t.Cleanup(func() { + _ = client.DeleteOpenAIEval(context.Background(), created.ID) + }) + t.Logf("%s accepted as %s", summary.Name, created.ID) + }) + } +} + +// TestLiveBuildRejectsMissingColumnsLocally proves the pre-flight check fires +// before the network call, so a user sees which column is missing instead of a +// service error naming an internal field path. +func TestLiveBuildRejectsMissingColumnsLocally(t *testing.T) { + client, judge := liveEvalClient(t) + ctx := context.Background() + + listed, err := client.ListEvaluators(ctx, eval_api.EvaluatorTypeBuiltin, ProjectEndpointAPIVersion) + require.NoError(t, err) + schemas := listed.ByName() + + target, ok := schemas["builtin.ifeval"] + if !ok { + t.Skip("builtin.ifeval is not available in this project") + } + require.NotNil(t, target.DataSchema()) + require.NotEmpty(t, target.DataSchema().Required, + "this test relies on ifeval declaring required inputs") + + group := &project.EvalGroup{ + Name: "azd-live-negative", + Dataset: "inline", + Target: &project.Target{Type: "agent", Name: "probe-agent"}, + Evaluators: []evalcore.EvaluatorRef{{Name: "builtin.ifeval"}}, + Options: &project.Options{EvalModel: judge}, + } + + // A dataset with only `query` cannot satisfy ifeval. + _, err = buildEvalGroupRequest(group, schemas, map[string]bool{"query": true}) + require.Error(t, err) + require.Contains(t, err.Error(), "instruction_id_list") + t.Logf("pre-flight error: %v", err) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build_test.go new file mode 100644 index 00000000000..f4b9fa2104e --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build_test.go @@ -0,0 +1,277 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "testing" + + "azureaieval/internal/pkg/eval_api" + "azureaieval/internal/pkg/evalcore" + "azureaieval/internal/project" + + "github.com/stretchr/testify/require" +) + +// schema builds an evaluator contract the way the service publishes one. +func schema(name string, dataRequired, dataProps, initRequired, initProps []string, levels ...string) *eval_api.EvaluatorSummary { + toProps := func(names []string) map[string]any { + if names == nil { + return nil + } + out := map[string]any{} + for _, n := range names { + out[n] = map[string]any{"type": "string"} + } + return out + } + return &eval_api.EvaluatorSummary{ + Name: name, + SupportedEvaluationLevels: levels, + Definition: &eval_api.EvaluatorContract{ + DataSchema: &eval_api.JSONSchema{Required: dataRequired, Properties: toProps(dataProps)}, + InitParameters: &eval_api.JSONSchema{Required: initRequired, Properties: toProps(initProps)}, + }, + } +} + +func groupWith(evaluators []evalcore.EvaluatorRef, opts *project.Options) *project.EvalGroup { + return &project.EvalGroup{ + Name: "g", + Dataset: "d", + Target: &project.Target{Type: "agent", Name: "my-agent"}, + Evaluators: evaluators, + Options: opts, + } +} + +// An agent evaluator takes its response from the sample and its query from the +// dataset. +func TestBuildBindsAgentFieldsFromSample(t *testing.T) { + schemas := map[string]*eval_api.EvaluatorSummary{ + "builtin.task_adherence": schema("builtin.task_adherence", + nil, []string{"query", "response", "tool_definitions", "messages"}, + []string{"deployment_name"}, []string{"deployment_name", "threshold", "evaluation_level"}, + "turn"), + } + group := groupWith( + []evalcore.EvaluatorRef{{Name: "builtin.task_adherence"}}, + &project.Options{EvalModel: "gpt-4.1-nano"}, + ) + + req, err := buildEvalGroupRequest(group, schemas, map[string]bool{"query": true}) + require.NoError(t, err) + require.Len(t, req.TestingCriteria, 1) + + mapping := req.TestingCriteria[0].DataMapping + require.Equal(t, "{{item.query}}", mapping["query"]) + require.Equal(t, "{{sample.output_items}}", mapping["response"]) + require.Equal(t, "{{sample.tool_definitions}}", mapping["tool_definitions"]) + // `messages` is not a dataset column here, so it stays unbound. + require.NotContains(t, mapping, "messages") +} + +// A required field the dataset does not carry is reported before the request +// is sent, naming the field. +func TestBuildRejectsUnsatisfiableEvaluator(t *testing.T) { + schemas := map[string]*eval_api.EvaluatorSummary{ + "builtin.ifeval": schema("builtin.ifeval", + []string{"response", "instruction_id_list", "instruction_kwargs"}, + []string{"response", "instruction_id_list", "instruction_kwargs"}, + nil, nil, "turn"), + } + group := groupWith([]evalcore.EvaluatorRef{{Name: "builtin.ifeval"}}, nil) + + _, err := buildEvalGroupRequest(group, schemas, map[string]bool{"query": true}) + require.Error(t, err) + require.Contains(t, err.Error(), "instruction_id_list") + require.Contains(t, err.Error(), "instruction_kwargs") +} + +// The same evaluator succeeds once the dataset supplies the columns. +func TestBuildAcceptsEvaluatorWhenDatasetSupplies(t *testing.T) { + schemas := map[string]*eval_api.EvaluatorSummary{ + "builtin.ifeval": schema("builtin.ifeval", + []string{"response", "instruction_id_list", "instruction_kwargs"}, + []string{"response", "instruction_id_list", "instruction_kwargs"}, + nil, nil, "turn"), + } + group := groupWith([]evalcore.EvaluatorRef{{Name: "builtin.ifeval"}}, nil) + + req, err := buildEvalGroupRequest(group, schemas, map[string]bool{ + "instruction_id_list": true, + "instruction_kwargs": true, + }) + require.NoError(t, err) + + mapping := req.TestingCriteria[0].DataMapping + // response is satisfied by the agent target. + require.Equal(t, "{{sample.output_items}}", mapping["response"]) + require.Equal(t, "{{item.instruction_id_list}}", mapping["instruction_id_list"]) + + // The item schema has to declare the columns the criteria reference. + props := req.DataSourceConfig.ItemSchema["properties"].(map[string]any) + require.Contains(t, props, "instruction_id_list") + require.Contains(t, props, "instruction_kwargs") +} + +// Initialization parameters are filtered to what the evaluator accepts. +// builtin.ifeval takes none, so nothing is sent even when a model is set. +func TestBuildOmitsUnacceptedInitParameters(t *testing.T) { + threshold := 4.0 + schemas := map[string]*eval_api.EvaluatorSummary{ + "builtin.ifeval": schema("builtin.ifeval", + nil, []string{"response"}, nil, nil, "turn"), + "builtin.similarity": schema("builtin.similarity", + nil, []string{"query", "response", "ground_truth"}, + []string{"deployment_name"}, []string{"deployment_name", "threshold"}, "turn"), + } + group := groupWith([]evalcore.EvaluatorRef{ + {Name: "builtin.ifeval", Threshold: &threshold}, + {Name: "builtin.similarity", Threshold: &threshold}, + }, &project.Options{EvalModel: "gpt-4.1-nano"}) + + req, err := buildEvalGroupRequest(group, schemas, map[string]bool{ + "query": true, "ground_truth": true, + }) + require.NoError(t, err) + + // ifeval accepts no init parameters at all. + require.Empty(t, req.TestingCriteria[0].InitializationParameters) + + // similarity accepts both, and never the `model` alias. + params := req.TestingCriteria[1].InitializationParameters + require.Equal(t, "gpt-4.1-nano", params["deployment_name"]) + require.InDelta(t, 4.0, params["threshold"], 0.0001) + require.NotContains(t, params, "model") +} + +// evaluation_level is an initialization parameter, not run metadata, and only +// on evaluators that declare it. +func TestBuildPassesEvaluationLevelAsInitParameter(t *testing.T) { + schemas := map[string]*eval_api.EvaluatorSummary{ + "builtin.task_completion": schema("builtin.task_completion", + nil, []string{"query", "response"}, + []string{"deployment_name"}, []string{"deployment_name", "evaluation_level"}, + "conversation", "turn"), + "builtin.similarity": schema("builtin.similarity", + nil, []string{"query", "response"}, + []string{"deployment_name"}, []string{"deployment_name", "threshold"}, "turn"), + } + group := groupWith([]evalcore.EvaluatorRef{ + {Name: "builtin.task_completion"}, + {Name: "builtin.similarity"}, + }, &project.Options{EvalModel: "m", EvaluationLevel: "turn"}) + + req, err := buildEvalGroupRequest(group, schemas, map[string]bool{"query": true}) + require.NoError(t, err) + + require.Equal(t, "turn", req.TestingCriteria[0].InitializationParameters["evaluation_level"]) + require.NotContains(t, req.TestingCriteria[1].InitializationParameters, "evaluation_level") +} + +// An evaluator that does not support the requested level is rejected with the +// levels it does support. +func TestBuildRejectsUnsupportedLevel(t *testing.T) { + schemas := map[string]*eval_api.EvaluatorSummary{ + "builtin.similarity": schema("builtin.similarity", + nil, []string{"query", "response"}, + []string{"deployment_name"}, []string{"deployment_name"}, "turn"), + } + group := groupWith([]evalcore.EvaluatorRef{{Name: "builtin.similarity"}}, + &project.Options{EvalModel: "m", EvaluationLevel: "conversation"}) + + _, err := buildEvalGroupRequest(group, schemas, map[string]bool{"query": true}) + require.Error(t, err) + require.Contains(t, err.Error(), "conversation") + require.Contains(t, err.Error(), "turn") +} + +// A required init parameter with no judge model configured is caught locally. +func TestBuildRequiresJudgeModelWhenEvaluatorDoes(t *testing.T) { + schemas := map[string]*eval_api.EvaluatorSummary{ + "builtin.similarity": schema("builtin.similarity", + nil, []string{"query", "response"}, + []string{"deployment_name"}, []string{"deployment_name"}, "turn"), + } + group := groupWith([]evalcore.EvaluatorRef{{Name: "builtin.similarity"}}, nil) + + _, err := buildEvalGroupRequest(group, schemas, map[string]bool{"query": true}) + require.Error(t, err) + require.Contains(t, err.Error(), "deployment_name") +} + +// An evaluator with no published contract keeps the historical agent-target +// shape, so custom evaluators still deploy. +func TestBuildFallsBackWithoutSchema(t *testing.T) { + group := groupWith([]evalcore.EvaluatorRef{{Name: "my-custom-evaluator"}}, + &project.Options{EvalModel: "m"}) + + req, err := buildEvalGroupRequest(group, nil, nil) + require.NoError(t, err) + + mapping := req.TestingCriteria[0].DataMapping + require.Equal(t, "{{item.query}}", mapping["query"]) + require.Equal(t, "{{sample.output_items}}", mapping["response"]) + require.Equal(t, "{{sample.tool_calls}}", mapping["tool_calls"]) + require.Equal(t, "{{sample.tool_definitions}}", mapping["tool_definitions"]) + require.Equal(t, "m", req.TestingCriteria[0].InitializationParameters["deployment_name"]) +} + +// `messages` and `query`/`response` are mutually exclusive; the evaluation +// level picks which shape is bound. Sending both is rejected by the service. +func TestBuildResolvesConversationTurnExclusivity(t *testing.T) { + schemas := map[string]*eval_api.EvaluatorSummary{ + "builtin.task_completion": schema("builtin.task_completion", + nil, []string{"query", "response", "messages", "tool_definitions"}, + []string{"deployment_name"}, []string{"deployment_name", "evaluation_level"}, + "conversation", "turn"), + } + columns := map[string]bool{"query": true, "messages": true, "response": true} + + // Turn level keeps query/response and drops messages. + turn := groupWith([]evalcore.EvaluatorRef{{Name: "builtin.task_completion"}}, + &project.Options{EvalModel: "m", EvaluationLevel: "turn"}) + req, err := buildEvalGroupRequest(turn, schemas, columns) + require.NoError(t, err) + mapping := req.TestingCriteria[0].DataMapping + require.Contains(t, mapping, "query") + require.NotContains(t, mapping, "messages") + + // Conversation level keeps messages and drops query/response. + conv := groupWith([]evalcore.EvaluatorRef{{Name: "builtin.task_completion"}}, + &project.Options{EvalModel: "m", EvaluationLevel: "conversation"}) + req, err = buildEvalGroupRequest(conv, schemas, columns) + require.NoError(t, err) + mapping = req.TestingCriteria[0].DataMapping + require.Contains(t, mapping, "messages") + require.NotContains(t, mapping, "query") + require.NotContains(t, mapping, "response") + + // An unset level behaves as turn, matching the service default. + dflt := groupWith([]evalcore.EvaluatorRef{{Name: "builtin.task_completion"}}, + &project.Options{EvalModel: "m"}) + req, err = buildEvalGroupRequest(dflt, schemas, columns) + require.NoError(t, err) + require.NotContains(t, req.TestingCriteria[0].DataMapping, "messages") +} + +// Without an agent target the sample bindings are unavailable, so every field +// has to come from the dataset and the sample schema is not requested. +func TestBuildWithoutTargetSourcesEverythingFromDataset(t *testing.T) { + schemas := map[string]*eval_api.EvaluatorSummary{ + "builtin.similarity": schema("builtin.similarity", + []string{"query", "response", "ground_truth"}, + []string{"query", "response", "ground_truth"}, + nil, nil, "turn"), + } + group := groupWith([]evalcore.EvaluatorRef{{Name: "builtin.similarity"}}, nil) + group.Target = nil + + req, err := buildEvalGroupRequest(group, schemas, map[string]bool{ + "query": true, "response": true, "ground_truth": true, + }) + require.NoError(t, err) + require.False(t, req.DataSourceConfig.IncludeSampleSchema) + require.Equal(t, "{{item.response}}", req.TestingCriteria[0].DataMapping["response"]) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go index e2f433fe84f..cc3fd2f9a33 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go @@ -152,6 +152,7 @@ func (r *evalReconciler) EnsureEvaluator( func (r *evalReconciler) EnsureEvalGroup( ctx context.Context, group project.EvalGroup, + datasetPath string, recreate bool, ) (string, error) { if group.ID != "" { @@ -165,7 +166,15 @@ func (r *evalReconciler) EnsureEvalGroup( } } - created, err := r.ec.evalClient.CreateOpenAIEval(ctx, buildEvalGroupRequest(&group)) + req, err := buildEvalGroupRequest( + &group, + r.ec.evaluatorSchemas(ctx), + datasetColumnsFromPath(datasetPath), + ) + if err != nil { + return "", err + } + created, err := r.ec.evalClient.CreateOpenAIEval(ctx, req) if err != nil { return "", err } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go index 7085d5b7ad7..abae99924ac 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go @@ -68,7 +68,8 @@ func newRunCommand() *cobra.Command { return err } - evalID, err = ec.resolveEvalGroupID(ctx, group, out, isJSON(cmd)) + evalID, err = ec.resolveEvalGroupID( + ctx, group, configPath, resolveLevel(level, group), out, isJSON(cmd)) if err != nil { return err } @@ -148,6 +149,8 @@ func newRunCommand() *cobra.Command { func (ec *evalContext) resolveEvalGroupID( ctx context.Context, group *project.EvalGroup, + configPath string, + level string, out interface{ Write([]byte) (int, error) }, jsonMode bool, ) (string, error) { @@ -165,7 +168,28 @@ func (ec *evalContext) resolveEvalGroupID( if !jsonMode { fmt.Fprintf(out, "Creating eval group %q...\n", group.Name) } - created, err := ec.evalClient.CreateOpenAIEval(ctx, buildEvalGroupRequest(group)) + + // The level from the flag wins over the group's own options, so it has to + // reach the criteria that accept evaluation_level. + effective := *group + if level != "" { + opts := project.Options{} + if group.Options != nil { + opts = *group.Options + } + opts.EvaluationLevel = level + effective.Options = &opts + } + + req, err := buildEvalGroupRequest( + &effective, + ec.evaluatorSchemas(ctx), + datasetColumns(configPath, group), + ) + if err != nil { + return "", err + } + created, err := ec.evalClient.CreateOpenAIEval(ctx, req) if err != nil { return "", fmt.Errorf("creating eval group %q: %w", group.Name, err) } @@ -211,6 +235,35 @@ func buildRunDataSource( return ds, nil } +// datasetColumns reports the columns a group's dataset provides, so criteria +// bind only to fields that exist and a missing required field is caught +// locally rather than as a service rejection. +// +// A nil result means the columns are unknown, which is the case for a dataset +// already registered in the project. The builder then assumes every field an +// evaluator accepts is present. +func datasetColumns(configPath string, group *project.EvalGroup) map[string]bool { + return datasetColumnsFromPath(localDatasetPath(configPath, group)) +} + +// datasetColumnsFromPath reads one row to learn the dataset's shape. An empty +// path, or an unreadable file, yields nil. +func datasetColumnsFromPath(localPath string) map[string]bool { + if localPath == "" { + return nil + } + // One row is enough to learn the shape. + items, err := readJSONL(localPath, 1) + if err != nil || len(items) == 0 { + return nil + } + columns := make(map[string]bool, len(items[0])) + for name := range items[0] { + columns[name] = true + } + return columns +} + // localDatasetPath resolves the dataset's local source relative to the config // file, returning empty when the dataset is registered rather than local. func localDatasetPath(configPath string, group *project.EvalGroup) string { diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/models.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/models.go index 7e6c8633c09..13fe6da2d9d 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/models.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/models.go @@ -35,11 +35,11 @@ type Dataset struct { Format string `json:"format,omitempty"` // camelCase spellings (project endpoint). - DataURICamel string `json:"dataUri,omitempty"` - BlobURICamel string `json:"blobUri,omitempty"` - ContentURICamel string `json:"contentUri,omitempty"` - IsSingleFile bool `json:"isSingleFile,omitempty"` - ConnectionName string `json:"connectionName,omitempty"` + DataURICamel string `json:"dataUri,omitempty"` + BlobURICamel string `json:"blobUri,omitempty"` + ContentURICamel string `json:"contentUri,omitempty"` + IsSingleFile bool `json:"isSingleFile,omitempty"` + ConnectionName string `json:"connectionName,omitempty"` // snake_case spellings. BlobURI string `json:"blob_uri,omitempty"` diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/evaluators.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/evaluators.go index 771f894b03d..dfa0abf60ca 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/evaluators.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/evaluators.go @@ -8,17 +8,95 @@ import ( "fmt" "net/http" "net/url" + "sort" + "strings" ) // EvaluatorTypeBuiltin selects the platform-provided evaluators. const EvaluatorTypeBuiltin = "Builtin" +// JSONSchema is the subset of JSON Schema the evaluator contract uses. +type JSONSchema struct { + Type string `json:"type,omitempty"` + Required []string `json:"required,omitempty"` + Properties map[string]any `json:"properties,omitempty"` +} + +// PropertyNames returns the accepted property names, sorted for stable output. +func (s *JSONSchema) PropertyNames() []string { + if s == nil { + return nil + } + names := make([]string, 0, len(s.Properties)) + for name := range s.Properties { + names = append(names, name) + } + sort.Strings(names) + return names +} + +// Accepts reports whether the schema declares the named property. +func (s *JSONSchema) Accepts(name string) bool { + if s == nil || s.Properties == nil { + return false + } + _, ok := s.Properties[name] + return ok +} + +// EvaluatorContract is the published input contract for an evaluator: which +// data fields it consumes and which initialization parameters it takes. +type EvaluatorContract struct { + Type string `json:"type,omitempty"` + DataSchema *JSONSchema `json:"data_schema,omitempty"` + InitParameters *JSONSchema `json:"init_parameters,omitempty"` +} + // EvaluatorSummary is a single entry in an evaluator listing. +// +// The listing carries the full contract, so callers can shape a request to +// match an evaluator instead of guessing and taking a service-side rejection. type EvaluatorSummary struct { Name string `json:"name"` Version string `json:"version,omitempty"` Type string `json:"type,omitempty"` Description string `json:"description,omitempty"` + + Categories []string `json:"categories,omitempty"` + SupportedEvaluationLevels []string `json:"supported_evaluation_levels,omitempty"` + Definition *EvaluatorContract `json:"definition,omitempty"` +} + +// SupportsLevel reports whether the evaluator runs at the given evaluation +// level. An evaluator that declares no levels is treated as unconstrained. +func (e *EvaluatorSummary) SupportsLevel(level string) bool { + if level == "" || len(e.SupportedEvaluationLevels) == 0 { + return true + } + for _, supported := range e.SupportedEvaluationLevels { + if strings.EqualFold(supported, level) { + return true + } + } + return false +} + +// DataSchema returns the evaluator's input schema, or nil when the listing +// did not describe one. +func (e *EvaluatorSummary) DataSchema() *JSONSchema { + if e == nil || e.Definition == nil { + return nil + } + return e.Definition.DataSchema +} + +// InitSchema returns the evaluator's initialization-parameter schema, or nil +// when the listing did not describe one. +func (e *EvaluatorSummary) InitSchema() *JSONSchema { + if e == nil || e.Definition == nil { + return nil + } + return e.Definition.InitParameters } // EvaluatorListResponse is the paged response for an evaluator listing. @@ -27,6 +105,18 @@ type EvaluatorListResponse struct { NextLink string `json:"nextLink,omitempty"` } +// ByName indexes the listing by evaluator name. +func (r *EvaluatorListResponse) ByName() map[string]*EvaluatorSummary { + if r == nil { + return nil + } + index := make(map[string]*EvaluatorSummary, len(r.Value)) + for i := range r.Value { + index[r.Value[i].Name] = &r.Value[i] + } + return index +} + // ListEvaluators returns the evaluators visible to the project. Pass // EvaluatorTypeBuiltin to list only the platform's built-ins. func (c *EvalClient) ListEvaluators( diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/operations.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/operations.go index 510b2680b31..795e8301e0c 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/operations.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/operations.go @@ -165,6 +165,13 @@ func (c *EvalClient) GetOpenAIEval(ctx context.Context, evalID string) (*OpenAIE return doRequestTyped[OpenAIEval](c, ctx, http.MethodGet, path, nil, nil, "") } +// DeleteOpenAIEval removes an eval definition and its runs. +func (c *EvalClient) DeleteOpenAIEval(ctx context.Context, evalID string) error { + path := pathOpenAIEvals + "/" + url.PathEscape(evalID) + _, err := c.doRequest(ctx, http.MethodDelete, path, nil, nil, "") + return err +} + // CreateOpenAIEvalRun starts a run for an OpenAI eval definition. func (c *EvalClient) CreateOpenAIEvalRun( ctx context.Context, diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go index ce8a1439b5d..734e163195b 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go @@ -37,8 +37,10 @@ type Reconciler interface { // differs from what the service already holds. EnsureEvaluator(ctx context.Context, decl EvaluatorDecl, localPath string) (version string, changed bool, err error) // EnsureEvalGroup creates the group when it is absent or its resolved - // evaluators or options changed, returning its id. - EnsureEvalGroup(ctx context.Context, group EvalGroup, recreate bool) (id string, err error) + // evaluators or options changed, returning its id. datasetPath is the local + // dataset backing the group, or empty when it is already registered; it lets + // the reconciler bind criteria to the columns that actually exist. + EnsureEvalGroup(ctx context.Context, group EvalGroup, datasetPath string, recreate bool) (id string, err error) } // EvalServiceTargetProvider deploys eval resources during `azd up`. azd owns @@ -171,7 +173,11 @@ func (p *EvalServiceTargetProvider) Deploy( // group must be created and the stored id replaced. for _, group := range cfg.EvalGroups { report(progress, fmt.Sprintf("Reconciling eval group %s", group.Name)) - id, err := reconciler.EnsureEvalGroup(ctx, group, anyChanged) + datasetPath := "" + if decl, ok := cfg.Dataset(group.Dataset); ok { + datasetPath = resolveSource(baseDir, decl.Source) + } + id, err := reconciler.EnsureEvalGroup(ctx, group, datasetPath, anyChanged) if err != nil { return nil, fmt.Errorf("eval group %q: %w", group.Name, err) } diff --git a/cli/azd/extensions/azure.ai.evaluations/tests/live/live_test.go b/cli/azd/extensions/azure.ai.evaluations/tests/live/live_test.go index c04af2d47a6..9174ff294f4 100644 --- a/cli/azd/extensions/azure.ai.evaluations/tests/live/live_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/tests/live/live_test.go @@ -195,7 +195,7 @@ func TestLiveEvalGroupLifecycle(t *testing.T) { ctx, eval_api.EvaluatorTypeBuiltin, projectAPIVersion) require.NoError(t, err) require.NotEmpty(t, builtins.Value, "need at least one built-in evaluator") -evaluatorName := pickQualityEvaluator(t, builtins.Value) + evaluatorName := pickQualityEvaluator(t, builtins.Value) threshold := 3.0 req := &eval_api.CreateOpenAIEvalRequest{ @@ -249,7 +249,7 @@ func TestLiveRun(t *testing.T) { ctx, eval_api.EvaluatorTypeBuiltin, projectAPIVersion) require.NoError(t, err) require.NotEmpty(t, builtins.Value) -evaluatorName := pickQualityEvaluator(t, builtins.Value) + evaluatorName := pickQualityEvaluator(t, builtins.Value) group, err := env.evalClient.CreateOpenAIEval(ctx, &eval_api.CreateOpenAIEvalRequest{ Name: uniqueName("azd-eval-e2e-run"), From 4d265585679572afffcf97bb3483f8983482c0ba Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 28 Jul 2026 05:50:23 +0300 Subject: [PATCH 012/320] build(azure.ai.evaluations): add the build scripts the extension kit invokes Without build.ps1 and build.sh the dev kit reported a successful build in under a second and produced no binary, so azd x pack had nothing to package and azd x publish failed with 'Artifacts not found'. Copied from azure.ai.agents with the version package path retargeted, plus its golangci config. Verified end to end: build, pack, publish, install from the local registry, and 'azd ai eval --help' listing every command. --- .../azure.ai.evaluations/.golangci.yaml | 21 +++++ .../extensions/azure.ai.evaluations/build.ps1 | 78 +++++++++++++++++++ .../extensions/azure.ai.evaluations/build.sh | 66 ++++++++++++++++ 3 files changed, 165 insertions(+) create mode 100644 cli/azd/extensions/azure.ai.evaluations/.golangci.yaml create mode 100644 cli/azd/extensions/azure.ai.evaluations/build.ps1 create mode 100644 cli/azd/extensions/azure.ai.evaluations/build.sh diff --git a/cli/azd/extensions/azure.ai.evaluations/.golangci.yaml b/cli/azd/extensions/azure.ai.evaluations/.golangci.yaml new file mode 100644 index 00000000000..9777522d023 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/.golangci.yaml @@ -0,0 +1,21 @@ +version: "2" + +linters: + default: none + enable: + - gosec + - lll + - unused + - errorlint + settings: + lll: + line-length: 220 + tab-width: 4 + gosec: + excludes: + - G204 # Subprocess launched with variable (bicep build invoked in tests) + - G304 # Potential file inclusion via variable + +formatters: + enable: + - gofmt diff --git a/cli/azd/extensions/azure.ai.evaluations/build.ps1 b/cli/azd/extensions/azure.ai.evaluations/build.ps1 new file mode 100644 index 00000000000..f37f80cabf0 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/build.ps1 @@ -0,0 +1,78 @@ +# Ensure script fails on any error +$ErrorActionPreference = 'Stop' + +# Get the directory of the script +$EXTENSION_DIR = Split-Path -Parent $MyInvocation.MyCommand.Path + +# Change to the script directory +Set-Location -Path $EXTENSION_DIR + +# Create a safe version of EXTENSION_ID replacing dots with dashes +$EXTENSION_ID_SAFE = $env:EXTENSION_ID -replace '\.', '-' + +# Define output directory +$OUTPUT_DIR = if ($env:OUTPUT_DIR) { $env:OUTPUT_DIR } else { Join-Path $EXTENSION_DIR "bin" } + +# Create output directory if it doesn't exist +if (-not (Test-Path -Path $OUTPUT_DIR)) { + New-Item -ItemType Directory -Path $OUTPUT_DIR | Out-Null +} + +# Get Git commit hash and build date +$COMMIT = git rev-parse HEAD +if ($LASTEXITCODE -ne 0) { + Write-Host "Error: Failed to get git commit hash" + exit 1 +} +$BUILD_DATE = (Get-Date -Format "yyyy-MM-ddTHH:mm:ssZ") + +# List of OS and architecture combinations +if ($env:EXTENSION_PLATFORM) { + $PLATFORMS = @($env:EXTENSION_PLATFORM) +} +else { + $PLATFORMS = @( + "windows/amd64", + "windows/arm64", + "darwin/amd64", + "darwin/arm64", + "linux/amd64", + "linux/arm64" + ) +} + +$VERSION_PATH = "azureaieval/internal/version" + +# Loop through platforms and build +foreach ($PLATFORM in $PLATFORMS) { + $OS, $ARCH = $PLATFORM -split '/' + + $OUTPUT_NAME = Join-Path $OUTPUT_DIR "$EXTENSION_ID_SAFE-$OS-$ARCH" + + if ($OS -eq "windows") { + $OUTPUT_NAME += ".exe" + } + + Write-Host "Building for $OS/$ARCH..." + + # Delete the output file if it already exists + if (Test-Path -Path $OUTPUT_NAME) { + Remove-Item -Path $OUTPUT_NAME -Force + } + + # Set environment variables for Go build + $env:GOOS = $OS + $env:GOARCH = $ARCH + + go build ` + -ldflags="-X '$VERSION_PATH.Version=$env:EXTENSION_VERSION' -X '$VERSION_PATH.Commit=$COMMIT' -X '$VERSION_PATH.BuildDate=$BUILD_DATE'" ` + -o $OUTPUT_NAME + + if ($LASTEXITCODE -ne 0) { + Write-Host "An error occurred while building for $OS/$ARCH" + exit 1 + } +} + +Write-Host "Build completed successfully!" +Write-Host "Binaries are located in the $OUTPUT_DIR directory." diff --git a/cli/azd/extensions/azure.ai.evaluations/build.sh b/cli/azd/extensions/azure.ai.evaluations/build.sh new file mode 100644 index 00000000000..4165a516ac4 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/build.sh @@ -0,0 +1,66 @@ +#!/bin/bash + +# Get the directory of the script +EXTENSION_DIR="$(cd "$(dirname "$0")" && pwd)" + +# Change to the script directory +cd "$EXTENSION_DIR" || exit + +# Create a safe version of EXTENSION_ID replacing dots with dashes +EXTENSION_ID_SAFE="${EXTENSION_ID//./-}" + +# Define output directory +OUTPUT_DIR="${OUTPUT_DIR:-$EXTENSION_DIR/bin}" + +# Create output and target directories if they don't exist +mkdir -p "$OUTPUT_DIR" + +# Get Git commit hash and build date +COMMIT=$(git rev-parse HEAD) +BUILD_DATE=$(date -u +%Y-%m-%dT%H:%M:%SZ) + +# List of OS and architecture combinations +if [ -n "$EXTENSION_PLATFORM" ]; then + PLATFORMS=("$EXTENSION_PLATFORM") +else + PLATFORMS=( + "windows/amd64" + "windows/arm64" + "darwin/amd64" + "darwin/arm64" + "linux/amd64" + "linux/arm64" + ) +fi + +VERSION_PATH="azureaieval/internal/version" + +# Loop through platforms and build +for PLATFORM in "${PLATFORMS[@]}"; do + OS=$(echo "$PLATFORM" | cut -d'/' -f1) + ARCH=$(echo "$PLATFORM" | cut -d'/' -f2) + + OUTPUT_NAME="$OUTPUT_DIR/$EXTENSION_ID_SAFE-$OS-$ARCH" + + if [ "$OS" = "windows" ]; then + OUTPUT_NAME+='.exe' + fi + + echo "Building for $OS/$ARCH..." + + # Delete the output file if it already exists + [ -f "$OUTPUT_NAME" ] && rm -f "$OUTPUT_NAME" + + # Set environment variables for Go build + GOOS=$OS GOARCH=$ARCH go build \ + -ldflags="-X '$VERSION_PATH.Version=$EXTENSION_VERSION' -X '$VERSION_PATH.Commit=$COMMIT' -X '$VERSION_PATH.BuildDate=$BUILD_DATE'" \ + -o "$OUTPUT_NAME" + + if [ $? -ne 0 ]; then + echo "An error occurred while building for $OS/$ARCH" + exit 1 + fi +done + +echo "Build completed successfully!" +echo "Binaries are located in the $OUTPUT_DIR directory." From f92e69be38391477314a7e72bcc38deaadbba455 Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 28 Jul 2026 05:52:36 +0300 Subject: [PATCH 013/320] fix(azure.ai.evaluations): read the evaluator kind the listing actually returns The listing spells it evaluator_type, so the TYPE column in 'evaluator list' and 'evaluator builtins' was always blank. Both spellings are now accepted. --- .../azure.ai.evaluations/internal/cmd/evaluator.go | 2 +- .../internal/pkg/eval_api/evaluators.go | 14 +++++++++++++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator.go index e374079dd9e..9bbc7e41b1a 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator.go @@ -203,7 +203,7 @@ func renderEvaluators(cmd *cobra.Command, list *eval_api.EvaluatorListResponse) } rows := make([][]string, 0, len(list.Value)) for _, e := range list.Value { - rows = append(rows, []string{e.Name, e.Version, e.Type}) + rows = append(rows, []string{e.Name, e.Version, e.Type()}) } return emitTable(cmd.OutOrStdout(), []string{"NAME", "VERSION", "TYPE"}, rows) } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/evaluators.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/evaluators.go index dfa0abf60ca..e03a7059efd 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/evaluators.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/evaluators.go @@ -59,14 +59,26 @@ type EvaluatorContract struct { type EvaluatorSummary struct { Name string `json:"name"` Version string `json:"version,omitempty"` - Type string `json:"type,omitempty"` Description string `json:"description,omitempty"` + // The listing spells this evaluator_type; `type` is accepted too because + // other evaluator payloads use it. + EvaluatorType string `json:"evaluator_type,omitempty"` + TypeAlias string `json:"type,omitempty"` + Categories []string `json:"categories,omitempty"` SupportedEvaluationLevels []string `json:"supported_evaluation_levels,omitempty"` Definition *EvaluatorContract `json:"definition,omitempty"` } +// Type reports the evaluator kind across both spellings. +func (e *EvaluatorSummary) Type() string { + if e.EvaluatorType != "" { + return e.EvaluatorType + } + return e.TypeAlias +} + // SupportsLevel reports whether the evaluator runs at the given evaluation // level. An evaluator that declares no levels is treated as unconstrained. func (e *EvaluatorSummary) SupportsLevel(level string) bool { From 5a670ca81162602aaf87022e5a624f17c9aef518 Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 28 Jul 2026 06:02:33 +0300 Subject: [PATCH 014/320] fix(azure.ai.evaluations): make azd deploy work end to end Running a real 'azd deploy' against the service-target provider surfaced two failures that no unit test covered. Evaluator references only decoded from YAML. azd hands the service entry to the extension as JSON, so a group written as '- builtin.task_adherence' -- the form the CLI's own init command writes -- failed with 'cannot unmarshal string into EvaluatorRef'. EvaluatorList now decodes and encodes the mixed string-or-mapping form through JSON as well, and a test asserts the two decoders agree. The dataset reconciler passed the declared version straight to UploadNewVersion, which derives the next version from it. A declaration without an explicit version passed empty, so every deploy retried 1.0 and the service returned 409 TemporaryDataReferencesForExistingAsset once that version existed. It now looks up the latest registered version first. Verified against a live project: first deploy publishes the dataset at 1.0 and creates the group; an unchanged redeploy reports 'unchanged at version 1.0' and uploads nothing; and editing the dataset publishes 2.0 and recreates the group, since groups are immutable. --- .../internal/cmd/reconciler.go | 21 ++++++- .../internal/pkg/evalcore/evaluator.go | 57 +++++++++++++++++ .../internal/pkg/evalcore/evaluator_test.go | 62 +++++++++++++++++++ 3 files changed, 139 insertions(+), 1 deletion(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/evaluator_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go index cc3fd2f9a33..92a55f808f4 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go @@ -84,8 +84,17 @@ func (r *evalReconciler) EnsureDataset( dir = filepath.Dir(localPath) } + // UploadNewVersion derives the next version from the one passed in, so it + // has to be told what is already registered. Passing an empty version + // restarts at 1.0 and the service rejects the pending upload with + // TemporaryDataReferencesForExistingAsset once that version exists. + currentVersion := decl.Version + if currentVersion == "" { + currentVersion = r.latestDatasetVersion(ctx, decl.Name) + } + ds, err := r.ec.datasetClient.UploadNewVersion( - ctx, decl.Name, decl.Version, dir, ProjectEndpointAPIVersion, + ctx, decl.Name, currentVersion, dir, ProjectEndpointAPIVersion, ) if err != nil { return "", false, err @@ -98,6 +107,16 @@ func (r *evalReconciler) EnsureDataset( return ds.Version, true, nil } +// latestDatasetVersion reports the newest registered version, or empty when the +// dataset does not exist yet so the first upload starts at 1.0. +func (r *evalReconciler) latestDatasetVersion(ctx context.Context, name string) string { + list, err := r.ec.datasetClient.ListDatasetVersions(ctx, name, ProjectEndpointAPIVersion) + if err != nil || list == nil || len(list.Value) == 0 { + return "" + } + return dataset_api.LatestVersion(list.Value) +} + // EnsureEvaluator publishes a new version when the local definition differs // from what the service holds. Evaluator definitions come back inline, so this // compares content directly rather than relying on a cached digest. diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/evaluator.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/evaluator.go index 0a4847ad4bd..4832d52c42d 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/evaluator.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/evaluator.go @@ -4,6 +4,8 @@ package evalcore import ( + "bytes" + "encoding/json" "fmt" "strings" @@ -87,3 +89,58 @@ func (el EvaluatorList) MarshalYAML() (any, error) { } return out, nil } + +// UnmarshalJSON accepts the same mixed string-or-mapping form as the YAML +// decoder. +// +// This matters for the service-target provider: azd hands the service entry to +// the extension as JSON, so a config written as `- builtin.task_adherence` +// arrives as a bare string and would otherwise fail to decode. +func (el *EvaluatorList) UnmarshalJSON(data []byte) error { + var entries []json.RawMessage + if err := json.Unmarshal(data, &entries); err != nil { + return fmt.Errorf("evaluators must be a list: %w", err) + } + + result := make([]EvaluatorRef, 0, len(entries)) + for _, entry := range entries { + trimmed := bytes.TrimSpace(entry) + if len(trimmed) > 0 && trimmed[0] == '"' { + var name string + if err := json.Unmarshal(trimmed, &name); err != nil { + return fmt.Errorf("decoding evaluator name: %w", err) + } + result = append(result, EvaluatorRef{Name: name}) + continue + } + + var ref EvaluatorRef + if err := json.Unmarshal(trimmed, &ref); err != nil { + return fmt.Errorf("decoding evaluator: %w", err) + } + if ref.Name == "" { + return fmt.Errorf("evaluator entry is missing 'name'") + } + result = append(result, ref) + } + + *el = result + return nil +} + +// MarshalJSON mirrors MarshalYAML's compact form. +func (el EvaluatorList) MarshalJSON() ([]byte, error) { + out := make([]any, 0, len(el)) + for _, ref := range el { + if ref.Threshold == nil && ref.Version == "" { + out = append(out, ref.Name) + continue + } + out = append(out, struct { + Name string `json:"name"` + Version string `json:"version,omitempty"` + Threshold *float64 `json:"threshold,omitempty"` + }{ref.Name, ref.Version, ref.Threshold}) + } + return json.Marshal(out) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/evaluator_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/evaluator_test.go new file mode 100644 index 00000000000..5408776bec7 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/evaluator_test.go @@ -0,0 +1,62 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package evalcore + +import ( + "encoding/json" + "testing" + + "github.com/stretchr/testify/require" + "go.yaml.in/yaml/v3" +) + +// The service-target provider receives the config as JSON, not YAML, so the +// mixed string-or-mapping form has to decode through both paths. Supporting +// only YAML made `azd deploy` fail on a config the CLI itself writes. +func TestEvaluatorListDecodesMixedEntriesFromJSON(t *testing.T) { + const payload = `[ + "builtin.task_adherence", + {"name": "support-quality", "threshold": 4.0}, + {"name": "pinned", "version": "3"} + ]` + + var list EvaluatorList + require.NoError(t, json.Unmarshal([]byte(payload), &list)) + require.Len(t, list, 3) + + require.Equal(t, "builtin.task_adherence", list[0].Name) + require.Nil(t, list[0].Threshold) + + require.Equal(t, "support-quality", list[1].Name) + require.NotNil(t, list[1].Threshold) + require.InDelta(t, 4.0, *list[1].Threshold, 0.0001) + + require.Equal(t, "pinned", list[2].Name) + require.Equal(t, "3", list[2].Version) +} + +// The JSON and YAML decoders must agree, otherwise a config behaves one way +// through the CLI and another through `azd up`. +func TestEvaluatorListJSONMatchesYAML(t *testing.T) { + const doc = ` +- builtin.task_adherence +- { name: support-quality, threshold: 4.0 } +` + var fromYAML EvaluatorList + require.NoError(t, yaml.Unmarshal([]byte(doc), &fromYAML)) + + encoded, err := json.Marshal(fromYAML) + require.NoError(t, err) + + var fromJSON EvaluatorList + require.NoError(t, json.Unmarshal(encoded, &fromJSON)) + require.Equal(t, fromYAML, fromJSON) +} + +func TestEvaluatorListRejectsEntryWithoutName(t *testing.T) { + var list EvaluatorList + err := json.Unmarshal([]byte(`[{"threshold": 4.0}]`), &list) + require.Error(t, err) + require.Contains(t, err.Error(), "name") +} From d21bbd3bce67735104d6c7f25544eee014957153 Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 28 Jul 2026 06:07:59 +0300 Subject: [PATCH 015/320] fix(azure.ai.evaluations): resolve $ref includes in the service entry azd core does not resolve $ref for extensions. It strips the ServiceConfig fields it owns and leaves $ref at the top of the map for the owning extension to resolve, so a service authored the way the spec documents it -- host: azure.ai.eval plus $ref: ./evals/azure.yaml -- parsed to an empty config. azd deploy then reported success in three seconds having created nothing, which is worse than failing. The provider now calls foundry.ResolveFileRefs with the project root from the azd project client. Relative source paths inside an included file are written against that file, but ResolveFileRefs inlines content without rebasing them, so the include's own directory is now the base for source resolution. Verified against a live project: the $ref form deploys, and the dataset fingerprint matches the one from the equivalent inline config, confirming both forms resolve to the same file. --- .../internal/project/service_target_eval.go | 48 ++++++++- .../project/service_target_eval_test.go | 98 +++++++++++++++++++ 2 files changed, 143 insertions(+), 3 deletions(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go index 734e163195b..ef24f4d00e2 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go @@ -14,6 +14,7 @@ import ( "strings" "github.com/azure/azure-dev/cli/azd/pkg/azdext" + "github.com/azure/azure-dev/cli/azd/pkg/foundry" "google.golang.org/protobuf/types/known/structpb" ) @@ -129,7 +130,7 @@ func (p *EvalServiceTargetProvider) Deploy( targetResource *azdext.TargetResource, progress azdext.ProgressReporter, ) (*azdext.ServiceDeployResult, error) { - cfg, err := EvalConfigFromService(serviceConfig) + cfg, err := EvalConfigFromService(serviceConfig, p.projectRoot(ctx)) if err != nil { return nil, err } @@ -187,6 +188,19 @@ func (p *EvalServiceTargetProvider) Deploy( return &azdext.ServiceDeployResult{}, nil } +// projectRoot is the directory `$ref` paths resolve against. It is the +// directory holding azure.yaml, which only azd can report. +func (p *EvalServiceTargetProvider) projectRoot(ctx context.Context) string { + if p.azdClient == nil { + return "" + } + resp, err := p.azdClient.Project().Get(ctx, &azdext.EmptyRequest{}) + if err != nil || resp.GetProject() == nil { + return "" + } + return resp.GetProject().GetPath() +} + // describeResult reports whether a version was published or reused, so a // no-op deploy is visibly a no-op. func describeResult(kind, name, version string, changed bool) string { @@ -205,7 +219,13 @@ func report(progress azdext.ProgressReporter, message string) { // EvalConfigFromService reads the eval configuration carried inline on the // service entry. azd captures unknown keys into AdditionalProperties and hands // them to the extension untouched. -func EvalConfigFromService(svc *azdext.ServiceConfig) (*EvalConfig, error) { +// +// azd core deliberately does not resolve `$ref` includes for extensions — it +// strips the ServiceConfig fields it owns and leaves `$ref` at the top of the +// map for the owning extension to resolve. Without this call a service written +// as `host: azure.ai.eval` + `$ref: ./evals/azure.yaml` deploys nothing at all, +// because the config parses to an empty set of datasets and groups. +func EvalConfigFromService(svc *azdext.ServiceConfig, projectRoot string) (*EvalConfig, error) { props := serviceProps(svc) if props == nil || len(props.GetFields()) == 0 { return nil, fmt.Errorf( @@ -213,7 +233,16 @@ func EvalConfigFromService(svc *azdext.ServiceConfig) (*EvalConfig, error) { svc.GetName()) } - raw, err := props.MarshalJSON() + values := props.AsMap() + if projectRoot != "" { + resolved, err := foundry.ResolveFileRefs(values, projectRoot) + if err != nil { + return nil, fmt.Errorf("resolving $ref in the eval service configuration: %w", err) + } + values = resolved + } + + raw, err := json.Marshal(values) if err != nil { return nil, fmt.Errorf("reading the eval service configuration: %w", err) } @@ -235,10 +264,23 @@ func serviceProps(svc *azdext.ServiceConfig) *structpb.Struct { } // serviceRelativeDir returns the directory that `source:` paths resolve against. +// +// When the service is authored as `host:` + `$ref: ./evals/azure.yaml`, the +// paths inside that file are written relative to the file itself, so the +// include's own directory is the base. ResolveFileRefs inlines the content +// without rebasing paths, so the base has to be recovered from the `$ref` +// value before resolution. func serviceRelativeDir(svc *azdext.ServiceConfig) string { if svc == nil { return "." } + if props := serviceProps(svc); props != nil { + if ref, ok := props.AsMap()["$ref"].(string); ok && ref != "" { + if dir := filepath.Dir(filepath.FromSlash(ref)); dir != "" { + return dir + } + } + } if p := svc.GetRelativePath(); p != "" { return p } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval_test.go new file mode 100644 index 00000000000..a709de919dc --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval_test.go @@ -0,0 +1,98 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package project + +import ( + "path/filepath" + "testing" + + "github.com/azure/azure-dev/cli/azd/pkg/azdext" + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/types/known/structpb" +) + +func propsFrom(t *testing.T, values map[string]any) *structpb.Struct { + t.Helper() + s, err := structpb.NewStruct(values) + require.NoError(t, err) + return s +} + +// A service authored as `host:` + `$ref: ./evals/azure.yaml` has its relative +// source paths written against the included file, not the project root. +// ResolveFileRefs inlines the content without rebasing them, so the base has to +// come from the $ref value. +func TestServiceRelativeDirUsesRefDirectory(t *testing.T) { + svc := &azdext.ServiceConfig{ + Name: "evals", + AdditionalProperties: propsFrom(t, map[string]any{ + "$ref": "./evals/azure.yaml", + }), + } + require.Equal(t, filepath.FromSlash("evals"), serviceRelativeDir(svc)) +} + +// A nested include keeps its own directory. +func TestServiceRelativeDirUsesNestedRefDirectory(t *testing.T) { + svc := &azdext.ServiceConfig{ + Name: "evals", + AdditionalProperties: propsFrom(t, map[string]any{ + "$ref": "./config/evals/azure.yaml", + }), + } + require.Equal(t, filepath.FromSlash("config/evals"), serviceRelativeDir(svc)) +} + +// Without a $ref the service's own relative path is the base. +func TestServiceRelativeDirFallsBackToRelativePath(t *testing.T) { + svc := &azdext.ServiceConfig{ + Name: "evals", + RelativePath: "evals", + AdditionalProperties: propsFrom(t, map[string]any{ + "datasets": []any{}, + }), + } + require.Equal(t, "evals", serviceRelativeDir(svc)) +} + +// With neither, sources resolve against the project root. +func TestServiceRelativeDirDefaultsToProjectRoot(t *testing.T) { + require.Equal(t, ".", serviceRelativeDir(&azdext.ServiceConfig{Name: "evals"})) + require.Equal(t, ".", serviceRelativeDir(nil)) +} + +// An inline config still parses when no project root is available to resolve +// includes against. +func TestEvalConfigFromServiceReadsInlineConfig(t *testing.T) { + svc := &azdext.ServiceConfig{ + Name: "evals", + AdditionalProperties: propsFrom(t, map[string]any{ + "datasets": []any{ + map[string]any{"name": "golden", "source": "./datasets/golden.jsonl"}, + }, + "evalGroups": []any{ + map[string]any{ + "name": "quality", + "dataset": "golden", + "evaluators": []any{"builtin.task_adherence"}, + "target": map[string]any{"type": "agent", "name": "my-agent"}, + }, + }, + }), + } + + cfg, err := EvalConfigFromService(svc, "") + require.NoError(t, err) + require.Len(t, cfg.Datasets, 1) + require.Equal(t, "golden", cfg.Datasets[0].Name) + require.Len(t, cfg.EvalGroups, 1) + require.Len(t, cfg.EvalGroups[0].Evaluators, 1) + require.Equal(t, "builtin.task_adherence", cfg.EvalGroups[0].Evaluators[0].Name) +} + +func TestEvalConfigFromServiceRejectsEmptyService(t *testing.T) { + _, err := EvalConfigFromService(&azdext.ServiceConfig{Name: "evals"}, "") + require.Error(t, err) + require.Contains(t, err.Error(), "no eval configuration") +} From ae6f43c549c054db7635abd807cc0548d7d89e37 Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 28 Jul 2026 06:18:24 +0300 Subject: [PATCH 016/320] fix(azure.ai.evaluations): recreate an eval group when its own declaration changes Change detection only covered upstream artifacts, so retargeting a group at a different agent, swapping an evaluator, or changing the judge model left the old group in place. Groups are immutable, so the edit silently had no effect and later runs kept evaluating the previous definition. The group's declaration is now fingerprinted alongside the dataset and evaluator artifacts. The id and description are excluded: one is server assigned and the other is cosmetic, so neither should force a recreate. The digest is recorded when an existing group is reused as well as when one is created. Recording it only on create meant a group deployed before this change never established a baseline, and the first edit after it would still go undetected. Verified against a live project: changing the target produced a new group id, and two further deploys with no change reused it. --- .../internal/cmd/reconciler.go | 17 ++++++ .../internal/project/service_target_eval.go | 21 +++++++ .../project/service_target_eval_test.go | 61 +++++++++++++++++++ 3 files changed, 99 insertions(+) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go index 92a55f808f4..3b04655d768 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go @@ -178,9 +178,25 @@ func (r *evalReconciler) EnsureEvalGroup( return group.ID, nil } + // Groups are immutable, so a change to the group's own declaration — + // evaluators, target, or options — needs a new group just as much as a + // change to an upstream artifact does. + digest, err := project.FingerprintGroup(group) + if err != nil { + return "", err + } + key := project.FingerprintKey("evalgroup", group.Name) + if prior := r.ec.getEnvValue(ctx, key); prior != "" && prior != digest { + recreate = true + } + cached := r.ec.getEnvValue(ctx, envKeyEvalGroupID) if cached != "" && !recreate { if _, err := r.ec.evalClient.GetOpenAIEval(ctx, cached); err == nil { + // Record the digest on reuse as well, otherwise a group deployed + // before fingerprinting existed never establishes a baseline and + // later edits go undetected. + _ = r.ec.setEnvValue(ctx, key, digest) return cached, nil } } @@ -197,6 +213,7 @@ func (r *evalReconciler) EnsureEvalGroup( if err != nil { return "", err } + _ = r.ec.setEnvValue(ctx, key, digest) _ = r.ec.setEnvValue(ctx, envKeyEvalGroupID, created.ID) return created.ID, nil } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go index ef24f4d00e2..b6dd930ced4 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go @@ -313,6 +313,27 @@ func Fingerprint(path string) (string, error) { return hex.EncodeToString(sum[:]), nil } +// FingerprintGroup hashes an eval group's own declaration. +// +// Change detection on upstream artifacts is not sufficient: editing a group's +// evaluators, target, or options changes what the group means, and groups are +// immutable, so the group has to be recreated even when the dataset and +// evaluators are untouched. Without this a retargeted group keeps running +// against the old definition. +func FingerprintGroup(group EvalGroup) (string, error) { + // The id is server-assigned and the description is cosmetic; neither + // changes what the group evaluates. + group.ID = "" + group.Description = "" + + data, err := json.Marshal(group) + if err != nil { + return "", fmt.Errorf("hashing eval group %q: %w", group.Name, err) + } + sum := sha256.Sum256(data) + return hex.EncodeToString(sum[:]), nil +} + // FingerprintKey is the azd environment key holding an artifact's fingerprint. func FingerprintKey(kind, name string) string { safe := strings.Map(func(r rune) rune { diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval_test.go index a709de919dc..6052cb99c55 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval_test.go @@ -7,6 +7,8 @@ import ( "path/filepath" "testing" + "azureaieval/internal/pkg/evalcore" + "github.com/azure/azure-dev/cli/azd/pkg/azdext" "github.com/stretchr/testify/require" "google.golang.org/protobuf/types/known/structpb" @@ -96,3 +98,62 @@ func TestEvalConfigFromServiceRejectsEmptyService(t *testing.T) { require.Error(t, err) require.Contains(t, err.Error(), "no eval configuration") } + +// Groups are immutable, so a change to the group's own declaration has to be +// detectable. Upstream artifact fingerprints do not cover it: retargeting a +// group at a different agent leaves the dataset and evaluators untouched. +func TestFingerprintGroupTracksMeaningfulChanges(t *testing.T) { + base := EvalGroup{ + Name: "quality", + Dataset: "golden", + Evaluators: evalcore.EvaluatorList{{Name: "builtin.task_adherence"}}, + Target: &Target{Type: "agent", Name: "agent-a"}, + Options: &Options{EvalModel: "gpt-4.1-nano"}, + } + + original, err := FingerprintGroup(base) + require.NoError(t, err) + + same, err := FingerprintGroup(base) + require.NoError(t, err) + require.Equal(t, original, same, "an unchanged group must keep its fingerprint") + + cases := map[string]func(g *EvalGroup){ + "target": func(g *EvalGroup) { g.Target = &Target{Type: "agent", Name: "agent-b"} }, + "evaluators": func(g *EvalGroup) { + g.Evaluators = append(g.Evaluators, evalcore.EvaluatorRef{Name: "builtin.similarity"}) + }, + "options": func(g *EvalGroup) { g.Options = &Options{EvalModel: "gpt-4o-mini"} }, + "dataset": func(g *EvalGroup) { g.Dataset = "other" }, + } + for name, mutate := range cases { + t.Run(name, func(t *testing.T) { + changed := base + changed.Evaluators = append(evalcore.EvaluatorList(nil), base.Evaluators...) + mutate(&changed) + + digest, err := FingerprintGroup(changed) + require.NoError(t, err) + require.NotEqual(t, original, digest, "changing %s must change the fingerprint", name) + }) + } +} + +// Server-assigned and cosmetic fields must not force a recreate. +func TestFingerprintGroupIgnoresIdAndDescription(t *testing.T) { + base := EvalGroup{ + Name: "quality", + Dataset: "golden", + Evaluators: evalcore.EvaluatorList{{Name: "builtin.task_adherence"}}, + } + original, err := FingerprintGroup(base) + require.NoError(t, err) + + noisy := base + noisy.ID = "eval_abc123" + noisy.Description = "reworded" + + digest, err := FingerprintGroup(noisy) + require.NoError(t, err) + require.Equal(t, original, digest) +} From e71fc44bb8122a9aef0adf67ea7a00a4019dc4e5 Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 28 Jul 2026 06:25:17 +0300 Subject: [PATCH 017/320] fix(azure.ai.evaluations): keep HTTP traces out of normal output The data-plane clients trace every request and response through log.Printf, which Go writes to stderr by default, so a plain command interleaved raw URLs and status lines with its own output. A long generate run was mostly HTTP traces. Ports the debug setup from the agents extension: the standard logger is discarded unless --debug or AZD_EXT_DEBUG is set, and debug output goes to a dated file rather than the terminal. The hook chains the SDK PersistentPreRunE instead of replacing it. Assigning PersistentPreRun has no effect once the E variant is set, and overwriting the E variant would drop the SDK own setup. Also reports jobs as submitted when generate is given --no-wait, which is a successful submission rather than an empty result. --- .../internal/cmd/debug.go | 65 +++++++++++++++++++ .../internal/cmd/generate.go | 8 +++ .../azure.ai.evaluations/internal/cmd/root.go | 19 ++++++ 3 files changed, 92 insertions(+) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/debug.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/debug.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/debug.go new file mode 100644 index 00000000000..48fc0802aaf --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/debug.go @@ -0,0 +1,65 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "fmt" + "io" + "log" + "os" + "strconv" + "time" + + azcorelog "github.com/Azure/azure-sdk-for-go/sdk/azcore/log" + "github.com/spf13/pflag" +) + +// setupDebugLogging silences the standard logger unless debug mode is on. +// +// The data-plane clients trace every request through log.Printf, which Go +// writes to stderr by default. Without this the CLI interleaves raw HTTP traces +// with its own output on every command. Returns a cleanup function the caller +// should defer. +func setupDebugLogging(flags *pflag.FlagSet) func() { + if !isDebug(flags) { + log.SetOutput(io.Discard) + azcorelog.SetListener(nil) + return func() {} + } + + logFileName := fmt.Sprintf("azd-ai-eval-%s.log", time.Now().Format("2006-01-02")) + + //nolint:gosec // the name is generated locally from the date, not user input + logFile, err := os.OpenFile(logFileName, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0600) + + var w io.Writer + var closeFile func() + if err != nil { + w = os.Stderr + closeFile = func() {} + } else { + w = logFile + closeFile = func() { logFile.Close() } //nolint:gosec // best-effort cleanup + } + + log.SetOutput(w) + azcorelog.SetListener(func(event azcorelog.Event, msg string) { + fmt.Fprintf(w, "[%s] %s: %s\n", time.Now().Format(time.RFC3339), event, msg) + }) + + return func() { + log.SetOutput(io.Discard) + azcorelog.SetListener(nil) + closeFile() + } +} + +// isDebug reports whether --debug or AZD_EXT_DEBUG is set. +func isDebug(flags *pflag.FlagSet) bool { + if debugFlag, err := flags.GetBool("debug"); err == nil && debugFlag { + return true + } + debug, _ := strconv.ParseBool(os.Getenv("AZD_EXT_DEBUG")) + return debug +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go index 3ce03cbfd65..334a95c6b61 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go @@ -95,6 +95,14 @@ func newGenerateCommand() *cobra.Command { } if len(datasetRefs) == 0 && len(evaluatorRefs) == 0 { + // With --no-wait the jobs were submitted and nothing was + // downloaded, which is success, not an empty result. + if noWait { + fmt.Fprintln(out, + "\nJobs submitted. Re-run without --no-wait to download the artifacts "+ + "and reference them from the deployment spec.") + return nil + } fmt.Fprintln(out, "Nothing was generated.") return nil } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/root.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/root.go index 3a289d6347d..a763a54b924 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/root.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/root.go @@ -25,6 +25,25 @@ func NewRootCommand() *cobra.Command { rootCmd.SilenceErrors = true rootCmd.CompletionOptions.DisableDefaultCmd = true + // The data-plane clients trace requests through the standard logger, which + // Go writes to stderr, so it has to be silenced unless debug was asked for. + // + // The SDK's own hook is chained rather than replaced, and cobra ignores + // PersistentPreRun entirely once PersistentPreRunE is set. The SDK sets + // cobra.EnableTraverseRunHooks, so this still runs alongside subcommand + // hooks. The cleanup func is discarded on purpose: log writes are + // unbuffered and the OS closes the file at exit. + sdkPreRun := rootCmd.PersistentPreRunE + rootCmd.PersistentPreRunE = func(cmd *cobra.Command, args []string) error { + if sdkPreRun != nil { + if err := sdkPreRun(cmd, args); err != nil { + return err + } + } + setupDebugLogging(cmd.Flags()) + return nil + } + rootCmd.AddCommand( newInitCommand(), newGenerateCommand(), From 02e9e953e24885df53fd6e5b3edceb068f7825f5 Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 28 Jul 2026 06:34:41 +0300 Subject: [PATCH 018/320] feat(azure.ai.evaluations): explain why agent-seeded data generation fails Data generation with an agent source is accepted and then fails within seconds with DataGenerationJobSystemError, whose message says only that something went wrong and to try again. It is not transient: it reproduces for every agent tried, while the identical request without the agent source runs normally. The CLI now names the agent, says a retry will not help, and points at the two workarounds, instead of relaying advice that cannot succeed. --- .../internal/cmd/generate.go | 29 +++++++++++- .../internal/cmd/generate_test.go | 45 +++++++++++++++++++ 2 files changed, 73 insertions(+), 1 deletion(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go index 334a95c6b61..d3b2f4ce94f 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go @@ -266,7 +266,7 @@ func (ec *evalContext) generateDataset( completed, err := ec.pollGeneration(ctx, job.ID, DataGenerationAPIVersion, ec.evalClient.GetDataGenerationJob) if err != nil { - return nil, fmt.Errorf("data generation: %w", err) + return nil, fmt.Errorf("data generation: %w", explainDataGenerationFailure(err, cfg.Agent.Name)) } name, version := completed.ResolvedNameVersion() @@ -295,6 +295,33 @@ func (ec *evalContext) generateDataset( return &project.ArtifactRef{Name: spec.Name, Source: relativeSource(baseDir, path)}, nil } +// explainDataGenerationFailure adds context to the service's opaque system +// error. +// +// Seeding generation from an agent currently fails server-side with +// DataGenerationJobSystemError for every agent, within seconds, while the same +// request without the agent source runs normally. The raw message says only +// that something went wrong and to try again, which sends users into a retry +// loop against a deterministic failure. +func explainDataGenerationFailure(err error, agentName string) error { + if err == nil || agentName == "" { + return err + } + // The poller surfaces the service's message; the code is not always in it. + text := err.Error() + if !strings.Contains(text, "DataGenerationJobSystemError") && + !strings.Contains(text, "Something went wrong during data generation") { + return err + } + return fmt.Errorf( + "%w\n\n"+ + "This job seeded generation from agent %q. Agent-seeded data generation is "+ + "currently failing in the service for every agent, so retrying will not help.\n"+ + "Workarounds: supply your own dataset with --dataset, or run without --target "+ + "to generate from the instruction alone.", + err, agentName) +} + // pollGeneration waits for a generation job using the raised budget. func (ec *evalContext) pollGeneration( ctx context.Context, diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_test.go new file mode 100644 index 00000000000..e4c04baebf8 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_test.go @@ -0,0 +1,45 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "errors" + "fmt" + "testing" + + "github.com/stretchr/testify/require" +) + +// The service's system error says only that something went wrong and to try +// again, but agent-seeded generation fails deterministically, so a bare retry +// suggestion sends users into a loop. +func TestExplainDataGenerationFailureAddsAgentContext(t *testing.T) { + err := errors.New( + `job failed with status "failed": Something went wrong during data generation. Please try again.`) + + explained := explainDataGenerationFailure(err, "my-agent") + require.Error(t, explained) + require.Contains(t, explained.Error(), "my-agent") + require.Contains(t, explained.Error(), "--dataset") + require.ErrorIs(t, explained, err, "the original error must stay in the chain") +} + +// The code spelling is matched as well, in case the poller starts surfacing it. +func TestExplainDataGenerationFailureMatchesErrorCode(t *testing.T) { + err := fmt.Errorf("job failed: DataGenerationJobSystemError") + explained := explainDataGenerationFailure(err, "my-agent") + require.Contains(t, explained.Error(), "Workarounds") +} + +// Unrelated failures are passed through untouched, and so is a job that had no +// agent source to blame. +func TestExplainDataGenerationFailureLeavesOthersAlone(t *testing.T) { + other := errors.New("submitting the data generation job: 403 Forbidden") + require.Equal(t, other, explainDataGenerationFailure(other, "my-agent")) + + systemErr := errors.New("Something went wrong during data generation") + require.Equal(t, systemErr, explainDataGenerationFailure(systemErr, "")) + + require.NoError(t, explainDataGenerationFailure(nil, "my-agent")) +} From a049fa64793a43c2ce451ddcf37e59a87321f9c0 Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 28 Jul 2026 06:45:05 +0300 Subject: [PATCH 019/320] feat(azure.ai.evaluations): complete the atomic run operations The spec lists run start, list, show and cancel, and M1 requires every operation to be reachable atomically, but run was a single composite command with no subcommands. Listing runs, inspecting one, and cancelling an in-flight run were unreachable, even though the client already had the calls. Adds run list, run show and run cancel. Each takes the eval group id as an optional argument and otherwise falls back to the id recorded in the azd environment, matching results show. Cancelling a run that already reached a terminal state is refused locally, because the service reports success either way and the CLI would otherwise claim to have cancelled a finished run. Two related fixes. Passing --project-endpoint disabled the azd environment cache entirely: the environment name was only resolved when the endpoint came from azd, so every cached eval group and run id lookup returned empty. The name is now resolved independently of where the endpoint came from. The spec documents --wait and --no-wait, but cobra does not derive the negative form from a bool, so --no-wait was rejected as an unknown flag. Verified live: start with --no-wait, list, show, cancel, and the terminal-state guard on a second cancel. JSON output checked on the new subcommands. --- .../internal/cmd/context.go | 9 +- .../azure.ai.evaluations/internal/cmd/run.go | 11 ++ .../internal/cmd/run_ops.go | 180 ++++++++++++++++++ .../internal/cmd/run_ops_test.go | 52 +++++ 4 files changed, 251 insertions(+), 1 deletion(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/context.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/context.go index f9ba6d0ec8d..ef7562644fa 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/context.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/context.go @@ -46,10 +46,17 @@ func newEvalContext(ctx context.Context, endpointFlag string) (*evalContext, err ec := &evalContext{azdClient: azdClient} + // The environment name is resolved regardless of where the endpoint comes + // from: it is what the cached eval group and run ids are read from and + // written to. Deriving it only when the endpoint came from azd meant + // --project-endpoint silently disabled that cache. + azdEndpoint, envName := lookupEndpointFromAzd(ctx, azdClient) + ec.envName = envName + if endpointFlag != "" { ec.endpoint = endpointFlag } else { - ec.endpoint, ec.envName = lookupEndpointFromAzd(ctx, azdClient) + ec.endpoint = azdEndpoint } if ec.endpoint == "" { ec.endpoint = os.Getenv(projectEndpointEnvKey) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go index abae99924ac..30fe1c42100 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go @@ -139,7 +139,18 @@ func newRunCommand() *cobra.Command { cmd.Flags().IntVar(&maxSamples, "max-samples", 0, "Cap the rows sent from a local dataset file. Ignored for registered datasets.") cmd.Flags().BoolVar(&wait, "wait", true, "Block until the run reaches a terminal state.") + // The spec documents --no-wait, and cobra does not derive it from a bool. + var noWait bool + cmd.Flags().BoolVar(&noWait, "no-wait", false, "Submit the run and return immediately.") + cmd.PreRun = func(*cobra.Command, []string) { + if noWait { + wait = false + } + } + cmd.MarkFlagsMutuallyExclusive("wait", "no-wait") cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") + + addRunSubcommands(cmd) return cmd } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go new file mode 100644 index 00000000000..c0eee5226cc --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go @@ -0,0 +1,180 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "fmt" + + "azureaieval/internal/pkg/eval_api" + + "github.com/spf13/cobra" +) + +// addRunSubcommands attaches the atomic run operations. +// +// `azd ai eval run` stays the composite that creates the group if needed and +// starts a run; these expose the individual operations so every one is +// reachable without the config file. +func addRunSubcommands(cmd *cobra.Command) { + cmd.AddCommand( + newRunListCommand(), + newRunShowCommand(), + newRunCancelCommand(), + ) +} + +func newRunListCommand() *cobra.Command { + var endpointFlg string + + cmd := &cobra.Command{ + Use: "list [eval-id]", + Short: "List runs for an eval group.", + Args: cobra.MaximumNArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + ctx := cmd.Context() + ec, err := newEvalContext(ctx, endpointFlg) + if err != nil { + return err + } + defer ec.Close() + + evalID, err := resolveEvalID(cmd, ec, args) + if err != nil { + return err + } + + list, err := ec.evalClient.ListOpenAIEvalRuns(ctx, evalID, 0) + if err != nil { + return fmt.Errorf("listing runs for %q: %w", evalID, err) + } + if isJSON(cmd) { + return emitJSON(cmd.OutOrStdout(), list) + } + if list == nil || len(list.Data) == 0 { + fmt.Fprintf(cmd.OutOrStdout(), "Eval group %s has no runs yet.\n", evalID) + return nil + } + + rows := make([][]string, 0, len(list.Data)) + for _, run := range list.Data { + rows = append(rows, []string{run.ID, run.Name, run.Status, summarizeCounts(run.ResultCounts)}) + } + return emitTable(cmd.OutOrStdout(), + []string{"RUN ID", "NAME", "STATUS", "RESULTS"}, rows) + }, + } + cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") + return cmd +} + +func newRunShowCommand() *cobra.Command { + var ( + runID string + endpointFlg string + ) + + cmd := &cobra.Command{ + Use: "show [eval-id]", + Short: "Show a single run.", + Args: cobra.MaximumNArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + ctx := cmd.Context() + ec, err := newEvalContext(ctx, endpointFlg) + if err != nil { + return err + } + defer ec.Close() + + evalID, err := resolveEvalID(cmd, ec, args) + if err != nil { + return err + } + + run, err := ec.latestOrNamedRun(cmd, evalID, runID) + if err != nil { + return err + } + if isJSON(cmd) { + return emitJSON(cmd.OutOrStdout(), run) + } + + out := cmd.OutOrStdout() + fmt.Fprintf(out, "Run %s\n", run.ID) + fmt.Fprintf(out, " name : %s\n", run.Name) + fmt.Fprintf(out, " status : %s\n", run.Status) + if counts := summarizeCounts(run.ResultCounts); counts != "" { + fmt.Fprintf(out, " results : %s\n", counts) + } + if run.ReportURL != "" { + fmt.Fprintf(out, " report : %s\n", run.ReportURL) + } + return nil + }, + } + cmd.Flags().StringVar(&runID, "run-id", "", "Run to show. Defaults to the most recent run.") + cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") + return cmd +} + +func newRunCancelCommand() *cobra.Command { + var ( + runID string + endpointFlg string + ) + + cmd := &cobra.Command{ + Use: "cancel [eval-id]", + Short: "Cancel an in-flight run.", + Args: cobra.MaximumNArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + ctx := cmd.Context() + ec, err := newEvalContext(ctx, endpointFlg) + if err != nil { + return err + } + defer ec.Close() + + evalID, err := resolveEvalID(cmd, ec, args) + if err != nil { + return err + } + + target, err := ec.latestOrNamedRun(cmd, evalID, runID) + if err != nil { + return err + } + // Cancelling a run that already finished is a no-op worth naming, + // since the service reports success either way. + if terminalRunStates[target.Status] { + return fmt.Errorf("run %s already finished with status %q", + target.ID, target.Status) + } + + canceled, err := ec.evalClient.CancelOpenAIEvalRun(ctx, evalID, target.ID) + if err != nil { + return fmt.Errorf("cancelling run %s: %w", target.ID, err) + } + if isJSON(cmd) { + return emitJSON(cmd.OutOrStdout(), canceled) + } + status := canceled.Status + if status == "" { + status = "cancelling" + } + fmt.Fprintf(cmd.OutOrStdout(), "Run %s is now %s\n", target.ID, status) + return nil + }, + } + cmd.Flags().StringVar(&runID, "run-id", "", "Run to cancel. Defaults to the most recent run.") + cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") + return cmd +} + +func summarizeCounts(counts *eval_api.EvalRunResultCounts) string { + if counts == nil { + return "" + } + return fmt.Sprintf("%d passed, %d failed, %d errored", + counts.Passed, counts.Failed, counts.Errored) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops_test.go new file mode 100644 index 00000000000..01601ef023e --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops_test.go @@ -0,0 +1,52 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "testing" + + "azureaieval/internal/pkg/eval_api" + + "github.com/stretchr/testify/require" +) + +func TestSummarizeCounts(t *testing.T) { + require.Equal(t, "", summarizeCounts(nil)) + require.Equal(t, "3 passed, 1 failed, 0 errored", + summarizeCounts(&eval_api.EvalRunResultCounts{Total: 4, Passed: 3, Failed: 1})) +} + +// Cancelling a finished run is rejected locally. The service reports success +// either way, so without this the CLI would claim it cancelled a run that had +// already completed. +func TestTerminalRunStatesCoverServiceVocabulary(t *testing.T) { + for _, status := range []string{"completed", "failed", "canceled", "cancelled", "error"} { + require.True(t, terminalRunStates[status], "%q should be terminal", status) + } + for _, status := range []string{"in_progress", "queued", "running", ""} { + require.False(t, terminalRunStates[status], "%q should not be terminal", status) + } +} + +// The atomic run operations have to be reachable as subcommands; the spec +// requires start, list, show and cancel to exist alongside the composite. +func TestRunCommandExposesAtomicSubcommands(t *testing.T) { + cmd := newRunCommand() + + found := map[string]bool{} + for _, sub := range cmd.Commands() { + found[sub.Name()] = true + } + for _, name := range []string{"list", "show", "cancel"} { + require.True(t, found[name], "run should expose the %q subcommand", name) + } +} + +// --no-wait is documented in the spec, and cobra does not derive it from the +// --wait bool. +func TestRunCommandAcceptsNoWait(t *testing.T) { + cmd := newRunCommand() + require.NotNil(t, cmd.Flags().Lookup("no-wait"), "run should accept --no-wait") + require.NotNil(t, cmd.Flags().Lookup("wait"), "run should keep --wait") +} From 1ccb6d9cba252fd855e55d4b23d2181832aad641 Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 28 Jul 2026 06:51:29 +0300 Subject: [PATCH 020/320] feat(azure.ai.evaluations): add run start to complete the atomic surface The spec lists run start alongside list, show and cancel. The behaviour existed only as the composite `azd ai eval run`, so the atomic name in the spec did not resolve. Both forms are now built by one constructor, so their flags cannot drift apart, and a test asserts that. --- .../azure.ai.evaluations/internal/cmd/run.go | 19 ++++++++++++++--- .../internal/cmd/run_ops_test.go | 21 ++++++++++++++++++- 2 files changed, 36 insertions(+), 4 deletions(-) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go index 30fe1c42100..66cc843e103 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go @@ -28,7 +28,21 @@ var terminalRunStates = map[string]bool{ "error": true, } +// newRunCommand builds the composite `azd ai eval run` and attaches the atomic +// run operations, including `run start` which the spec lists as the atomic form +// of this same command. func newRunCommand() *cobra.Command { + cmd := buildRunCommand( + "run", "Run an evaluation, creating the eval group if it does not exist yet.") + addRunSubcommands(cmd) + cmd.AddCommand(buildRunCommand( + "start", "Start a run, creating the eval group if it does not exist yet.")) + return cmd +} + +// buildRunCommand is shared by `run` and `run start` so the two forms cannot +// drift apart. +func buildRunCommand(use, short string) *cobra.Command { var ( configPath string groupName string @@ -41,8 +55,8 @@ func newRunCommand() *cobra.Command { ) cmd := &cobra.Command{ - Use: "run", - Short: "Run an evaluation, creating the eval group if it does not exist yet.", + Use: use, + Short: short, RunE: func(cmd *cobra.Command, args []string) error { ctx := cmd.Context() out := cmd.OutOrStdout() @@ -150,7 +164,6 @@ func newRunCommand() *cobra.Command { cmd.MarkFlagsMutuallyExclusive("wait", "no-wait") cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") - addRunSubcommands(cmd) return cmd } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops_test.go index 01601ef023e..63ef3344e2e 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops_test.go @@ -8,6 +8,7 @@ import ( "azureaieval/internal/pkg/eval_api" + "github.com/spf13/cobra" "github.com/stretchr/testify/require" ) @@ -38,11 +39,29 @@ func TestRunCommandExposesAtomicSubcommands(t *testing.T) { for _, sub := range cmd.Commands() { found[sub.Name()] = true } - for _, name := range []string{"list", "show", "cancel"} { + for _, name := range []string{"start", "list", "show", "cancel"} { require.True(t, found[name], "run should expose the %q subcommand", name) } } +// `run start` is the atomic form of the composite and must accept the same +// flags, otherwise the two forms diverge. +func TestRunStartMirrorsCompositeFlags(t *testing.T) { + composite := newRunCommand() + + var start *cobra.Command + for _, sub := range composite.Commands() { + if sub.Name() == "start" { + start = sub + } + } + require.NotNil(t, start) + + for _, flag := range []string{"eval-id", "eval-group", "name", "level", "max-samples", "wait", "no-wait"} { + require.NotNil(t, start.Flags().Lookup(flag), "run start should accept --%s", flag) + } +} + // --no-wait is documented in the spec, and cobra does not derive it from the // --wait bool. func TestRunCommandAcceptsNoWait(t *testing.T) { From 45802246fbcfa60d3b038465afbb8cf7028088e0 Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 28 Jul 2026 07:06:47 +0300 Subject: [PATCH 021/320] fix(azure.ai.evaluations): make the dataset and evaluator write paths work Exercising the atomic write commands against a live project found three failures. None were covered by tests, because none of these paths had been run end to end. dataset update always collided. It passed the --version flag straight to UploadNewVersion, which derives the next version from what it is given, so an omitted flag restarted at 1.0 and the service returned 409 TemporaryDataReferencesForExistingAsset. The flag help promised the opposite, that omitting it would take the next version. This is the same defect that was fixed in the deploy reconciler earlier, so the discovery is now centralised in DatasetClient.UploadNextVersion and both callers use it, rather than being fixed twice and available to be missed a third time. evaluator upload rejected every hand-authored rubric. The service needs a type discriminator on the definition, and without it fails the whole request with "The request field is required", which names a field that is present. Generated rubrics carry the type, so only the hand-authored path documented in the spec was affected. The type is now filled in when absent and left alone when set. evaluator show returned 404. It omitted the version segment from the path, but the service has no route for an unversioned evaluator, despite the doc comment claiming the latest would be fetched. The latest version is now resolved first, comparing numerically because versions are integers as strings and a lexical compare ranks "9" above "15" -- the service already publishes evaluators at version 15 and 17. Verified live: dataset create, show, update to 2.0, list and delete; evaluator upload, show resolving the latest, update to version 2, list and delete. Both suites leave nothing behind. --- .../internal/cmd/dataset.go | 4 +- .../internal/cmd/evaluator.go | 36 ++++++++- .../internal/cmd/evaluator_test.go | 81 +++++++++++++++++++ .../internal/cmd/reconciler.go | 25 +----- .../internal/pkg/dataset_api/operations.go | 25 ++++++ .../pkg/eval_api/evaluators_version_test.go | 47 +++++++++++ .../internal/pkg/eval_api/operations.go | 64 ++++++++++++++- 7 files changed, 253 insertions(+), 29 deletions(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator_test.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/evaluators_version_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset.go index 41685ae7e18..ebdc4439e83 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset.go @@ -76,7 +76,7 @@ func newDatasetCreateCommand(update bool) *cobra.Command { } defer ec.Close() - ds, err := ec.datasetClient.UploadNewVersion( + ds, err := ec.datasetClient.UploadNextVersion( ctx, name, version, localDir, ProjectEndpointAPIVersion, ) if err != nil { @@ -99,7 +99,7 @@ func newDatasetCreateCommand(update bool) *cobra.Command { cmd.Flags().StringVar(&name, "name", "", "Name of the dataset.") cmd.Flags().StringVar(&file, "file", "", "Path to a .jsonl file, or a directory containing one.") cmd.Flags().StringVar(&version, "version", "", - "Current version to increment from. Omit to let the server assign the next version.") + "Current version to increment from. Omit to increment from the latest registered version.") cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") return cmd } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator.go index 9bbc7e41b1a..ca1d597eaa3 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator.go @@ -98,14 +98,42 @@ func newEvaluatorUploadCommand(update bool) *cobra.Command { // normalizeRubricBody accepts either a bare definition ({type, dimensions}) or // a full evaluator document ({name, definition}) and returns the request body. +// rubricDefinitionType is the discriminator the service uses to deserialize a +// rubric definition. +const rubricDefinitionType = "rubric" + +// ensureDefinitionType adds the type discriminator when a definition omits it. +// +// Without it the service cannot tell which definition kind it is holding and +// rejects the whole request with "The request field is required", which points +// at the wrong field entirely. Generated rubrics carry the type; hand-authored +// ones written to the shape the spec documents — a bare list of weighted +// dimensions — do not. +func ensureDefinitionType(definition json.RawMessage) (json.RawMessage, error) { + var doc map[string]json.RawMessage + if err := json.Unmarshal(definition, &doc); err != nil { + return nil, fmt.Errorf("the definition is not a JSON object: %w", err) + } + if _, ok := doc["type"]; ok { + return definition, nil + } + doc["type"] = json.RawMessage(fmt.Sprintf("%q", rubricDefinitionType)) + return json.Marshal(doc) +} + func normalizeRubricBody(name string, raw []byte) (json.RawMessage, error) { var probe map[string]json.RawMessage if err := json.Unmarshal(raw, &probe); err != nil { return nil, fmt.Errorf("not valid JSON: %w", err) } - if _, hasDefinition := probe["definition"]; hasDefinition { + if definition, hasDefinition := probe["definition"]; hasDefinition { // Already a full document; make sure the name matches the flag. + typed, err := ensureDefinitionType(definition) + if err != nil { + return nil, err + } + probe["definition"] = typed probe["name"] = json.RawMessage(fmt.Sprintf("%q", name)) out, err := json.Marshal(probe) if err != nil { @@ -119,9 +147,13 @@ func normalizeRubricBody(name string, raw []byte) (json.RawMessage, error) { "expected a rubric definition with 'dimensions', or a document with 'definition'") } + typed, err := ensureDefinitionType(raw) + if err != nil { + return nil, err + } doc := map[string]any{ "name": name, - "definition": json.RawMessage(raw), + "definition": typed, } out, err := json.Marshal(doc) if err != nil { diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator_test.go new file mode 100644 index 00000000000..e7ad1feab01 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator_test.go @@ -0,0 +1,81 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "encoding/json" + "testing" + + "github.com/stretchr/testify/require" +) + +// The service needs a type discriminator to deserialize a definition. Without +// it the whole request is rejected with "The request field is required", which +// names the wrong field, so a hand-authored rubric failed to upload. +func TestNormalizeRubricBodyAddsDefinitionType(t *testing.T) { + raw := []byte(`{"dimensions":[{"id":"accuracy","description":"Correct.","weight":5}]}`) + + body, err := normalizeRubricBody("support-quality", raw) + require.NoError(t, err) + + var doc struct { + Name string `json:"name"` + Definition struct { + Type string `json:"type"` + Dimensions []struct { + ID string `json:"id"` + Weight int `json:"weight"` + } `json:"dimensions"` + } `json:"definition"` + } + require.NoError(t, json.Unmarshal(body, &doc)) + require.Equal(t, "support-quality", doc.Name) + require.Equal(t, "rubric", doc.Definition.Type) + require.Len(t, doc.Definition.Dimensions, 1) + require.Equal(t, 5, doc.Definition.Dimensions[0].Weight) +} + +// A definition that already declares its type keeps it, so a generated rubric +// round-trips unchanged. +func TestNormalizeRubricBodyKeepsExistingType(t *testing.T) { + raw := []byte(`{"type":"custom_kind","dimensions":[{"id":"a","weight":1}]}`) + + body, err := normalizeRubricBody("x", raw) + require.NoError(t, err) + + var doc struct { + Definition struct { + Type string `json:"type"` + } `json:"definition"` + } + require.NoError(t, json.Unmarshal(body, &doc)) + require.Equal(t, "custom_kind", doc.Definition.Type) +} + +// A full document is normalized the same way, and the name follows the flag. +func TestNormalizeRubricBodyHandlesFullDocument(t *testing.T) { + raw := []byte(`{"name":"stale","definition":{"dimensions":[{"id":"a","weight":1}]}}`) + + body, err := normalizeRubricBody("actual-name", raw) + require.NoError(t, err) + + var doc struct { + Name string `json:"name"` + Definition struct { + Type string `json:"type"` + } `json:"definition"` + } + require.NoError(t, json.Unmarshal(body, &doc)) + require.Equal(t, "actual-name", doc.Name) + require.Equal(t, "rubric", doc.Definition.Type) +} + +func TestNormalizeRubricBodyRejectsNonRubric(t *testing.T) { + _, err := normalizeRubricBody("x", []byte(`{"something":1}`)) + require.Error(t, err) + require.Contains(t, err.Error(), "dimensions") + + _, err = normalizeRubricBody("x", []byte(`not json`)) + require.Error(t, err) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go index 3b04655d768..965f67a4634 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go @@ -84,17 +84,10 @@ func (r *evalReconciler) EnsureDataset( dir = filepath.Dir(localPath) } - // UploadNewVersion derives the next version from the one passed in, so it - // has to be told what is already registered. Passing an empty version - // restarts at 1.0 and the service rejects the pending upload with - // TemporaryDataReferencesForExistingAsset once that version exists. - currentVersion := decl.Version - if currentVersion == "" { - currentVersion = r.latestDatasetVersion(ctx, decl.Name) - } - - ds, err := r.ec.datasetClient.UploadNewVersion( - ctx, decl.Name, currentVersion, dir, ProjectEndpointAPIVersion, + // UploadNextVersion discovers the currently registered version when none is + // declared, so the upload does not restart at 1.0 and collide. + ds, err := r.ec.datasetClient.UploadNextVersion( + ctx, decl.Name, decl.Version, dir, ProjectEndpointAPIVersion, ) if err != nil { return "", false, err @@ -107,16 +100,6 @@ func (r *evalReconciler) EnsureDataset( return ds.Version, true, nil } -// latestDatasetVersion reports the newest registered version, or empty when the -// dataset does not exist yet so the first upload starts at 1.0. -func (r *evalReconciler) latestDatasetVersion(ctx context.Context, name string) string { - list, err := r.ec.datasetClient.ListDatasetVersions(ctx, name, ProjectEndpointAPIVersion) - if err != nil || list == nil || len(list.Value) == 0 { - return "" - } - return dataset_api.LatestVersion(list.Value) -} - // EnsureEvaluator publishes a new version when the local definition differs // from what the service holds. Evaluator definitions come back inline, so this // compares content directly rather than relying on a cached digest. diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/operations.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/operations.go index e29b6710f12..501f558408f 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/operations.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/operations.go @@ -80,6 +80,31 @@ func (c *DatasetClient) CreateDataset( return doRequestTyped[Dataset](c, ctx, http.MethodPost, pathDatasets, nil, request, apiVersion) } +// UploadNextVersion registers the next version of a dataset, discovering the +// current one from the service when currentVersion is empty. +// +// Prefer this over UploadNewVersion. That function derives the next version +// from whatever it is handed, so an empty value restarts at 1.0 and the +// service rejects the pending upload with a 409 +// TemporaryDataReferencesForExistingAsset as soon as 1.0 exists. Callers +// almost always mean "the version after whatever is registered", which is what +// this does. +func (c *DatasetClient) UploadNextVersion( + ctx context.Context, + name string, + currentVersion string, + localDir string, + apiVersion string, +) (*Dataset, error) { + if currentVersion == "" { + list, err := c.ListDatasetVersions(ctx, name, apiVersion) + if err == nil && list != nil && len(list.Value) > 0 { + currentVersion = LatestVersion(list.Value) + } + } + return c.UploadNewVersion(ctx, name, currentVersion, localDir, apiVersion) +} + // UploadNewVersion reads the first JSONL file from localDir, computes the next // version from currentVersion, and uploads it as a new dataset version using // the 3-step pending upload flow: diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/evaluators_version_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/evaluators_version_test.go new file mode 100644 index 00000000000..46f7524acbf --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/evaluators_version_test.go @@ -0,0 +1,47 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package eval_api + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +// Evaluator versions are integers rendered as strings, so a lexical compare +// ranks "9" above "15". The live service already has evaluators at version 15 +// and 17, so this is not hypothetical. +func TestPickLatestEvaluatorVersionIsNumeric(t *testing.T) { + cases := []struct { + name string + versions []string + want string + }{ + {"single", []string{"1"}, "1"}, + {"ascending", []string{"1", "2", "3"}, "3"}, + {"unordered", []string{"3", "1", "2"}, "3"}, + {"double digits beat single", []string{"9", "15"}, "15"}, + {"realistic", []string{"1", "9", "10", "17", "2"}, "17"}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + entries := make([]EvaluatorSummary, 0, len(tc.versions)) + for _, v := range tc.versions { + entries = append(entries, EvaluatorSummary{Name: "e", Version: v}) + } + require.Equal(t, tc.want, pickLatestVersion(entries)) + }) + } +} + +// A non-numeric version is only used when nothing numeric exists, so one odd +// entry cannot mask the real latest. +func TestPickLatestEvaluatorVersionHandlesNonNumeric(t *testing.T) { + require.Equal(t, "2", pickLatestVersion([]EvaluatorSummary{ + {Version: "draft"}, {Version: "1"}, {Version: "2"}, + })) + require.Equal(t, "draft", pickLatestVersion([]EvaluatorSummary{{Version: "draft"}})) + require.Equal(t, "", pickLatestVersion(nil)) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/operations.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/operations.go index 795e8301e0c..d04a3104636 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/operations.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/operations.go @@ -127,20 +127,76 @@ func (c *EvalClient) CreateEvaluatorVersion( } // GetEvaluatorRaw gets an evaluator by name and version as raw JSON. -// If version is empty, the latest version is fetched. +// If version is empty, the latest version is resolved first. +// +// The service has no route for an unversioned evaluator: GET +// /evaluators/{name} returns 404 with no body, so the version cannot simply be +// left off the path. func (c *EvalClient) GetEvaluatorRaw( ctx context.Context, name string, version string, apiVersion string, ) (json.RawMessage, error) { - path := pathEvaluators + "/" + url.PathEscape(name) - if version != "" { - path += "/versions/" + url.PathEscape(version) + if version == "" { + latest, err := c.LatestEvaluatorVersion(ctx, name, apiVersion) + if err != nil { + return nil, err + } + version = latest } + path := pathEvaluators + "/" + url.PathEscape(name) + + "/versions/" + url.PathEscape(version) return c.doRequest(ctx, http.MethodGet, path, nil, nil, apiVersion) } +// LatestEvaluatorVersion returns the newest registered version of an evaluator. +func (c *EvalClient) LatestEvaluatorVersion( + ctx context.Context, + name string, + apiVersion string, +) (string, error) { + list, err := c.ListEvaluatorVersions(ctx, name, apiVersion) + if err != nil { + return "", err + } + if list == nil || len(list.Value) == 0 { + return "", fmt.Errorf("evaluator %q has no versions", name) + } + latest := pickLatestVersion(list.Value) + if latest == "" { + return "", fmt.Errorf("evaluator %q has no usable version", name) + } + return latest, nil +} + +// pickLatestVersion selects the highest evaluator version. +// +// Versions are integers rendered as strings, so they are compared numerically: +// a lexical compare would rank "9" above "15", and the service already +// publishes evaluators at version 15 and 17. A non-numeric version is used +// only when nothing numeric is present. +func pickLatestVersion(entries []EvaluatorSummary) string { + best := "" + bestNum := -1 + for _, entry := range entries { + if entry.Version == "" { + continue + } + num, err := strconv.Atoi(entry.Version) + if err != nil { + if best == "" { + best = entry.Version + } + continue + } + if num > bestNum { + bestNum, best = num, entry.Version + } + } + return best +} + // CreateOpenAIEval creates an OpenAI eval definition. func (c *EvalClient) CreateOpenAIEval( ctx context.Context, From 331d3fe6c01a6fa00091737027a3a651d62b7036 Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 28 Jul 2026 07:13:00 +0300 Subject: [PATCH 022/320] fix(azure.ai.evaluations): support custom evaluators in a deployed config Deploying a config that declares a custom evaluator, rather than only built-in ones, failed in two ways. Every earlier test used built-ins, so neither showed up. The evaluator was republished on every deploy. The service enriches a definition when it stores it, so a rubric consisting of nothing but type and dimensions comes back carrying data_schema, init_parameters and metrics it was never given. Comparing whole documents therefore never matched. Only the keys the author actually wrote are compared now, structurally, so key order and formatting are not changes either. This is what the spec means by repeated azd up creating no redundant versions. The eval group was then rejected with a request for a model that had been set. Evaluators disagree on what the judge model is called: built-ins declare deployment_name, and a custom rubric declares model. The builder sent only deployment_name, so the custom evaluator saw its required parameter missing. The judge model is now bound under whichever name the evaluator declares. Verified live: first deploy publishes the evaluator and creates the group, two redeploys report it unchanged and publish nothing, editing the rubric publishes the next version and recreates the group, and a further redeploy is a no-op again. --- .../internal/cmd/build.go | 11 +++- .../internal/cmd/build_test.go | 29 +++++++++ .../internal/cmd/reconciler.go | 54 +++++++++++----- .../internal/cmd/reconciler_test.go | 63 +++++++++++++++++++ 4 files changed, 141 insertions(+), 16 deletions(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build.go index 7e880001488..b9c585c417c 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build.go @@ -183,8 +183,15 @@ func planCriterion( return initSchema.Accepts(name) } - if evalModel != "" && accepts("deployment_name") { - plan.initParams["deployment_name"] = evalModel + // Evaluators disagree on what the judge model is called: built-ins declare + // deployment_name, custom rubrics declare model. Bind whichever the + // evaluator actually accepts rather than guessing one spelling. + if evalModel != "" { + for _, alias := range []string{"deployment_name", "model"} { + if accepts(alias) { + plan.initParams[alias] = evalModel + } + } } if ref.Threshold != nil && accepts("threshold") { plan.initParams["threshold"] = *ref.Threshold diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build_test.go index f4b9fa2104e..89785096c8f 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build_test.go @@ -256,6 +256,35 @@ func TestBuildResolvesConversationTurnExclusivity(t *testing.T) { require.NotContains(t, req.TestingCriteria[0].DataMapping, "messages") } +// Evaluators disagree on what the judge model is called. Built-ins declare +// deployment_name; a custom rubric declares model, and rejects the group with +// "requires model" if only deployment_name is sent. +func TestBuildBindsJudgeModelUnderTheDeclaredName(t *testing.T) { + schemas := map[string]*eval_api.EvaluatorSummary{ + "builtin.similarity": schema("builtin.similarity", + nil, []string{"query", "response"}, + []string{"deployment_name"}, []string{"deployment_name"}, "turn"), + "my-rubric": schema("my-rubric", + nil, []string{"query", "response"}, + []string{"model"}, []string{"model"}, "turn"), + } + group := groupWith([]evalcore.EvaluatorRef{ + {Name: "builtin.similarity"}, + {Name: "my-rubric"}, + }, &project.Options{EvalModel: "gpt-4.1-nano"}) + + req, err := buildEvalGroupRequest(group, schemas, map[string]bool{"query": true}) + require.NoError(t, err) + + builtin := req.TestingCriteria[0].InitializationParameters + require.Equal(t, "gpt-4.1-nano", builtin["deployment_name"]) + require.NotContains(t, builtin, "model") + + custom := req.TestingCriteria[1].InitializationParameters + require.Equal(t, "gpt-4.1-nano", custom["model"]) + require.NotContains(t, custom, "deployment_name") +} + // Without an agent target the sample bindings are unavailable, so every field // has to come from the dataset and the sample schema is not requested. func TestBuildWithoutTargetSourcesEverythingFromDataset(t *testing.T) { diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go index 965f67a4634..6f010e4ecd8 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go @@ -9,6 +9,7 @@ import ( "fmt" "os" "path/filepath" + "reflect" "azureaieval/internal/pkg/dataset_api" "azureaieval/internal/project" @@ -201,31 +202,56 @@ func (r *evalReconciler) EnsureEvalGroup( return created.ID, nil } -// sameDefinition compares only the definition body, ignoring server-assigned -// fields such as version and timestamps. +// sameDefinition reports whether the locally authored definition already +// matches what the service holds. +// +// Only the keys the candidate declares are compared. The service enriches a +// definition when it is created — a rubric of nothing but `type` and +// `dimensions` comes back carrying data_schema, init_parameters and metrics it +// was never given — so comparing whole documents never matches and every +// deploy publishes a redundant version. func sameDefinition(existing, candidate []byte) bool { - extract := func(raw []byte) string { + extract := func(raw []byte) map[string]json.RawMessage { var doc map[string]json.RawMessage if err := json.Unmarshal(raw, &doc); err != nil { - return "" + return nil } def, ok := doc["definition"] if !ok { - return "" + return nil } - var normalized any - if err := json.Unmarshal(def, &normalized); err != nil { - return "" + var fields map[string]json.RawMessage + if err := json.Unmarshal(def, &fields); err != nil { + return nil } - out, err := json.Marshal(normalized) - if err != nil { - return "" + return fields + } + + onService, authored := extract(existing), extract(candidate) + if onService == nil || authored == nil { + return false + } + + for key, want := range authored { + got, ok := onService[key] + if !ok || !equalJSON(got, want) { + return false } - return string(out) } + return true +} - a, b := extract(existing), extract(candidate) - return a != "" && a == b +// equalJSON compares two JSON values structurally, so key order and +// whitespace do not register as a change. +func equalJSON(a, b json.RawMessage) bool { + var left, right any + if err := json.Unmarshal(a, &left); err != nil { + return false + } + if err := json.Unmarshal(b, &right); err != nil { + return false + } + return reflect.DeepEqual(left, right) } func versionFromRaw(raw []byte, fallback string) string { diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler_test.go new file mode 100644 index 00000000000..8086fd96c43 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler_test.go @@ -0,0 +1,63 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +// The service enriches a definition when it stores it: a rubric of nothing but +// type and dimensions comes back carrying data_schema, init_parameters and +// metrics. Comparing whole documents therefore never matched, and every deploy +// published a redundant version. +func TestSameDefinitionIgnoresServerAddedFields(t *testing.T) { + authored := []byte(`{ + "name": "r", + "definition": { + "type": "rubric", + "dimensions": [{"id":"accuracy","description":"Correct.","weight":5}] + } + }`) + + onService := []byte(`{ + "name": "r", + "version": "2", + "created_at": "2026-07-28T00:00:00Z", + "definition": { + "type": "rubric", + "dimensions": [{"id":"accuracy","description":"Correct.","weight":5}], + "data_schema": {"type":"object","properties":{"query":{"type":"string"}}}, + "init_parameters": {"required":["model"],"properties":{"model":{"type":"string"}}}, + "metrics": {"score":{"type":"number"}} + } + }`) + + require.True(t, sameDefinition(onService, authored), + "server-added fields must not count as a change") +} + +// A real edit still registers. +func TestSameDefinitionDetectsAuthoredChange(t *testing.T) { + authored := []byte(`{"definition":{"type":"rubric","dimensions":[{"id":"a","weight":7}]}}`) + onService := []byte(`{"definition":{"type":"rubric","dimensions":[{"id":"a","weight":5}],"metrics":{}}}`) + + require.False(t, sameDefinition(onService, authored)) +} + +// Key order and whitespace are not changes. +func TestSameDefinitionIsStructural(t *testing.T) { + authored := []byte(`{"definition":{"type":"rubric","dimensions":[{"id":"a","weight":5}]}}`) + onService := []byte("{\"definition\":{\n \"dimensions\": [ {\"weight\":5,\"id\":\"a\"} ],\n \"type\":\"rubric\"\n}}") + + require.True(t, sameDefinition(onService, authored)) +} + +func TestSameDefinitionRejectsMalformed(t *testing.T) { + good := []byte(`{"definition":{"type":"rubric"}}`) + require.False(t, sameDefinition([]byte(`not json`), good)) + require.False(t, sameDefinition(good, []byte(`not json`))) + require.False(t, sameDefinition([]byte(`{"no":"definition"}`), good)) +} From 4d1f8271454226af585b68880411248e9eb6c8e5 Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 28 Jul 2026 07:23:22 +0300 Subject: [PATCH 023/320] fix(azure.ai.evaluations): honour --dataset when it names a registered dataset The flag is documented as taking a path or the name of a registered dataset, and means use this one instead of generating. It only suppressed generation when the value looked like a local path, so passing the name of an existing dataset still submitted a generation job and, since agent-seeded generation is currently broken server-side, failed the whole command. --evaluator already skipped unconditionally, so the two flags disagreed. Both the skip and the default-spec synthesis now key off whether the flag was supplied at all. This was the last thing standing between a generated config and the documented end-to-end flow. Verified live: init scaffolds a group referencing its own rubric, generate writes that rubric and merges the reference into the same file while preserving comments and ordering, azd up registers the dataset and evaluator and creates the group, and the run completes and scores against the generated rubric. --- .../internal/cmd/generate.go | 7 ++- .../internal/cmd/generate_defaults_test.go | 47 +++++++++++++++++++ 2 files changed, 52 insertions(+), 2 deletions(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_defaults_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go index d3b2f4ce94f..2f3be4b9d38 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go @@ -82,7 +82,10 @@ func newGenerateCommand() *cobra.Command { } } - if datasetFlag != "" && looksLikeLocalDataset(datasetFlag) { + // --dataset means use this one, whether it names a local file or a + // dataset already registered on the project. Either way there is + // nothing to generate, which is how --evaluator behaves too. + if datasetFlag != "" { fmt.Fprintf(out, "Using the supplied dataset; skipping data generation.\n") } else if cfg.Generate.Dataset != nil { ref, err := ec.generateDataset(ctx, cfg, instruction, baseDir, out, noWait) @@ -166,7 +169,7 @@ func resolveGenerateConfig( LocalDir: "./" + project.DefaultEvaluatorsDir, } } - if cfg.Generate.Dataset == nil && !looksLikeLocalDataset(datasetFlag) { + if cfg.Generate.Dataset == nil && datasetFlag == "" { cfg.Generate.Dataset = &project.DatasetSpec{ Name: cfg.Agent.Name + "-golden", Strategy: project.StrategySynthetic, diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_defaults_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_defaults_test.go new file mode 100644 index 00000000000..97f141a2f53 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_defaults_test.go @@ -0,0 +1,47 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" +) + +// --dataset is documented as taking a path or the name of a registered +// dataset, and means "use this one instead of generating". Only a local path +// used to suppress generation, so passing a registered name still submitted a +// generation job. +func TestGenerateScaffoldSkipsDatasetWhenSupplied(t *testing.T) { + cases := []struct { + name string + datasetFlag string + wantSpec bool + }{ + {"registered name", "prod-sample", false}, + {"relative path", "./data/golden.jsonl", false}, + {"bare filename", "golden.jsonl", false}, + {"not supplied", "", true}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + // A path that does not exist means flags alone drive the config. + cfg, err := resolveGenerateConfig( + filepath.Join(t.TempDir(), "absent.yaml"), + "my-agent", "gpt-4.1-nano", tc.datasetFlag, 0, 0) + require.NoError(t, err) + + if tc.wantSpec { + require.NotNil(t, cfg.Generate.Dataset, + "a dataset spec is needed when none was supplied") + } else { + require.Nil(t, cfg.Generate.Dataset, + "a supplied dataset must not produce a generation spec") + } + require.NotNil(t, cfg.Generate.Rubric, "the rubric spec is independent") + }) + } +} From 9e2797feb6555939446957962a02a33566337842 Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 28 Jul 2026 07:31:16 +0300 Subject: [PATCH 024/320] fix(azure.ai.evaluations): survive the eventually consistent version listing GET /datasets/{name}/versions returns nothing for a second or two after a version is created, even though the version itself reads back immediately. Measured: empty at 0s, populated at 2s. That undermines the version discovery added for dataset update, which reads the listing to decide what to increment from. An empty listing is ambiguous -- it means either a new dataset or a stale read -- so back-to-back create and update could still restart at 1.0 and take a 409. Rather than delaying every first upload to wait for the index, a conflict is now treated as the stale read it is: re-read the listing, which by then reflects reality, and retry once. The common path is unchanged. The live test asserted on the first listing response and was failing for the same reason. It now polls, and says why. Verified: create immediately followed by update produces 2.0 rather than a conflict, and the full live suite passes. --- .../internal/pkg/dataset_api/operations.go | 48 +++++++++++++++++-- .../tests/live/live_test.go | 12 +++-- 2 files changed, 52 insertions(+), 8 deletions(-) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/operations.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/operations.go index 501f558408f..2589981ed15 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/operations.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/operations.go @@ -8,6 +8,7 @@ import ( "context" "encoding/json" "encoding/xml" + "errors" "fmt" "io" "log" @@ -89,6 +90,12 @@ func (c *DatasetClient) CreateDataset( // TemporaryDataReferencesForExistingAsset as soon as 1.0 exists. Callers // almost always mean "the version after whatever is registered", which is what // this does. +// +// The version listing is eventually consistent — it returns nothing for a +// second or two after a version is created — so an empty listing cannot be +// trusted to mean the dataset is new. A conflict is therefore treated as a +// stale read and retried once against a re-read listing, rather than adding a +// delay to every first upload. func (c *DatasetClient) UploadNextVersion( ctx context.Context, name string, @@ -97,12 +104,43 @@ func (c *DatasetClient) UploadNextVersion( apiVersion string, ) (*Dataset, error) { if currentVersion == "" { - list, err := c.ListDatasetVersions(ctx, name, apiVersion) - if err == nil && list != nil && len(list.Value) > 0 { - currentVersion = LatestVersion(list.Value) - } + currentVersion = c.latestRegisteredVersion(ctx, name, apiVersion) + } + + ds, err := c.UploadNewVersion(ctx, name, currentVersion, localDir, apiVersion) + if err == nil || !isVersionConflict(err) { + return ds, err + } + + latest := c.latestRegisteredVersion(ctx, name, apiVersion) + if latest == "" || latest == currentVersion { + return nil, err + } + return c.UploadNewVersion(ctx, name, latest, localDir, apiVersion) +} + +// latestRegisteredVersion returns the newest registered version, or empty when +// the dataset is unknown or the listing has not caught up yet. +func (c *DatasetClient) latestRegisteredVersion( + ctx context.Context, + name string, + apiVersion string, +) string { + list, err := c.ListDatasetVersions(ctx, name, apiVersion) + if err != nil || list == nil || len(list.Value) == 0 { + return "" + } + return LatestVersion(list.Value) +} + +// isVersionConflict reports whether the service refused the upload because the +// target version already exists. +func isVersionConflict(err error) bool { + var respErr *azcore.ResponseError + if !errors.As(err, &respErr) { + return false } - return c.UploadNewVersion(ctx, name, currentVersion, localDir, apiVersion) + return respErr.StatusCode == http.StatusConflict } // UploadNewVersion reads the first JSONL file from localDir, computes the next diff --git a/cli/azd/extensions/azure.ai.evaluations/tests/live/live_test.go b/cli/azd/extensions/azure.ai.evaluations/tests/live/live_test.go index 9174ff294f4..9a4a209bffb 100644 --- a/cli/azd/extensions/azure.ai.evaluations/tests/live/live_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/tests/live/live_test.go @@ -168,9 +168,15 @@ func TestLiveDatasetLifecycle(t *testing.T) { require.NoError(t, err, "reading the dataset back") t.Logf("dataset uri: %q (empty means a credential call is required)", fetched.ResolvedBlobURI()) - versions, err := env.datasetClient.ListDatasetVersions(ctx, name, projectAPIVersion) - require.NoError(t, err, "listing dataset versions") - require.NotEmpty(t, versions.Value) + // The version listing is eventually consistent: it returns nothing for a + // second or two after a version is created, even though the version itself + // reads back fine. Poll rather than asserting on the first response. + var versions *dataset_api.DatasetList + require.Eventually(t, func() bool { + var err error + versions, err = env.datasetClient.ListDatasetVersions(ctx, name, projectAPIVersion) + return err == nil && versions != nil && len(versions.Value) > 0 + }, 30*time.Second, 2*time.Second, "the version listing never caught up") require.Equal(t, first.Version, dataset_api.LatestVersion(versions.Value)) // A second upload must advance the version, not conflict. From 2606b685f848f619c8043a6b73ddea6f9610d98e Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 28 Jul 2026 07:40:20 +0300 Subject: [PATCH 025/320] build(azure.ai.evaluations): add the CI scripts the release pipeline requires The shared extension build template invokes ci-build.ps1 and ci-test.ps1 from the extension directory. Neither existed, so the release pipeline added alongside this extension would have failed on its first run. Both are modelled on the agents extension with two deliberate differences. ci-build.ps1 reads version.txt from the extension directory rather than its parent, where no such file exists, so the default works when the pipeline is not supplying -Version. It accepts -BuildRecordMode, which the template always passes, but builds nothing extra: this extension has no record/playback mode and no pipeline step consumes a record binary. ci-test.ps1 passes --junitfile explicitly. The pipeline publishes **/junitTestReport.xml from the extension directory, and the extension template does not set GOTESTSUM_JUNITFILE the way the CLI build does, so without this no test results would surface in the build. Verified locally with gotestsum installed: 97 tests across 7 suites reported. Also adds the README and CHANGELOG that 17 of the 21 extensions ship. The README documents the deployed shape, the command surface, the rubric weight constraint, and how to run the live tests. --- .../azure.ai.evaluations/CHANGELOG.md | 27 +++ .../extensions/azure.ai.evaluations/README.md | 158 ++++++++++++++++++ .../azure.ai.evaluations/ci-build.ps1 | 114 +++++++++++++ .../azure.ai.evaluations/ci-test.ps1 | 36 ++++ 4 files changed, 335 insertions(+) create mode 100644 cli/azd/extensions/azure.ai.evaluations/CHANGELOG.md create mode 100644 cli/azd/extensions/azure.ai.evaluations/README.md create mode 100644 cli/azd/extensions/azure.ai.evaluations/ci-build.ps1 create mode 100644 cli/azd/extensions/azure.ai.evaluations/ci-test.ps1 diff --git a/cli/azd/extensions/azure.ai.evaluations/CHANGELOG.md b/cli/azd/extensions/azure.ai.evaluations/CHANGELOG.md new file mode 100644 index 00000000000..67a8c1d121e --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/CHANGELOG.md @@ -0,0 +1,27 @@ +# Release History + +## 1.0.0-beta.1 (Unreleased) + +### Features Added + +- Initial release of the Foundry evaluations extension, `azd ai eval`. +- `init` scaffolds `evals/eval_generate.yaml` and `evals/azure.yaml` next to an + agent, making no service calls. +- `generate` synthesizes a rubric and dataset from the agent's context, writes + them under `evals/`, and merges `source:` references into the deployment spec + while preserving comments, ordering and neighbouring entries. +- `run` creates the eval group when it does not exist, starts a run, and + summarizes the result. +- `azure.ai.eval` service-target provider deploys datasets, evaluators and eval + groups during `azd up`, reconciling them in dependency order. +- Change detection so a repeated `azd up` publishes no redundant versions: + datasets are fingerprinted locally, evaluator definitions are compared on the + keys the author wrote, and eval groups are recreated only when their own + declaration changes. +- Atomic commands for every operation: `dataset`, `evaluator`, `run` and + `results` subcommands, all supporting `-o json` and `--no-prompt`. +- Testing criteria are shaped from each evaluator's published contract, so + evaluators requiring inputs beyond the agent shape — `ground_truth`, + `context`, `instruction_id_list` — work by binding them to dataset columns. + A required column the dataset does not carry is reported before the request + is sent, naming the column. diff --git a/cli/azd/extensions/azure.ai.evaluations/README.md b/cli/azd/extensions/azure.ai.evaluations/README.md new file mode 100644 index 00000000000..2a5c80e62fe --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/README.md @@ -0,0 +1,158 @@ +# Azure Developer CLI (azd) Evaluations Extension + +Define Foundry evaluations alongside your agent in `azure.yaml`, deploy them +with `azd up`, and run them from the terminal. + +```bash +azd ai eval init # scaffold evals/ next to your agent +azd ai eval generate # synthesize a rubric and dataset from the agent +azd up # register datasets and evaluators, create the eval group +azd ai eval run # run the evaluation and summarize the results +``` + +## What gets deployed + +Eval resources are one service entry in `azure.yaml`, normally a `$ref` to a +file under `evals/`: + +```yaml +# azure.yaml +services: + ai-project: + host: azure.ai.project + evals: + host: azure.ai.eval + uses: [ai-project] + $ref: ./evals/azure.yaml +``` + +```yaml +# evals/azure.yaml +datasets: + - name: support-golden + source: ./datasets/support-golden.jsonl + +evaluators: + - name: support-quality + source: ./evaluators/support-quality.json + +evalGroups: + - name: support-quality + dataset: support-golden + evaluators: + - builtin.task_adherence + - support-quality + target: + type: agent + name: support-agent + options: + eval_model: gpt-4.1-nano +``` + +`azd up` reconciles **datasets → evaluators → eval groups**, in that order, +because a group references the versions the first two resolve to. + +Relative paths inside a `$ref`'d file resolve against **that file's** +directory, so `./datasets/x.jsonl` above means `evals/datasets/x.jsonl`. + +### Repeated deploys do not create redundant versions + +Datasets are fingerprinted locally, because the dataset API exposes no content +hash and comparing against the service would mean downloading the blob on every +deploy. Evaluator definitions are compared against the service, but only on the +keys you authored — the service adds `data_schema`, `init_parameters` and +`metrics` of its own. + +Eval groups are immutable, so a change to a group's evaluators, target or +options creates a new group and a new id. The id is cached in the azd +environment so repeat runs stay comparable. + +## Commands + +| Group | Commands | +|---|---| +| `azd ai eval` | `init` · `generate` · `run` | +| `azd ai eval dataset` | `create` · `list` · `show` · `update` · `delete` | +| `azd ai eval evaluator` | `upload` · `list` · `show` · `update` · `delete` · `builtins` | +| `azd ai eval run` | `start` · `list` · `show` · `cancel` | +| `azd ai eval results` | `show` · `export` | + +`create` and `update` both publish a new immutable version; the server +auto-increments and nothing mutates in place. + +Every command supports `-o json` and `--no-prompt`, so the whole surface is +usable from CI. + +## Evaluators + +Built-ins need no declaration — reference them as `builtin.` and list +them with `azd ai eval evaluator builtins`. + +Evaluators do not share an input contract, so the CLI reads each one's +published contract and shapes the request to match. An evaluator needing an +input your dataset does not carry is reported before the request is sent, with +the column named, rather than as a service-side rejection. + +A custom rubric is a JSON list of weighted dimensions: + +```json +{ + "dimensions": [ + { "id": "accuracy", "description": "The answer is factually correct.", "weight": 5 }, + { "id": "tone", "description": "The answer is polite and professional.", "weight": 2 } + ] +} +``` + +`weight` is an **integer from 1 to 10**. Weights do not need to sum to +anything. + +## Choosing a project + +The project endpoint is resolved in this order: + +1. `--project-endpoint` +2. `FOUNDRY_PROJECT_ENDPOINT` in the active azd environment +3. the host environment variable of the same name + +## Local development + +### Prerequisites + +- Go (the version in `go.mod`; `GOTOOLCHAIN=auto` fetches it) +- [azd](https://aka.ms/azd) and the extension developer kit: + `azd ext install microsoft.azd.extensions` + +### Build, test, install + +```bash +azd x build # compile and install into the local azd +azd x pack # package the artifacts +azd x publish # register in the local extension source +azd ext install azure.ai.evaluations --source local +``` + +```bash +go test ./internal/... # unit tests +``` + +### Live integration tests + +These talk to a real Foundry project, so they are excluded from the default +build by the `live` tag and additionally gated on an environment variable: + +```bash +export AZURE_AI_EVAL_E2E_LIVE=1 +export FOUNDRY_PROJECT_ENDPOINT=https://.services.ai.azure.com/api/projects/ +export AZURE_AI_EVAL_MODEL=gpt-4.1-nano # optional judge model +export AZURE_AI_EVAL_AGENT= # optional, enables the run phase + +go test -tags live ./internal/cmd/ ./tests/live/ +``` + +They clean up every resource they create. + +### Debug logging + +Request tracing is off by default. `--debug`, or `AZD_EXT_DEBUG=true`, writes +it to a dated log file rather than the terminal. diff --git a/cli/azd/extensions/azure.ai.evaluations/ci-build.ps1 b/cli/azd/extensions/azure.ai.evaluations/ci-build.ps1 new file mode 100644 index 00000000000..403bc23b08d --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/ci-build.ps1 @@ -0,0 +1,114 @@ +param( + [string] $Version = (Get-Content "$PSScriptRoot/version.txt"), + [string] $SourceVersion = (git rev-parse HEAD), + [switch] $CodeCoverageEnabled, + # Accepted because the shared CI template always passes it. This extension + # has no record/playback mode, so there is no second binary to produce. + [switch] $BuildRecordMode, + [string] $MSYS2Shell, # path to msys2_shell.cmd + [string] $OutputFileName +) +$PSNativeCommandArgumentPassing = 'Legacy' + +# Remove any previously built binaries. +go clean + +if ($LASTEXITCODE) { + Write-Host "Error running go clean" + exit $LASTEXITCODE +} + +# Run `go help build` for detail on these flags. +$buildFlags = @( + # Remove file system paths from the binary. Recorded file names become a + # module path@version, or a plain import path for the standard library. + "-trimpath", + + # Position Independent Executable, for memory-corruption hardening across + # platforms. On Windows this enables ASLR and sets DYNAMICBASE and + # HIGH-ENTROPY-VA in the PE header. + "-buildmode=pie" +) + +if ($CodeCoverageEnabled) { + $buildFlags += "-cover" +} + +# cfi: Control Flow Integrity, cfg: Control Flow Guard, +# osusergo: use the pure Go user lookup. +$tagsFlag = "-tags=cfi,cfg,osusergo" + +# -s: omit the symbol table, -w: omit DWARF, -X: set a variable at link time. +$ldFlag = "-ldflags=-s -w " + + "-X 'azureaieval/internal/version.Version=$Version' " + + "-X 'azureaieval/internal/version.Commit=$SourceVersion' " + + "-X 'azureaieval/internal/version.BuildDate=$(Get-Date -Format o)' " + +if ($IsWindows) { + Write-Host "Building for Windows" +} +elseif ($IsLinux) { + Write-Host "Building for linux" + + # Disable cgo for the x64 Linux build. This also links statically, which + # widens compatibility with older Linux distributions. + if ($env:GOARCH -ne "arm64") { + $env:CGO_ENABLED = "0" + } +} +elseif ($IsMacOS) { + Write-Host "Building for macOS" +} + +$outputFlag = "-o=$OutputFileName" + +$buildFlags += @( + $tagsFlag, + $ldFlag, + $outputFlag +) + +function PrintFlags() { + param( + [string] $flags + ) + + # Format the flags so they can be pasted straight into pwsh. + $i = 0 + foreach ($buildFlag in $buildFlags) { + # Quote values so characters such as ',' survive a repaste. Not needed + # for the direct invocation below. + $argWithValue = $buildFlag.Split('=', 2) + if ($argWithValue.Length -eq 2 -and !$argWithValue[1].StartsWith("`"")) { + $buildFlag = "$($argWithValue[0])=`"$($argWithValue[1])`"" + } + + if ($i -eq $buildFlags.Length - 1) { + Write-Host " $buildFlag" + } + else { + Write-Host " $buildFlag ``" + } + $i++ + } +} + +$oldGOEXPERIMENT = $env:GOEXPERIMENT +# Opt into per-iteration loop variables, which is what most readers expect and +# what the Go team intends to make the default. +$env:GOEXPERIMENT = "loopvar" + +try { + Write-Host "Running: go build ``" + PrintFlags -flags $buildFlags + go build @buildFlags + if ($LASTEXITCODE) { + Write-Host "Error running go build" + exit $LASTEXITCODE + } + + Write-Host "go build succeeded" +} +finally { + $env:GOEXPERIMENT = $oldGOEXPERIMENT +} diff --git a/cli/azd/extensions/azure.ai.evaluations/ci-test.ps1 b/cli/azd/extensions/azure.ai.evaluations/ci-test.ps1 new file mode 100644 index 00000000000..314095e75b3 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/ci-test.ps1 @@ -0,0 +1,36 @@ +# Runs the unit tests and writes a JUnit report. +# +# The pipeline publishes **/junitTestReport.xml from the extension directory, +# so the report has to be written under that name for results to show up in the +# build. gotestsum produces it; the go test fallback does not, so the fallback +# only runs when gotestsum is unavailable. +# +# The live integration tests are excluded: they carry the `live` build tag, so +# an untagged run does not compile them, and they additionally require +# AZURE_AI_EVAL_E2E_LIVE and a project endpoint. + +$gopath = go env GOPATH +$gotestsumBinary = "gotestsum" +if ($IsWindows) { + $gotestsumBinary += ".exe" +} +$gotestsum = Join-Path $gopath "bin" $gotestsumBinary + +Write-Host "Running unit tests..." + +if (Test-Path $gotestsum) { + & $gotestsum --format testname --junitfile junitTestReport.xml -- ./... -count=1 +} else { + Write-Host "gotestsum not found; falling back to go test (no JUnit report)." -ForegroundColor Yellow + go test ./... -v -count=1 +} + +if ($LASTEXITCODE -ne 0) { + Write-Host "" + Write-Host "Tests failed with exit code: $LASTEXITCODE" -ForegroundColor Red + exit $LASTEXITCODE +} + +Write-Host "" +Write-Host "All tests passed!" -ForegroundColor Green +exit 0 From 288927f6b178b37cce3bcde74ce7dfb79877fe0f Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 28 Jul 2026 07:49:41 +0300 Subject: [PATCH 026/320] fix(azure.ai.evaluations): make the manifest and the binary agree The manifest declared two capabilities the extension did not back. metadata was declared but the command was never registered, so azd could not discover the command tree: azd ai eval metadata failed with unknown command while the same call against a peer extension returned its full tree. azd uses this for discovery, so the declaration was actively misleading. The command is now registered and reports nine commands. lifecycle-events was declared but no event handlers exist. The SDK only starts its event manager when handlers are registered, so the capability was an unused permission rather than a broken promise. It is removed; the listen command is still invoked because the service-target-provider capability triggers it, which a deploy after the change confirms. Adds tests over the manifest so neither can drift again: every declared capability must be backed by the command that implements it, the declared provider name must match the host the code registers, and version.txt must agree with the manifest version, which until now was only a comment asking for it. --- .../azure.ai.evaluations/extension.yaml | 1 - .../internal/cmd/manifest_test.go | 101 ++++++++++++++++++ .../azure.ai.evaluations/internal/cmd/root.go | 7 ++ 3 files changed, 108 insertions(+), 1 deletion(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/manifest_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/extension.yaml b/cli/azd/extensions/azure.ai.evaluations/extension.yaml index 2b8e8bbb58f..c73a8ad1db4 100644 --- a/cli/azd/extensions/azure.ai.evaluations/extension.yaml +++ b/cli/azd/extensions/azure.ai.evaluations/extension.yaml @@ -13,7 +13,6 @@ dependencies: language: go capabilities: - custom-commands - - lifecycle-events - service-target-provider - metadata providers: diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/manifest_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/manifest_test.go new file mode 100644 index 00000000000..faf104043b9 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/manifest_test.go @@ -0,0 +1,101 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/require" + "go.yaml.in/yaml/v3" +) + +// extensionManifest is the subset of extension.yaml this test asserts on. +type extensionManifest struct { + ID string `yaml:"id"` + Version string `yaml:"version"` + Capabilities []string `yaml:"capabilities"` + Providers []struct { + Name string `yaml:"name"` + Type string `yaml:"type"` + } `yaml:"providers"` +} + +func loadManifest(t *testing.T) extensionManifest { + t.Helper() + raw, err := os.ReadFile(filepath.Join("..", "..", "extension.yaml")) + require.NoError(t, err, "reading extension.yaml") + + var manifest extensionManifest + require.NoError(t, yaml.Unmarshal(raw, &manifest)) + return manifest +} + +// A declared capability azd cannot reach is worse than an undeclared one: azd +// invokes `metadata` to discover the command tree, and it was declared without +// the command being registered, so discovery failed with "unknown command". +func TestDeclaredCapabilitiesAreImplemented(t *testing.T) { + manifest := loadManifest(t) + root := NewRootCommand() + + hasCommand := func(name string) bool { + for _, sub := range root.Commands() { + if sub.Name() == name { + return true + } + } + return false + } + + for _, capability := range manifest.Capabilities { + switch capability { + case "metadata": + require.True(t, hasCommand("metadata"), + "the metadata capability requires a metadata command") + case "service-target-provider": + require.True(t, hasCommand("listen"), + "a service-target provider is registered through the listen command") + require.NotEmpty(t, manifest.Providers, + "the manifest must name the provider it registers") + case "custom-commands": + require.NotEmpty(t, root.Commands()) + case "lifecycle-events": + // The SDK only starts the event manager when handlers are + // registered, so declaring this without any is an unused + // permission. Nothing here registers handlers today. + t.Fatalf("lifecycle-events is declared but no event handlers are registered") + } + } +} + +// The provider name in the manifest is what azd matches a service's `host` +// against, so a mismatch silently means the provider is never invoked. +func TestManifestProviderMatchesHostConstant(t *testing.T) { + manifest := loadManifest(t) + require.NotEmpty(t, manifest.Providers) + + names := make([]string, 0, len(manifest.Providers)) + for _, p := range manifest.Providers { + names = append(names, p.Name) + } + require.Contains(t, names, "azure.ai.eval", + "the manifest must declare the host the provider registers for") +} + +// extension.yaml carries a note asking that version.txt be kept in sync. The +// build stamps the binary from version.txt while the registry reads +// extension.yaml, so a drift ships a binary that misreports its own version. +func TestManifestVersionMatchesVersionFile(t *testing.T) { + manifest := loadManifest(t) + + raw, err := os.ReadFile(filepath.Join("..", "..", "version.txt")) + require.NoError(t, err, "reading version.txt") + + require.Equal(t, + strings.TrimSpace(string(raw)), + strings.TrimSpace(manifest.Version), + "version.txt and extension.yaml must agree") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/root.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/root.go index a763a54b924..40724c5a0a7 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/root.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/root.go @@ -54,5 +54,12 @@ func NewRootCommand() *cobra.Command { newListenCommand(), ) + // The manifest declares the `metadata` capability, which azd uses to + // discover this extension's command tree. Without the command registered, + // that discovery fails with "unknown command". + rootCmd.AddCommand(azdext.NewMetadataCommand("1.0", "azure.ai.evaluations", func() *cobra.Command { + return rootCmd + })) + return rootCmd } From 909ce14430ae5f19d613156b2cfa0b0b0d8d7aa2 Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 28 Jul 2026 08:03:18 +0300 Subject: [PATCH 027/320] feat(azure.ai.evaluations): detect artifact drift and read the instruction from a file Auditing every flag and API sequence the spec documents against the running extension turned up two gaps. The spec describes a drift check that was never implemented. It matters because of how change detection works: when local content is unchanged, the version recorded at the last deploy is reused, so a version published outside the repo would be silently ignored and the eval group pinned to older data. A deploy now fails when the service holds a newer version than the recorded one, naming both versions. An explicit version: on the declaration skips the check, because that is the author stating which version they want. This was added after testing the remedy the error message suggests and finding it did not work -- the message now describes something that does. --gen-instruction-file was documented but absent. A useful generation instruction is usually longer than fits on a command line, and putting it in a file makes it reviewable with the rest of the config. Verified live: publishing a version out-of-band fails the next deploy, and pinning that version lets it through. --- .../internal/cmd/generate.go | 52 +++++++++++++++---- .../internal/cmd/instruction_test.go | 50 ++++++++++++++++++ .../internal/cmd/reconciler.go | 48 ++++++++++++++++- .../internal/pkg/dataset_api/list.go | 12 +++++ .../internal/pkg/dataset_api/version_test.go | 41 +++++++++++++++ 5 files changed, 191 insertions(+), 12 deletions(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/instruction_test.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/version_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go index 2f3be4b9d38..0a574635044 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go @@ -30,17 +30,18 @@ var generatePollBudget = eval_api.PollerOptions{ func newGenerateCommand() *cobra.Command { var ( - configPath string - deployPath string - target string - instruction string - datasetFlag string - evaluators []string - maxSamples int - traceDays int - evalModel string - noWait bool - endpointFlg string + configPath string + deployPath string + target string + instruction string + instructionFile string + datasetFlag string + evaluators []string + maxSamples int + traceDays int + evalModel string + noWait bool + endpointFlg string ) cmd := &cobra.Command{ @@ -50,6 +51,11 @@ func newGenerateCommand() *cobra.Command { ctx := cmd.Context() out := cmd.OutOrStdout() + instruction, err := resolveInstruction(instruction, instructionFile) + if err != nil { + return err + } + cfg, err := resolveGenerateConfig( configPath, target, evalModel, datasetFlag, maxSamples, traceDays, ) @@ -126,6 +132,9 @@ func newGenerateCommand() *cobra.Command { cmd.Flags().StringVar(&target, "target", "", "Agent whose context seeds generation.") cmd.Flags().StringVar(&instruction, "gen-instruction", "", "What the agent does and what to test.") + cmd.Flags().StringVar(&instructionFile, "gen-instruction-file", "", + "Read the generation instruction from this file. Mutually exclusive with --gen-instruction.") + cmd.MarkFlagsMutuallyExclusive("gen-instruction", "gen-instruction-file") cmd.Flags().StringVar(&datasetFlag, "dataset", "", "Use this dataset instead of generating one.") cmd.Flags().StringArrayVar(&evaluators, "evaluator", nil, @@ -140,6 +149,27 @@ func newGenerateCommand() *cobra.Command { return cmd } +// resolveInstruction returns the generation instruction, reading it from a +// file when one is named. +// +// A useful instruction describes the agent and what to test, which is often +// more than fits comfortably on a command line, so it can live in a file that +// is reviewable alongside the rest of the config. +func resolveInstruction(inline, path string) (string, error) { + if path == "" { + return inline, nil + } + raw, err := os.ReadFile(path) + if err != nil { + return "", fmt.Errorf("reading --gen-instruction-file %q: %w", path, err) + } + text := strings.TrimSpace(string(raw)) + if text == "" { + return "", fmt.Errorf("--gen-instruction-file %q is empty", path) + } + return text, nil +} + // resolveGenerateConfig loads the spec when present, then layers flags on top. // A missing file is not an error: flags alone are sufficient. func resolveGenerateConfig( diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/instruction_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/instruction_test.go new file mode 100644 index 00000000000..4063a6ab1b4 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/instruction_test.go @@ -0,0 +1,50 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" +) + +// A useful generation instruction is often longer than fits on a command +// line, so it can come from a file instead. +func TestResolveInstructionReadsFile(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "instruction.md") + require.NoError(t, os.WriteFile(path, + []byte(" A customer support agent answering billing questions.\n\n"), 0o600)) + + got, err := resolveInstruction("", path) + require.NoError(t, err) + require.Equal(t, "A customer support agent answering billing questions.", got, + "surrounding whitespace should be trimmed") +} + +func TestResolveInstructionPrefersInlineWhenNoFile(t *testing.T) { + got, err := resolveInstruction("inline text", "") + require.NoError(t, err) + require.Equal(t, "inline text", got) + + got, err = resolveInstruction("", "") + require.NoError(t, err) + require.Empty(t, got) +} + +// An unreadable or empty file is reported rather than silently generating from +// no instruction at all. +func TestResolveInstructionRejectsUnusableFile(t *testing.T) { + _, err := resolveInstruction("", filepath.Join(t.TempDir(), "absent.md")) + require.Error(t, err) + require.Contains(t, err.Error(), "gen-instruction-file") + + empty := filepath.Join(t.TempDir(), "empty.md") + require.NoError(t, os.WriteFile(empty, []byte(" \n"), 0o600)) + _, err = resolveInstruction("", empty) + require.Error(t, err) + require.Contains(t, err.Error(), "empty") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go index 6f010e4ecd8..a00ce08711b 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go @@ -73,8 +73,17 @@ func (r *evalReconciler) EnsureDataset( key := project.FingerprintKey("dataset", decl.Name) if prior := r.ec.getEnvValue(ctx, key); prior == digest { - // Unchanged since the last deploy; reuse the recorded version. + // Unchanged since the last deploy; reuse the recorded version, but only + // after confirming nobody published a newer one outside the repo. An + // explicit `version:` is the author saying which version they want, so + // it settles the question and the check does not apply. if version := r.ec.getEnvValue(ctx, versionKey("dataset", decl.Name)); version != "" { + if decl.Version != "" { + return decl.Version, false, nil + } + if err := r.checkDatasetDrift(ctx, decl.Name, version); err != nil { + return "", false, err + } return version, false, nil } } @@ -101,6 +110,43 @@ func (r *evalReconciler) EnsureDataset( return ds.Version, true, nil } +// checkDatasetDrift fails when the service holds a newer version than the one +// recorded at the last deploy. +// +// Local content being unchanged is not enough to reuse the recorded version: +// someone may have published a newer one outside the repo, and silently +// pinning the eval group to the older version would quietly evaluate against +// stale data. Publishing is not destructive — versions are immutable — so the +// remedy is to sync, not to overwrite. +func (r *evalReconciler) checkDatasetDrift( + ctx context.Context, + name, recorded string, +) error { + latest := r.latestDatasetVersion(ctx, name) + if latest == "" || latest == recorded { + return nil + } + if !dataset_api.VersionGreater(latest, recorded) { + return nil + } + return fmt.Errorf( + "dataset %q is at version %s on the project but %s was recorded at the last deploy; "+ + "someone published a version outside this repo. "+ + "Pin it with `version: %s` on the dataset, or pull the newer content locally, "+ + "then deploy again", + name, latest, recorded, latest) +} + +// latestDatasetVersion reports the newest registered version, or empty when the +// dataset is unknown or the listing has not caught up. +func (r *evalReconciler) latestDatasetVersion(ctx context.Context, name string) string { + list, err := r.ec.datasetClient.ListDatasetVersions(ctx, name, ProjectEndpointAPIVersion) + if err != nil || list == nil || len(list.Value) == 0 { + return "" + } + return dataset_api.LatestVersion(list.Value) +} + // EnsureEvaluator publishes a new version when the local definition differs // from what the service holds. Evaluator definitions come back inline, so this // compares content directly rather than relying on a cached digest. diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/list.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/list.go index f81f372131d..19b548c3e05 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/list.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/list.go @@ -74,6 +74,18 @@ func VersionOrder(version string) float64 { return -1 } +// VersionGreater reports whether a is a strictly newer version than b. +// +// Both must be orderable; when either is not, the answer is false so an +// unparseable version never triggers a drift failure on its own. +func VersionGreater(a, b string) bool { + orderA, orderB := VersionOrder(a), VersionOrder(b) + if orderA < 0 || orderB < 0 { + return false + } + return orderA > orderB +} + // LatestVersion returns the highest version in the list, falling back to the // last entry when none of the versions can be ordered. func LatestVersion(datasets []Dataset) string { diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/version_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/version_test.go new file mode 100644 index 00000000000..de822def3cd --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/version_test.go @@ -0,0 +1,41 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package dataset_api + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +// Drift detection compares the version on the service with the one recorded at +// the last deploy, so the ordering has to be numeric rather than lexical: +// "10.0" is newer than "9.0" even though it sorts earlier as a string. +func TestVersionGreater(t *testing.T) { + cases := []struct { + a, b string + want bool + }{ + {"2.0", "1.0", true}, + {"1.0", "2.0", false}, + {"1.0", "1.0", false}, + {"10.0", "9.0", true}, + {"9.0", "10.0", false}, + {"v3", "v2", true}, + } + + for _, tc := range cases { + require.Equal(t, tc.want, VersionGreater(tc.a, tc.b), + "VersionGreater(%q, %q)", tc.a, tc.b) + } +} + +// An unorderable version must never trigger a drift failure on its own: the +// deploy would be blocked with no way for the author to reason about it. +func TestVersionGreaterIgnoresUnorderable(t *testing.T) { + require.False(t, VersionGreater("draft", "1.0")) + require.False(t, VersionGreater("1.0", "draft")) + require.False(t, VersionGreater("", "1.0")) + require.False(t, VersionGreater("1.0", "")) +} From 800834e8fdf584b36f44107e236435bfe4cb6743 Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 28 Jul 2026 08:12:21 +0300 Subject: [PATCH 028/320] fix(azure.ai.evaluations): make --eval-id work and catch unregistered edits M1 exits on all the spec examples running end to end, so I ran them verbatim. Two did not. --eval-id could never work. It is meant to run an existing group ignoring the config, and appears in both the CI/CD example and the recovery advice, but a run needs a target and a dataset and an eval group carries neither: the group holds only its testing criteria, and the dataset travels on the run. Every --eval-id invocation failed asking for a target. The pairing survives in the group's previous run, so re-running a group now repeats what it last ran, and a group that has never run says so and points at the config-based path. The failure-and-recovery example promised an error that did not exist. A run sends a local dataset inline, so unregistered local edits were evaluated silently and the results could not be traced to any dataset version. That now fails with the message the spec documents, once a deploy has recorded a fingerprint to compare against. Before that there is nothing to have drifted from, and running is how a group first comes into existence. Verified live: the CI/CD example returns JSON with a run id, and with unregistered edits the config-based run fails while --eval-id succeeds, which is exactly the recovery the spec describes. --- .../azure.ai.evaluations/internal/cmd/run.go | 88 ++++++++++++++++++- 1 file changed, 87 insertions(+), 1 deletion(-) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go index 66cc843e103..db25185ecde 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go @@ -82,6 +82,10 @@ func buildRunCommand(use, short string) *cobra.Command { return err } + if err := ec.checkDatasetRegistered(ctx, cfg, group, configPath); err != nil { + return err + } + evalID, err = ec.resolveEvalGroupID( ctx, group, configPath, resolveLevel(level, group), out, isJSON(cmd)) if err != nil { @@ -89,7 +93,14 @@ func buildRunCommand(use, short string) *cobra.Command { } } - dataSource, err := buildRunDataSource(group, configPath, maxSamples) + // With --eval-id there is no config to read, so the pairing of + // target and dataset comes from the group's previous run. + var dataSource *eval_api.EvalRunDataSource + if group == nil { + dataSource, err = ec.reuseDataSourceFromLastRun(ctx, evalID) + } else { + dataSource, err = buildRunDataSource(group, configPath, maxSamples) + } if err != nil { return err } @@ -223,6 +234,81 @@ func (ec *evalContext) resolveEvalGroupID( return created.ID, nil } +// checkDatasetRegistered fails when the group's local dataset has edits that +// were never deployed. +// +// A run sends a local dataset inline, so without this the run would evaluate +// content that no registered version corresponds to: the results are attributed +// to the eval group but cannot be traced back to a dataset version, which +// makes them impossible to reproduce or compare. +// +// The check only applies once a deploy has recorded a fingerprint. Before that +// there is nothing to have drifted from, and running is how a group first comes +// into existence. +func (ec *evalContext) checkDatasetRegistered( + ctx context.Context, + cfg *project.EvalConfig, + group *project.EvalGroup, + configPath string, +) error { + localPath := localDatasetPath(configPath, group) + if localPath == "" { + return nil + } + + decl, ok := cfg.Dataset(group.Dataset) + if !ok { + return nil + } + + recorded := ec.getEnvValue(ctx, project.FingerprintKey("dataset", decl.Name)) + if recorded == "" { + return nil + } + + digest, err := project.Fingerprint(localPath) + if err != nil { + // Reading the file is the run's problem to report, not this check's. + return nil + } + if digest == recorded { + return nil + } + + return fmt.Errorf( + "dataset %q has local edits that are not registered.\n"+ + " Run `azd up` to register them, or `--eval-id ` to run against "+ + "an existing eval group", + decl.Name) +} + +// reuseDataSourceFromLastRun rebuilds a run's data source from the group's most +// recent run. +// +// `--eval-id` deliberately ignores the config, but a run still needs a target +// and a dataset, and an eval group carries neither: the group holds only its +// testing criteria, and the dataset travels on the run. The previous run is the +// only place that pairing survives, so re-running a group means repeating what +// it last ran. +func (ec *evalContext) reuseDataSourceFromLastRun( + ctx context.Context, + evalID string, +) (*eval_api.EvalRunDataSource, error) { + list, err := ec.evalClient.ListOpenAIEvalRuns(ctx, evalID, 1) + if err != nil { + return nil, fmt.Errorf("reading previous runs of eval group %s: %w", evalID, err) + } + if list == nil || len(list.Data) == 0 || list.Data[0].DataSource == nil { + return nil, fmt.Errorf( + "eval group %s has no previous run to repeat, so there is no target or dataset "+ + "to reuse.\n"+ + " Run it from the config once with `azd ai eval run`, or pass a config that "+ + "declares the group", + evalID) + } + return list.Data[0].DataSource, nil +} + // buildRunDataSource binds the dataset to the run. The eval group carries no // dataset today, so it is supplied here. func buildRunDataSource( From d09111e06c5a6828352076b3063082734468c2e6 Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 28 Jul 2026 08:19:22 +0300 Subject: [PATCH 029/320] fix(azure.ai.evaluations): rebase a supplied dataset path onto the deploy spec init --dataset ./tests/golden.jsonl wrote that path into evals/azure.yaml unchanged, but source: is resolved relative to the file it appears in, so the deploy looked for evals/tests/golden.jsonl and failed on a file the user had just pointed at. This is the spec's bring-your-own-data example exactly as written, so that example could never have worked. The path is now rebased onto the output directory, with forward slashes so the config reads the same on every platform, and absolute paths left alone. With this the documented examples all run end to end, which is what M1 exits on: bring-your-own-data through init, azd up and run; results show --failed-only -O writing its file; and the CI/CD sequence of dataset create, run start -o json and results export --format csv. --- .../azure.ai.evaluations/internal/cmd/init.go | 42 ++++++++++++++++++- .../internal/cmd/init_test.go | 15 ++++--- 2 files changed, 49 insertions(+), 8 deletions(-) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go index 047257edd2e..2905b7e42ca 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go @@ -64,7 +64,7 @@ func newInitCommand() *cobra.Command { return err } - depCfg := buildDeployScaffold(target, rubricName, dataset, evaluators, evalModel) + depCfg := buildDeployScaffold(target, rubricName, dataset, evaluators, evalModel, outDir) if err := writeYAML(depPath, depCfg); err != nil { return err } @@ -130,10 +130,45 @@ func buildGenerateScaffold(target, rubricName, evalModel string) *project.Genera } } +// relativeToConfig rewrites a path given relative to the working directory so +// it resolves from the directory holding the deploy spec. +// +// `--dataset ./tests/golden.jsonl` means "relative to where I am", but the +// deploy spec's `source:` is resolved relative to that file, so writing the +// path through unchanged sends the deploy looking inside evals/. An absolute +// path is left alone, and forward slashes are kept so the config reads the same +// on every platform. +func relativeToConfig(path, outDir string) string { + if filepath.IsAbs(path) { + return path + } + + absPath, err := filepath.Abs(path) + if err != nil { + return path + } + absOut, err := filepath.Abs(outDir) + if err != nil { + return path + } + + rel, err := filepath.Rel(absOut, absPath) + if err != nil { + return path + } + + rel = filepath.ToSlash(rel) + if !strings.HasPrefix(rel, ".") { + rel = "./" + rel + } + return rel +} + func buildDeployScaffold( target, rubricName, dataset string, evaluators []string, evalModel string, + outDir string, ) *project.EvalConfig { cfg := &project.EvalConfig{} @@ -141,7 +176,10 @@ func buildDeployScaffold( datasetSource := "" if dataset != "" { if looksLikeLocalDataset(dataset) { - datasetSource = dataset + // --dataset is given relative to where the user is standing, but + // source: is resolved relative to the deploy spec, so the path has + // to be rebased or the deploy looks for it inside evals/. + datasetSource = relativeToConfig(dataset, outDir) datasetName = strings.TrimSuffix(filepath.Base(dataset), filepath.Ext(dataset)) } else { // A bare name references an already-registered dataset. diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_test.go index 55820051efb..f0db8167284 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_test.go @@ -19,7 +19,7 @@ func TestScaffold_RoundTripsAndValidates(t *testing.T) { dir := t.TempDir() depPath := filepath.Join(dir, "azure.yaml") - cfg := buildDeployScaffold("support-agent", "support-agent-quality", "", nil, "gpt-4.1-nano") + cfg := buildDeployScaffold("support-agent", "support-agent-quality", "", nil, "gpt-4.1-nano", project.DefaultEvalDir) require.NoError(t, writeYAML(depPath, cfg)) loaded, err := project.LoadEvalConfig(depPath) @@ -54,7 +54,7 @@ func TestGenerateScaffold_RoundTripsAndValidates(t *testing.T) { func TestScaffold_BuiltinEvaluatorsAreNotDeclared(t *testing.T) { cfg := buildDeployScaffold( "support-agent", "unused", "", - []string{"builtin.task_adherence", "my-custom"}, "", + []string{"builtin.task_adherence", "my-custom"}, "", project.DefaultEvalDir, ) require.Len(t, cfg.Evaluators, 1, "only the custom evaluator should be declared") @@ -74,20 +74,23 @@ func TestScaffold_BuiltinEvaluatorsAreNotDeclared(t *testing.T) { // A bare name means an already-registered dataset; a path means a local file. func TestScaffold_DatasetReferenceForms(t *testing.T) { t.Run("local path becomes a source", func(t *testing.T) { - cfg := buildDeployScaffold("a", "r", "./tests/golden.jsonl", nil, "") - require.Equal(t, "./tests/golden.jsonl", cfg.Datasets[0].Source) + // --dataset is relative to the working directory, but source: is + // resolved relative to the deploy spec, so it has to be rebased. + cfg := buildDeployScaffold("a", "r", "./tests/golden.jsonl", nil, "", "evals") + require.Equal(t, "../tests/golden.jsonl", cfg.Datasets[0].Source, + "a dataset outside the eval dir must be reached with ..") require.Equal(t, "golden", cfg.Datasets[0].Name) }) t.Run("bare name references a registered dataset", func(t *testing.T) { - cfg := buildDeployScaffold("a", "r", "prod-sample", nil, "") + cfg := buildDeployScaffold("a", "r", "prod-sample", nil, "", project.DefaultEvalDir) require.Equal(t, "prod-sample", cfg.Datasets[0].Name) require.Empty(t, cfg.Datasets[0].Source, "a registered dataset must not get a local source") }) t.Run("no dataset flag scaffolds a local path", func(t *testing.T) { - cfg := buildDeployScaffold("support-agent", "r", "", nil, "") + cfg := buildDeployScaffold("support-agent", "r", "", nil, "", project.DefaultEvalDir) require.Contains(t, cfg.Datasets[0].Source, "support-agent-golden.jsonl") }) } From ce2ce3a8c2b6dd209cc7e7af8ebe861af2d05a23 Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 28 Jul 2026 08:29:24 +0300 Subject: [PATCH 030/320] feat(azure.ai.evaluations): say when a trace setting is being ignored agent.context.traces accepts source, window and sample, but the generation API takes a day count and nothing else, so source and sample were parsed and dropped without a word. An author who set sample: 500 believed they had narrowed the trace selection when nothing had changed. Both fields are documented in the spec, so this was reachable by following it. They are now reported as having no effect, naming each one, with the verb agreeing so one field reads "has" and two read "have". The warning goes to stdout rather than stderr because azd does not surface an extension's stderr -- written to stderr it was invisible in a real run even though the unit test passed -- and is suppressed under -o json so the output stays parseable. --- .../internal/cmd/generate.go | 44 ++++++++++++++++++ .../internal/cmd/trace_warning_test.go | 46 +++++++++++++++++++ 2 files changed, 90 insertions(+) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/trace_warning_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go index 0a574635044..cdc411a2228 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go @@ -65,6 +65,11 @@ func newGenerateCommand() *cobra.Command { if err := cfg.Validate(); err != nil { return err } + // Written to stdout because azd does not surface an extension's + // stderr, and guarded so `-o json` stays parseable. + if !isJSON(cmd) { + warnIgnoredTraceFields(cfg, out) + } ec, err := newEvalContext(ctx, endpointFlg) if err != nil { @@ -149,6 +154,45 @@ func newGenerateCommand() *cobra.Command { return cmd } +// warnIgnoredTraceFields reports trace settings that are accepted but have no +// effect yet. +// +// The generation API takes a day window and nothing else, so `source` and +// `sample` are parsed and dropped. Silently discarding them is worse than not +// accepting them: the author believes they narrowed the trace selection when +// nothing changed. +func warnIgnoredTraceFields(cfg *project.GenerateConfig, out io.Writer) { + traces := cfg.Agent.Context.Traces + if traces == nil { + return + } + + var ignored []string + if traces.Source != "" { + ignored = append(ignored, "source") + } + if traces.Sample > 0 { + ignored = append(ignored, "sample") + } + if len(ignored) == 0 { + return + } + + fields := make([]string, 0, len(ignored)) + for _, name := range ignored { + fields = append(fields, "agent.context.traces."+name) + } + + verb := "has" + if len(fields) > 1 { + verb = "have" + } + fmt.Fprintf(out, + "warning: %s %s no effect yet; trace seeding uses only `window`. "+ + "Trace selection lands with the trace scenarios.\n", + strings.Join(fields, " and "), verb) +} + // resolveInstruction returns the generation instruction, reading it from a // file when one is named. // diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/trace_warning_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/trace_warning_test.go new file mode 100644 index 00000000000..6c952569aa4 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/trace_warning_test.go @@ -0,0 +1,46 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "bytes" + "testing" + + "azureaieval/internal/project" + + "github.com/stretchr/testify/require" +) + +func warnFor(t *testing.T, traces *project.TraceSpec) string { + t.Helper() + cfg := &project.GenerateConfig{} + cfg.Agent.Context.Traces = traces + + var buf bytes.Buffer + warnIgnoredTraceFields(cfg, &buf) + return buf.String() +} + +// source and sample are accepted by the config model but the generation API +// takes only a day window, so they are dropped. Dropping them silently lets an +// author believe they narrowed the trace selection when nothing changed. +func TestWarnsAboutTraceFieldsWithNoEffect(t *testing.T) { + out := warnFor(t, &project.TraceSpec{Source: "production", Window: "30d", Sample: 500}) + require.Contains(t, out, "agent.context.traces.source") + require.Contains(t, out, "agent.context.traces.sample") + require.Contains(t, out, "window") + require.Contains(t, out, "have no effect", "two fields take a plural verb") + + out = warnFor(t, &project.TraceSpec{Source: "production", Window: "30d"}) + require.Contains(t, out, "agent.context.traces.source") + require.NotContains(t, out, "sample") + require.Contains(t, out, "has no effect", "one field takes a singular verb") +} + +// The field that does work draws no warning, and neither does an absent block. +func TestNoWarningWhenOnlyWindowIsSet(t *testing.T) { + require.Empty(t, warnFor(t, &project.TraceSpec{Window: "30d"})) + require.Empty(t, warnFor(t, nil)) + require.Empty(t, warnFor(t, &project.TraceSpec{})) +} From b10f7060b6465eafa8c8a8448bb88dfbdbe37403 Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 28 Jul 2026 08:37:06 +0300 Subject: [PATCH 031/320] fix(azure.ai.evaluations): honour the config fields that were being dropped Swept every field the two config files declare against what the code reads. Three were parsed and ignored. options.max_samples did nothing, so a group that caps its sample count in config sent the whole dataset and only a flag on every invocation honoured the cap. It now resolves the same way the evaluation level already did: flag first, then the group's options. generate.dataset.strategy accepted from-traces, passed validation, and then generated synthetic rows anyway. Answering a different question than the one asked is worse than refusing, so it is now rejected with a pointer to agent.context.traces.window, which is how traces actually seed generation. An eval group's description was dropped because the create request has no field for it. It now rides in metadata, which the service stores and returns. It stays out of the group fingerprint deliberately: recreating an immutable group over a reworded description would cost the group id and break comparison against earlier runs, so an edit lands the next time the group is recreated for a reason that matters. --- .../internal/cmd/build.go | 5 ++ .../internal/cmd/description_test.go | 46 +++++++++++++++++++ .../azure.ai.evaluations/internal/cmd/run.go | 19 +++++++- .../internal/project/generate_config.go | 15 ++++-- .../internal/project/generate_config_test.go | 40 ++++++++++++++++ .../internal/project/service_target_eval.go | 8 +++- 6 files changed, 127 insertions(+), 6 deletions(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/description_test.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/project/generate_config_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build.go index b9c585c417c..803eed1c4e0 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build.go @@ -237,6 +237,11 @@ func buildEvalGroupRequest( metadata["azd_agent"] = group.Target.Name } metadata["azd_eval_group"] = group.Name + // The create request has no description field, so the group's own + // description rides in metadata rather than being dropped. + if group.Description != "" { + metadata["azd_description"] = group.Description + } evalModel := "" level := "" diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/description_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/description_test.go new file mode 100644 index 00000000000..ea819af4ef9 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/description_test.go @@ -0,0 +1,46 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "testing" + + "azureaieval/internal/pkg/eval_api" + "azureaieval/internal/pkg/evalcore" + "azureaieval/internal/project" + + "github.com/stretchr/testify/require" +) + +// The create request has no description field, so a documented description +// would otherwise be parsed and dropped. +func TestBuildCarriesGroupDescriptionInMetadata(t *testing.T) { + schemas := map[string]*eval_api.EvaluatorSummary{ + "builtin.similarity": schema("builtin.similarity", + nil, []string{"query", "response"}, + []string{"deployment_name"}, []string{"deployment_name"}, "turn"), + } + group := groupWith([]evalcore.EvaluatorRef{{Name: "builtin.similarity"}}, + &project.Options{EvalModel: "m"}) + group.Description = "Quality gate for the support agent" + + req, err := buildEvalGroupRequest(group, schemas, map[string]bool{"query": true}) + require.NoError(t, err) + require.Equal(t, "Quality gate for the support agent", req.Metadata["azd_description"]) +} + +// An absent description adds no metadata key rather than an empty one. +func TestBuildOmitsEmptyDescription(t *testing.T) { + schemas := map[string]*eval_api.EvaluatorSummary{ + "builtin.similarity": schema("builtin.similarity", + nil, []string{"query", "response"}, + []string{"deployment_name"}, []string{"deployment_name"}, "turn"), + } + group := groupWith([]evalcore.EvaluatorRef{{Name: "builtin.similarity"}}, + &project.Options{EvalModel: "m"}) + + req, err := buildEvalGroupRequest(group, schemas, map[string]bool{"query": true}) + require.NoError(t, err) + require.NotContains(t, req.Metadata, "azd_description") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go index db25185ecde..7448c93329a 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go @@ -99,7 +99,8 @@ func buildRunCommand(use, short string) *cobra.Command { if group == nil { dataSource, err = ec.reuseDataSourceFromLastRun(ctx, evalID) } else { - dataSource, err = buildRunDataSource(group, configPath, maxSamples) + dataSource, err = buildRunDataSource( + group, configPath, resolveMaxSamples(maxSamples, group)) } if err != nil { return err @@ -435,6 +436,22 @@ func resolveLevel(flag string, group *project.EvalGroup) string { return "" } +// resolveMaxSamples prefers the flag, then the group's options, matching how +// the evaluation level resolves. +// +// Without this, options.max_samples parsed and did nothing: a group that caps +// its sample count in config would send the whole dataset, and only a flag on +// every invocation would honour the cap. +func resolveMaxSamples(flag int, group *project.EvalGroup) int { + if flag > 0 { + return flag + } + if group != nil && group.Options != nil && group.Options.MaxSamples > 0 { + return group.Options.MaxSamples + } + return 0 +} + // pollRun waits for the run to reach a terminal state, reporting status changes. func (ec *evalContext) pollRun( ctx context.Context, diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/generate_config.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/generate_config.go index 4e1d647387d..22ec21ab3e0 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/generate_config.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/generate_config.go @@ -114,11 +114,20 @@ func (c *GenerateConfig) Validate() error { return fmt.Errorf("generate.dataset.name is required") } switch d.Strategy { - case "", StrategySynthetic, StrategyFromTraces: + case "", StrategySynthetic: + case StrategyFromTraces: + // Accepting this and generating synthetic rows anyway would hand back + // data that looks nothing like what was asked for. The generation API + // takes one dataset strategy today; traces seed generation through + // the agent's context instead. + return fmt.Errorf( + "generate.dataset.strategy %q is not supported yet; "+ + "use %q, and set agent.context.traces.window to seed generation from traces", + StrategyFromTraces, StrategySynthetic) default: return fmt.Errorf( - "generate.dataset.strategy %q is invalid; expected %q or %q", - d.Strategy, StrategySynthetic, StrategyFromTraces) + "generate.dataset.strategy %q is invalid; expected %q", + d.Strategy, StrategySynthetic) } if d.SampleSize != 0 && (d.SampleSize < MinSampleSize || d.SampleSize > MaxSampleSize) { return fmt.Errorf( diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/generate_config_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/generate_config_test.go new file mode 100644 index 00000000000..227674b0fc4 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/generate_config_test.go @@ -0,0 +1,40 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package project + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func generateCfgWithStrategy(strategy string) *GenerateConfig { + cfg := &GenerateConfig{} + cfg.Agent.Name = "my-agent" + cfg.Generate.Rubric = &RubricSpec{Name: "r"} + cfg.Generate.Dataset = &DatasetSpec{Name: "d", Strategy: strategy} + return cfg +} + +// from-traces used to pass validation and then generate synthetic rows anyway, +// handing back data that looked nothing like what was asked for. Rejecting it +// is better than answering the wrong question. +func TestValidateRejectsUnsupportedDatasetStrategy(t *testing.T) { + err := generateCfgWithStrategy(StrategyFromTraces).Validate() + require.Error(t, err) + require.Contains(t, err.Error(), "not supported yet") + require.Contains(t, err.Error(), "agent.context.traces.window", + "the error should point at the way traces are actually used") +} + +func TestValidateAcceptsSupportedDatasetStrategies(t *testing.T) { + require.NoError(t, generateCfgWithStrategy("").Validate()) + require.NoError(t, generateCfgWithStrategy(StrategySynthetic).Validate()) +} + +func TestValidateRejectsUnknownDatasetStrategy(t *testing.T) { + err := generateCfgWithStrategy("made-up").Validate() + require.Error(t, err) + require.Contains(t, err.Error(), "invalid") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go index b6dd930ced4..39e6f8ed9fa 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go @@ -321,8 +321,12 @@ func Fingerprint(path string) (string, error) { // evaluators are untouched. Without this a retargeted group keeps running // against the old definition. func FingerprintGroup(group EvalGroup) (string, error) { - // The id is server-assigned and the description is cosmetic; neither - // changes what the group evaluates. + // The id is server-assigned. The description is carried in the group's + // metadata, so editing it does change the request, but recreating an + // immutable group over a reworded description would cost the group id and + // break comparison against earlier runs. It is documentation, not + // evaluation semantics, so an edit lands the next time the group is + // recreated for a reason that matters. group.ID = "" group.Description = "" From 2a101c05fe2d1ca878a6bc25acbce981b87665fd Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 28 Jul 2026 08:41:37 +0300 Subject: [PATCH 032/320] build(azure.ai.evaluations): ignore the artifacts the tooling produces ci-test.ps1 writes junitTestReport.xml for the pipeline to publish, and --debug writes a dated log, both into the extension directory. Neither was ignored, so a developer running the tests or debugging locally saw them offered for commit. Scoped to this extension rather than the root ignore file: the CLI's own test report has the same gap, but widening a repo-wide rule is not this change's to make. --- cli/azd/extensions/azure.ai.evaluations/.gitignore | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 cli/azd/extensions/azure.ai.evaluations/.gitignore diff --git a/cli/azd/extensions/azure.ai.evaluations/.gitignore b/cli/azd/extensions/azure.ai.evaluations/.gitignore new file mode 100644 index 00000000000..0d5b6d76489 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/.gitignore @@ -0,0 +1,5 @@ +# Test report written by ci-test.ps1 for the pipeline to publish. +junitTestReport.xml + +# Debug log written when --debug or AZD_EXT_DEBUG is set. +azd-ai-eval-*.log From 7f1b067c2e1e82fc742b4da7f3c833f1e63e6ec5 Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 28 Jul 2026 08:49:26 +0300 Subject: [PATCH 033/320] fix(azure.ai.evaluations): make the persistence warnings useful Two warnings about failing to persist a value into the azd environment went to stderr, which azd does not surface, so nobody ever saw them. The same mistake was already fixed for the trace warning; these were the remaining cases. They also fired for a situation that is not a problem. The atomic commands are meant to work standalone against the data plane, so running outside a project is ordinary, and reporting that there is nowhere to persist would be noise on every standalone invocation. An absent environment is now told apart from a write that genuinely failed, and only the latter is reported. What remains is written to stdout so it is visible, and suppressed under -o json so the output stays parseable. Verified both ways: creating a dataset outside any project prints only the result, and a deploy inside one still records its state. --- .../internal/cmd/context.go | 10 +++++- .../internal/cmd/dataset.go | 10 ++++-- .../internal/cmd/envwarn_test.go | 32 +++++++++++++++++++ .../azure.ai.evaluations/internal/cmd/run.go | 8 ++++- 4 files changed, 56 insertions(+), 4 deletions(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/envwarn_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/context.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/context.go index ef7562644fa..7ec1da4237a 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/context.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/context.go @@ -5,6 +5,7 @@ package cmd import ( "context" + "errors" "fmt" "os" "strings" @@ -100,13 +101,20 @@ func lookupEndpointFromAzd(ctx context.Context, azdClient *azdext.AzdClient) (en return val.Value, envResp.Environment.Name } +// errNoAzdEnvironment reports that there is no azd environment to persist into. +// +// The atomic commands are meant to work standalone against the data plane, so +// running outside a project is ordinary rather than a problem worth reporting. +// A write that fails for any other reason still is. +var errNoAzdEnvironment = errors.New("no active azd environment") + // setEnvValue persists a value into the active azd environment. azd itself // writes none of these keys — the extension owns them. func (ec *evalContext) setEnvValue(ctx context.Context, key, value string) error { if ec.envName == "" { envResp, err := ec.azdClient.Environment().GetCurrent(ctx, &azdext.EmptyRequest{}) if err != nil || envResp == nil || envResp.Environment == nil { - return fmt.Errorf("no active azd environment to write %s into", key) + return fmt.Errorf("%w to write %s into", errNoAzdEnvironment, key) } ec.envName = envResp.Environment.Name } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset.go index ebdc4439e83..5c18f453297 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset.go @@ -4,6 +4,7 @@ package cmd import ( + "errors" "fmt" "os" "path/filepath" @@ -84,8 +85,13 @@ func newDatasetCreateCommand(update bool) *cobra.Command { } if err := ec.setEnvValue(ctx, envKeyDatasetVersion, ds.Version); err != nil { - // Persisting is a convenience; do not fail the command over it. - fmt.Fprintf(cmd.ErrOrStderr(), "warning: %v\n", err) + // Persisting is a convenience, so this never fails the command. + // It goes to stdout because azd does not surface an extension's + // stderr, and is skipped outside a project, where having nowhere + // to persist is expected rather than notable. + if !errors.Is(err, errNoAzdEnvironment) && !isJSON(cmd) { + fmt.Fprintf(cmd.OutOrStdout(), "warning: %v\n", err) + } } if isJSON(cmd) { diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/envwarn_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/envwarn_test.go new file mode 100644 index 00000000000..8652dc6b958 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/envwarn_test.go @@ -0,0 +1,32 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "errors" + "fmt" + "testing" + + "github.com/stretchr/testify/require" +) + +// The atomic commands are meant to work standalone against the data plane, so +// running outside a project is ordinary. Warning about nowhere to persist would +// be noise on every standalone invocation. +func TestNoAzdEnvironmentIsRecognisable(t *testing.T) { + err := fmt.Errorf("%w to write %s into", errNoAzdEnvironment, "EVAL_RUN_ID") + + require.ErrorIs(t, err, errNoAzdEnvironment, + "callers rely on telling this apart from a failed write") + require.Contains(t, err.Error(), "EVAL_RUN_ID", + "the key is still named when the message is shown") +} + +// A write that fails for any other reason stays reportable. +func TestOtherEnvironmentFailuresStayReportable(t *testing.T) { + err := fmt.Errorf("writing %s to the azd environment: %w", "EVAL_RUN_ID", errors.New("rpc failed")) + + require.NotErrorIs(t, err, errNoAzdEnvironment) + require.Contains(t, err.Error(), "rpc failed") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go index 7448c93329a..a90e4994745 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go @@ -7,6 +7,7 @@ import ( "bufio" "context" "encoding/json" + "errors" "fmt" "os" "path/filepath" @@ -129,7 +130,12 @@ func buildRunCommand(use, short string) *cobra.Command { } if err := ec.setEnvValue(ctx, envKeyEvalRunID, run.ID); err != nil { - fmt.Fprintf(cmd.ErrOrStderr(), "warning: %v\n", err) + // Persisting the run id is a convenience for later commands. + // Reported on stdout because azd does not surface an + // extension's stderr, and skipped outside a project. + if !errors.Is(err, errNoAzdEnvironment) && !isJSON(cmd) { + fmt.Fprintf(out, "warning: %v\n", err) + } } if !wait { From 5853089bfec3e04ed774f16731b558078a8e8275 Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 28 Jul 2026 08:59:15 +0300 Subject: [PATCH 034/320] feat(azure.ai.evaluations): compare runs against a baseline M1 is complete and the comparison half of M2 was the next unblocked piece, so this adds `azd ai eval results compare`. Confirmed first that the APIs M2 needs are actually reachable in a live project rather than trusting the milestone note: schedules, evaluations/schedules and insights all answer. The spec described the request as {evalId, baselineRunId, treatmentRunIds}. Probing the live API found three things missing. displayName is required and is rejected before anything else is looked at. The comparison fields go inside a request wrapper. That wrapper is polymorphic and needs a type discriminator of EvaluationComparison, which the service itself named once asked. The synchronous POST /insights/sync returns a 500 for this shape, so the asynchronous form and a poll is the only usable path. With no flags it compares the two most recent completed runs, which is what asking whether a change helped usually means. --baseline and --treatment name runs explicitly, and --treatment repeats. Output carries the statistical method, per-metric baseline and treatment averages, a signed delta, the p-value, and the service's own effect classification. That last one matters: the probe returned TooFewSamples, and dropping it would make an inconclusive comparison read as a finding. --- .../internal/cmd/compare.go | 206 ++++++++++++++++++ .../internal/cmd/compare_test.go | 83 +++++++ .../internal/cmd/results.go | 2 +- .../internal/pkg/eval_api/insights.go | 121 ++++++++++ 4 files changed, 411 insertions(+), 1 deletion(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/compare.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/compare_test.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/insights.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/compare.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/compare.go new file mode 100644 index 00000000000..500f135b3fd --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/compare.go @@ -0,0 +1,206 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "context" + "fmt" + "time" + + "azureaieval/internal/pkg/eval_api" + + "github.com/spf13/cobra" +) + +// comparePollBudget bounds the wait for a comparison. The probe returned in +// about a second, so this is generous headroom rather than an expected wait. +const ( + comparePollInterval = 3 * time.Second + comparePollAttempts = 100 +) + +func newResultsCompareCommand() *cobra.Command { + var ( + baseline string + treatments []string + displayName string + endpointFlg string + ) + + cmd := &cobra.Command{ + Use: "compare [eval-id]", + Short: "Compare runs of an eval group against a baseline.", + Args: cobra.MaximumNArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + ctx := cmd.Context() + out := cmd.OutOrStdout() + + ec, err := newEvalContext(ctx, endpointFlg) + if err != nil { + return err + } + defer ec.Close() + + evalID, err := resolveEvalID(cmd, ec, args) + if err != nil { + return err + } + + baseline, treatments, err = ec.resolveComparisonRuns(ctx, evalID, baseline, treatments) + if err != nil { + return err + } + + if displayName == "" { + displayName = fmt.Sprintf("compare-%s", time.Now().UTC().Format("20060102-150405")) + } + + insight, err := ec.evalClient.CreateInsight(ctx, &eval_api.CreateInsightRequest{ + DisplayName: displayName, + Request: &eval_api.InsightRequest{ + Type: eval_api.InsightTypeEvaluationComparison, + EvalID: evalID, + BaselineRunID: baseline, + TreatmentRunIDs: treatments, + }, + }, ProjectEndpointAPIVersion) + if err != nil { + return fmt.Errorf("starting the comparison: %w", err) + } + + if !isJSON(cmd) { + fmt.Fprintf(out, "Comparing %d run(s) against %s...\n", len(treatments), baseline) + } + + completed, err := ec.pollInsight(ctx, insight.ID) + if err != nil { + return err + } + if isJSON(cmd) { + return emitJSON(out, completed) + } + return renderComparison(out, completed) + }, + } + + cmd.Flags().StringVar(&baseline, "baseline", "", + "Run to compare against. Defaults to the second most recent completed run.") + cmd.Flags().StringArrayVar(&treatments, "treatment", nil, + "Run to measure, repeatable. Defaults to the most recent completed run.") + cmd.Flags().StringVar(&displayName, "name", "", "Name for this comparison.") + cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") + return cmd +} + +// resolveComparisonRuns fills in whichever runs were not named. +// +// Comparing the two most recent completed runs is what "did my change help?" +// means most of the time, so neither flag is required. +func (ec *evalContext) resolveComparisonRuns( + ctx context.Context, + evalID, baseline string, + treatments []string, +) (string, []string, error) { + if baseline != "" && len(treatments) > 0 { + return baseline, treatments, nil + } + + list, err := ec.evalClient.ListOpenAIEvalRuns(ctx, evalID, 0) + if err != nil { + return "", nil, fmt.Errorf("listing runs of eval group %s: %w", evalID, err) + } + + completed := make([]string, 0, 2) + if list == nil { + return "", nil, fmt.Errorf("eval group %s has no runs", evalID) + } + for _, run := range list.Data { + if run.Status == "completed" { + completed = append(completed, run.ID) + } + } + + if len(treatments) == 0 { + if len(completed) == 0 { + return "", nil, fmt.Errorf( + "eval group %s has no completed runs to compare", evalID) + } + treatments = []string{completed[0]} + } + if baseline == "" { + if len(completed) < 2 { + return "", nil, fmt.Errorf( + "eval group %s has only one completed run, so there is nothing to compare it "+ + "against; run it again, or name a baseline with --baseline", + evalID) + } + baseline = completed[1] + } + return baseline, treatments, nil +} + +// pollInsight waits for the comparison to reach a terminal state. +func (ec *evalContext) pollInsight(ctx context.Context, insightID string) (*eval_api.Insight, error) { + for attempt := 0; attempt < comparePollAttempts; attempt++ { + select { + case <-ctx.Done(): + return nil, ctx.Err() + case <-time.After(comparePollInterval): + } + + insight, err := ec.evalClient.GetInsight(ctx, insightID, ProjectEndpointAPIVersion) + if err != nil { + return nil, fmt.Errorf("reading comparison %s: %w", insightID, err) + } + if !insight.Terminal() { + continue + } + if !insight.Succeeded() { + return nil, fmt.Errorf("comparison %s finished with state %q", insightID, insight.State) + } + return insight, nil + } + return nil, fmt.Errorf("comparison %s did not finish in time", insightID) +} + +// renderComparison prints one row per criterion per treatment run. +func renderComparison(w interface{ Write([]byte) (int, error) }, insight *eval_api.Insight) error { + if insight.Result == nil || len(insight.Result.Comparisons) == 0 { + fmt.Fprintln(w, "The comparison produced no metrics.") + return nil + } + + if insight.Result.Method != "" { + fmt.Fprintf(w, "Method: %s\n\n", insight.Result.Method) + } + + rows := [][]string{} + for _, c := range insight.Result.Comparisons { + baseAvg := "-" + if c.BaselineRunSummary != nil { + baseAvg = fmt.Sprintf("%.3f", c.BaselineRunSummary.Average) + } + for _, item := range c.CompareItems { + treatAvg := "-" + runID := "-" + if item.TreatmentRunSummary != nil { + treatAvg = fmt.Sprintf("%.3f", item.TreatmentRunSummary.Average) + runID = item.TreatmentRunSummary.RunID + } + rows = append(rows, []string{ + c.Metric, + runID, + baseAvg, + treatAvg, + fmt.Sprintf("%+.3f", item.DeltaEstimate), + fmt.Sprintf("%.3f", item.PValue), + item.TreatmentEffect, + }) + } + } + + return emitTable(w, + []string{"METRIC", "TREATMENT RUN", "BASELINE", "TREATMENT", "DELTA", "P-VALUE", "EFFECT"}, + rows) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/compare_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/compare_test.go new file mode 100644 index 00000000000..5bbf0ad2370 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/compare_test.go @@ -0,0 +1,83 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "bytes" + "testing" + + "azureaieval/internal/pkg/eval_api" + + "github.com/stretchr/testify/require" +) + +func TestInsightTerminalStates(t *testing.T) { + for _, state := range []string{"", "NotStarted", "Running", "InProgress", "Queued"} { + require.False(t, (&eval_api.Insight{State: state}).Terminal(), "%q is not terminal", state) + } + for _, state := range []string{"Succeeded", "Failed", "Cancelled"} { + require.True(t, (&eval_api.Insight{State: state}).Terminal(), "%q is terminal", state) + } + require.True(t, (&eval_api.Insight{State: "Succeeded"}).Succeeded()) + require.False(t, (&eval_api.Insight{State: "Failed"}).Succeeded()) +} + +// The rendered table is how a reader decides whether a change helped, so the +// delta carries its sign and the effect classification is not dropped. +func TestRenderComparisonShowsSignedDeltaAndEffect(t *testing.T) { + insight := &eval_api.Insight{ + State: "Succeeded", + Result: &eval_api.InsightResult{ + Method: "PairedTTest", + Comparisons: []eval_api.MetricComparison{{ + Metric: "task_adherence", + BaselineRunSummary: &eval_api.RunSummary{RunID: "base", Average: 0.75}, + CompareItems: []eval_api.CompareItem{{ + TreatmentRunSummary: &eval_api.RunSummary{RunID: "treat", Average: 0.5}, + DeltaEstimate: -0.25, + PValue: 0.39, + TreatmentEffect: "TooFewSamples", + }}, + }}, + }, + } + + var buf bytes.Buffer + require.NoError(t, renderComparison(&buf, insight)) + out := buf.String() + + require.Contains(t, out, "PairedTTest", "the statistical method qualifies the result") + require.Contains(t, out, "task_adherence") + require.Contains(t, out, "-0.250", "a regression must read as negative") + require.Contains(t, out, "0.390") + require.Contains(t, out, "TooFewSamples", + "an inconclusive result must not look like a finding") +} + +// A positive delta reads as an improvement rather than an unsigned number. +func TestRenderComparisonSignsImprovements(t *testing.T) { + insight := &eval_api.Insight{ + State: "Succeeded", + Result: &eval_api.InsightResult{ + Comparisons: []eval_api.MetricComparison{{ + Metric: "similarity", + BaselineRunSummary: &eval_api.RunSummary{Average: 0.5}, + CompareItems: []eval_api.CompareItem{{ + TreatmentRunSummary: &eval_api.RunSummary{RunID: "t", Average: 0.8}, + DeltaEstimate: 0.3, + }}, + }}, + }, + } + + var buf bytes.Buffer + require.NoError(t, renderComparison(&buf, insight)) + require.Contains(t, buf.String(), "+0.300") +} + +func TestRenderComparisonHandlesEmptyResult(t *testing.T) { + var buf bytes.Buffer + require.NoError(t, renderComparison(&buf, &eval_api.Insight{State: "Succeeded"})) + require.Contains(t, buf.String(), "no metrics") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/results.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/results.go index 3eb79a1bc22..985a4b7b461 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/results.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/results.go @@ -21,7 +21,7 @@ func newResultsCommand() *cobra.Command { Use: "results", Short: "Inspect evaluation results.", } - cmd.AddCommand(newResultsShowCommand(), newResultsExportCommand()) + cmd.AddCommand(newResultsShowCommand(), newResultsExportCommand(), newResultsCompareCommand()) return cmd } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/insights.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/insights.go new file mode 100644 index 00000000000..8296fd21992 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/insights.go @@ -0,0 +1,121 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package eval_api + +import ( + "context" + "fmt" + "net/http" + "net/url" +) + +// InsightTypeEvaluationComparison compares evaluation runs. The service also +// defines EvaluationRunClusterInsight and AgentClusterInsight, which this +// extension does not use. +const InsightTypeEvaluationComparison = "EvaluationComparison" + +const pathInsights = "/insights" + +// InsightRequest is the polymorphic body the service dispatches on. `type` is +// the discriminator; without it the request is rejected because the underlying +// contract is an interface. +type InsightRequest struct { + Type string `json:"type"` + EvalID string `json:"evalId"` + BaselineRunID string `json:"baselineRunId"` + TreatmentRunIDs []string `json:"treatmentRunIds"` +} + +// CreateInsightRequest wraps the request. DisplayName is required; the service +// rejects a body without it before it looks at anything else. +type CreateInsightRequest struct { + DisplayName string `json:"displayName"` + Request *InsightRequest `json:"request"` +} + +// RunSummary is one run's aggregate for a single metric. +type RunSummary struct { + RunID string `json:"runId"` + SampleCount int `json:"sampleCount"` + Average float64 `json:"average"` + StandardDeviation float64 `json:"standardDeviation"` +} + +// CompareItem is one treatment run measured against the baseline. +type CompareItem struct { + TreatmentRunSummary *RunSummary `json:"treatmentRunSummary,omitempty"` + DeltaEstimate float64 `json:"deltaEstimate"` + PValue float64 `json:"pValue"` + // TreatmentEffect classifies the result, e.g. TooFewSamples when the + // sample count cannot support a conclusion. + TreatmentEffect string `json:"treatmentEffect,omitempty"` +} + +// MetricComparison is the baseline and treatments for one testing criterion. +type MetricComparison struct { + TestingCriteria string `json:"testingCriteria"` + Metric string `json:"metric"` + Evaluator string `json:"evaluator"` + BaselineRunSummary *RunSummary `json:"baselineRunSummary,omitempty"` + CompareItems []CompareItem `json:"compareItems,omitempty"` +} + +// InsightResult carries the comparison once the insight succeeds. +type InsightResult struct { + Comparisons []MetricComparison `json:"comparisons,omitempty"` + // Method names the statistical test, e.g. PairedTTest. + Method string `json:"method,omitempty"` + Type string `json:"type,omitempty"` + Error any `json:"error,omitempty"` +} + +// Insight is the long-running operation the comparison runs as. +type Insight struct { + ID string `json:"id"` + DisplayName string `json:"displayName,omitempty"` + State string `json:"state,omitempty"` + Request *InsightRequest `json:"request,omitempty"` + Result *InsightResult `json:"result,omitempty"` +} + +// Succeeded reports whether the insight finished with a result. +func (i *Insight) Succeeded() bool { + return i != nil && i.State == "Succeeded" +} + +// Terminal reports whether the insight has stopped changing. +func (i *Insight) Terminal() bool { + if i == nil { + return false + } + switch i.State { + case "", "NotStarted", "Running", "InProgress", "Queued": + return false + default: + return true + } +} + +// CreateInsight starts a comparison. +// +// The synchronous variant, POST /insights/sync, returns a 500 for this request +// shape, so the asynchronous form is the only usable one and the caller polls. +func (c *EvalClient) CreateInsight( + ctx context.Context, + request *CreateInsightRequest, + apiVersion string, +) (*Insight, error) { + return doRequestTyped[Insight]( + c, ctx, http.MethodPost, pathInsights, nil, request, apiVersion) +} + +// GetInsight reads a comparison's current state. +func (c *EvalClient) GetInsight( + ctx context.Context, + insightID string, + apiVersion string, +) (*Insight, error) { + path := fmt.Sprintf("%s/%s", pathInsights, url.PathEscape(insightID)) + return doRequestTyped[Insight](c, ctx, http.MethodGet, path, nil, nil, apiVersion) +} From 63a7a33266e357d41a8517297f828eeb4642459f Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 28 Jul 2026 09:24:06 +0300 Subject: [PATCH 035/320] fix(azure.ai.evaluations): finish the generate step of the documented flow M1 exits on the spec examples running end to end, and the first one could not: its generate step failed. Two separate causes, the second only reachable once the first was out of the way. Agent-seeded data generation fails server-side for every agent, which no user can act on. The same request carrying only the prompt succeeds and produces a usable dataset, so a failure now retries without the agent and says that it did. The rubric still uses the agent's context; only the dataset falls back. That exposed the real defect underneath. A dataset's URI points at either the blob or the container holding it, and nothing in the payload distinguishes them: isSingleFile is true either way. Uploads end in the file name, generated datasets end in the container, and downloading a container returns 409. The download path also sent no SAS token, so it could not have authenticated even against the right URL. It had never run successfully, because generation always failed before reaching it. Downloads now fetch a credential, use the blob directly when the URI names one, and otherwise list the container and take the JSONL inside. This retires an assumption recorded earlier as holding: that a dataset blob URI can be downloaded directly. It is true only for uploads. --- .../internal/cmd/generate.go | 41 +++++++++- .../internal/pkg/dataset_api/download_test.go | 47 ++++++++++++ .../internal/pkg/dataset_api/operations.go | 75 +++++++++++++++++++ .../internal/pkg/eval_api/generation.go | 26 +++++++ 4 files changed, 186 insertions(+), 3 deletions(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/download_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go index cdc411a2228..eff801fc119 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go @@ -342,6 +342,27 @@ func (ec *evalContext) generateDataset( completed, err := ec.pollGeneration(ctx, job.ID, DataGenerationAPIVersion, ec.evalClient.GetDataGenerationJob) + if err != nil && isAgentSeededGenerationFailure(err) { + // Agent-seeded generation fails server-side for every agent, while the + // same request carrying only the prompt succeeds. Failing the whole + // command would block the documented flow on a defect the user cannot + // do anything about, so retry without the agent and say so. + promptOnly := eval_api.WithoutAgentSource(sources) + if eval_api.HasPromptSource(promptOnly) { + fmt.Fprintf(out, + " warning: generating from agent %q failed in the service; "+ + "retrying from the instruction alone.\n", cfg.Agent.Name) + + req = eval_api.NewDataGenerationJobRequest( + spec.Name, model, spec.SampleSize, promptOnly) + job, err = ec.evalClient.CreateDataGenerationJob(ctx, req, DataGenerationAPIVersion) + if err != nil { + return nil, fmt.Errorf("submitting the data generation job: %w", err) + } + completed, err = ec.pollGeneration(ctx, job.ID, DataGenerationAPIVersion, + ec.evalClient.GetDataGenerationJob) + } + } if err != nil { return nil, fmt.Errorf("data generation: %w", explainDataGenerationFailure(err, cfg.Agent.Name)) } @@ -351,11 +372,14 @@ func (ec *evalContext) generateDataset( return nil, fmt.Errorf("the data generation job returned no dataset reference") } - ds, err := ec.datasetClient.GetDataset(ctx, name, version, ProjectEndpointAPIVersion) - if err != nil { + // Confirm the version exists before reading it, so a missing dataset is + // reported as such rather than as a download failure. + if _, err := ec.datasetClient.GetDataset( + ctx, name, version, ProjectEndpointAPIVersion, + ); err != nil { return nil, fmt.Errorf("reading the generated dataset %q: %w", name, err) } - content, err := ec.datasetClient.DownloadDataset(ctx, ds.ResolvedBlobURI()) + content, err := ec.datasetClient.DownloadDatasetContent(ctx, name, version, ProjectEndpointAPIVersion) if err != nil { return nil, fmt.Errorf("downloading the generated dataset %q: %w", name, err) } @@ -372,6 +396,17 @@ func (ec *evalContext) generateDataset( return &project.ArtifactRef{Name: spec.Name, Source: relativeSource(baseDir, path)}, nil } +// isAgentSeededGenerationFailure recognises the service-side failure that hits +// every agent, so it can be retried without the agent rather than surfaced. +func isAgentSeededGenerationFailure(err error) bool { + if err == nil { + return false + } + text := err.Error() + return strings.Contains(text, "DataGenerationJobSystemError") || + strings.Contains(text, "Something went wrong during data generation") +} + // explainDataGenerationFailure adds context to the service's opaque system // error. // diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/download_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/download_test.go new file mode 100644 index 00000000000..9ac6f21e92b --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/download_test.go @@ -0,0 +1,47 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package dataset_api + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +// A dataset's URI points at either the blob or the container holding it, +// depending on how it was created, and nothing in the payload says which: +// isSingleFile is true either way. Uploaded datasets end in the file name; +// generated ones end in the container. Downloading a container returns 409. +func TestLooksLikeBlobURI(t *testing.T) { + uploaded := "https://acct.blob.core.windows.net:443/container-guid/azd-smoke-golden.jsonl" + generated := "https://acct.blob.core.windows.net/asayedahme-420d0b21-956c-513b-bb18-f60bfbf5e724" + + require.True(t, looksLikeBlobURI(uploaded), "an uploaded dataset names its file") + require.False(t, looksLikeBlobURI(generated), "a generated dataset names its container") +} + +// A SAS token on the URI must not change the answer. +func TestLooksLikeBlobURIIgnoresQuery(t *testing.T) { + require.True(t, looksLikeBlobURI( + "https://acct.blob.core.windows.net/c/data.jsonl?sv=2021&sig=abc")) + require.False(t, looksLikeBlobURI( + "https://acct.blob.core.windows.net/c?sv=2021&sig=abc")) + require.False(t, looksLikeBlobURI("https://acct.blob.core.windows.net/c/")) +} + +// An evaluation dataset is JSONL, so that is preferred when a container holds +// more than one file. +func TestPickDatasetBlobPrefersJSONL(t *testing.T) { + require.Equal(t, "data.jsonl", + pickDatasetBlob([]string{"_meta.json", "data.jsonl", "readme.txt"})) + require.Equal(t, "data.JSONL", + pickDatasetBlob([]string{"data.JSONL"}), "the extension match is case-insensitive") +} + +// With nothing recognisable, any real file beats returning nothing. +func TestPickDatasetBlobFallsBackToAnyFile(t *testing.T) { + require.Equal(t, "data.csv", pickDatasetBlob([]string{"data.csv"})) + require.Empty(t, pickDatasetBlob([]string{"folder/"})) + require.Empty(t, pickDatasetBlob(nil)) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/operations.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/operations.go index 2589981ed15..be1e3e03e5f 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/operations.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/operations.go @@ -14,6 +14,7 @@ import ( "log" "net/http" "net/url" + "path" "strings" "azureaieval/internal/version" @@ -276,6 +277,80 @@ func (c *DatasetClient) GetDatasetCredential( return doRequestTyped[DatasetCredential](c, ctx, http.MethodPost, path, nil, nil, apiVersion) } +// DownloadDatasetContent fetches a dataset version's content, whether its URI +// names a blob or a container. +// +// The two differ by origin, not by any field: a dataset uploaded through +// startPendingUpload gets a URI ending in the file name, while one produced by +// a generation job gets the container it was written into, with isSingleFile +// true either way. Downloading the container directly returns a 409, so the +// blob inside has to be found first. +// +// A credential is always fetched, because the URI on the dataset carries no +// SAS token and an unauthenticated read fails. +func (c *DatasetClient) DownloadDatasetContent( + ctx context.Context, + name string, + version string, + apiVersion string, +) ([]byte, error) { + cred, err := c.GetDatasetCredential(ctx, name, version, apiVersion) + if err != nil { + return nil, fmt.Errorf("reading download credentials for %q: %w", name, err) + } + + sasURI := cred.ResolvedDownloadURI() + if sasURI == "" { + return nil, fmt.Errorf("no download URI returned for dataset %q", name) + } + + // A URI whose last path segment carries a file extension is the blob + // itself; anything else is the container holding it. + if looksLikeBlobURI(sasURI) { + data, err := c.DownloadDataset(ctx, sasURI) + if err == nil { + return data, nil + } + log.Printf("[dataset_api] direct download failed (%v); treating the URI as a container", err) + } + + names, err := c.ListContainerBlobs(ctx, sasURI) + if err != nil { + return nil, fmt.Errorf("listing the content of dataset %q: %w", name, err) + } + blobName := pickDatasetBlob(names) + if blobName == "" { + return nil, fmt.Errorf("dataset %q holds no downloadable file", name) + } + return c.DownloadBlob(ctx, sasURI, blobName) +} + +// looksLikeBlobURI reports whether the URI's final segment names a file. +func looksLikeBlobURI(raw string) bool { + u, err := url.Parse(raw) + if err != nil { + return false + } + last := path.Base(strings.TrimSuffix(u.Path, "/")) + return path.Ext(last) != "" +} + +// pickDatasetBlob chooses the file to read from a container, preferring JSONL +// since that is what an evaluation dataset is. +func pickDatasetBlob(names []string) string { + for _, n := range names { + if strings.EqualFold(path.Ext(n), ".jsonl") { + return n + } + } + for _, n := range names { + if n != "" && !strings.HasSuffix(n, "/") { + return n + } + } + return "" +} + // DownloadDataset downloads dataset content from blob storage using a SAS-authenticated URL. // Returns the raw content as bytes. The downloadURL should be the full URL with SAS token // (e.g., from DatasetCredential.ResolvedDownloadURI()). diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/generation.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/generation.go index 2b12968065a..26727677270 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/generation.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/generation.go @@ -24,6 +24,32 @@ type TraceOptions struct { // A prompt source is included when instruction is non-empty, along with the // agent source. When traces is non-nil and Days > 0, a traces source is // appended with start_time computed from the current time. +// WithoutAgentSource returns the sources with the agent entry removed. +// +// Agent-seeded data generation currently fails server-side for every agent, +// while the same request carrying only the prompt succeeds, so this is what a +// retry falls back to. +func WithoutAgentSource(sources []GenerationSource) []GenerationSource { + kept := make([]GenerationSource, 0, len(sources)) + for _, s := range sources { + if s.Type == "agent" { + continue + } + kept = append(kept, s) + } + return kept +} + +// HasPromptSource reports whether anything remains to generate from. +func HasPromptSource(sources []GenerationSource) bool { + for _, s := range sources { + if s.Type == "prompt" && s.Prompt != "" { + return true + } + } + return false +} + func BuildGenerationSources(agentKind, agentName, version, instruction string, traces *TraceOptions) []GenerationSource { var sources []GenerationSource From b122b0f4ea5449ec37655db22ad7c15a18916bac Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 28 Jul 2026 10:27:10 +0300 Subject: [PATCH 036/320] fix(azure.ai.evaluations): seed generation from the agent, not from a source that never works MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Agent-seeded data generation fails server-side for every agent. Following it through the stack, the request reaches AOAI unchanged: RAISvc forwards it and FineTuning rewrites the path, and neither resolves the agent. Nothing in this repo can fix it, and nothing the client sends changes the outcome — a nonexistent agent name fails exactly like a real one, so the agent is never looked up at all. But the contract says what that source was for: it "references an agent to fetch instructions / metadata from". That is a read this client can do itself. So `generate` now resolves the agent's context locally, preferring what the author supplied: an explicit instruction, else the file named by agent.context.instructions, else the agent's published instructions. That last step is what makes the scaffolded flow work. `init` writes agent.context.instructions pointing at a conventional path, and nothing read it — the field, and its `tools` sibling, were accepted and dropped. With no instruction anywhere, generation had nothing to fall back to when the agent source failed, so `init` followed by `generate` could not produce a dataset. It now generates one from the agent's own instructions, on topic and scored by a rubric drawn from the same source. `tools` is still unread, so it is called out rather than silently dropped, and init no longer scaffolds it — warning about a field the user never chose is just noise. Also stops relying on the service to reject a missing model. Generation is billed against a deployment, and without one the request failed halfway through the command with a service error naming nothing the caller controls. It now fails before any network call, naming both ways to supply it. --- .../internal/cmd/agent_context_test.go | 149 ++++++++++++++++++ .../internal/cmd/generate.go | 146 ++++++++++++++--- .../azure.ai.evaluations/internal/cmd/init.go | 5 +- .../internal/pkg/eval_api/models.go | 34 +++- .../internal/pkg/eval_api/operations.go | 14 ++ 5 files changed, 322 insertions(+), 26 deletions(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/agent_context_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/agent_context_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/agent_context_test.go new file mode 100644 index 00000000000..5fc740610e5 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/agent_context_test.go @@ -0,0 +1,149 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "bytes" + "encoding/json" + "os" + "path/filepath" + "testing" + + "azureaieval/internal/pkg/eval_api" + "azureaieval/internal/project" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// The generation spec names an instructions file relative to itself, not to the +// working directory, so `generate --config ` reads the same file the +// author sees next to the spec. +func TestAgentContextInstructions_ResolvesRelativeToTheSpec(t *testing.T) { + dir := t.TempDir() + specDir := filepath.Join(dir, "evals") + require.NoError(t, os.MkdirAll(filepath.Join(specDir, "agent"), 0o755)) + + body := "Answer only from the product catalog." + require.NoError(t, os.WriteFile( + filepath.Join(specDir, "agent", "instructions.md"), []byte(" "+body+"\n"), 0o600)) + + cfg := &project.GenerateConfig{} + cfg.Agent.Context.Instructions = "./agent/instructions.md" + + got, err := agentContextInstructions(cfg, filepath.Join(specDir, "eval_generate.yaml")) + require.NoError(t, err) + assert.Equal(t, body, got, "the file's contents should be used, trimmed") +} + +// `init` writes the instructions path before that file exists. Treating the +// gap as an error would break the flow init itself scaffolds. +func TestAgentContextInstructions_MissingFileIsNotAnError(t *testing.T) { + dir := t.TempDir() + cfg := &project.GenerateConfig{} + cfg.Agent.Context.Instructions = "./agent/instructions.md" + + got, err := agentContextInstructions(cfg, filepath.Join(dir, "eval_generate.yaml")) + require.NoError(t, err) + assert.Empty(t, got) +} + +func TestAgentContextInstructions_UnsetIsEmpty(t *testing.T) { + got, err := agentContextInstructions(&project.GenerateConfig{}, "eval_generate.yaml") + require.NoError(t, err) + assert.Empty(t, got) +} + +// Only the newest version is read, and an agent with no published version must +// not panic the caller. +func TestAgentInstructions(t *testing.T) { + var agent eval_api.Agent + require.NoError(t, json.Unmarshal([]byte(`{ + "name": "support", + "versions": { "latest": { "version": "2", "definition": { + "model": "gpt-5-mini", + "instructions": " You are a support assistant.\n" } } } + }`), &agent)) + assert.Equal(t, "You are a support assistant.", agent.Instructions()) + + var empty eval_api.Agent + require.NoError(t, json.Unmarshal([]byte(`{"name":"x","versions":{}}`), &empty)) + assert.Empty(t, empty.Instructions(), "an agent with no published version has no instructions") + + var nilAgent *eval_api.Agent + assert.Empty(t, nilAgent.Instructions()) +} + +// Dataset generation has no model of its own; it runs against the judge model +// the spec declares. +func TestGenerationModel(t *testing.T) { + cfg := &project.GenerateConfig{} + assert.Empty(t, generationModel(cfg), "no rubric means no model to borrow") + + cfg.Generate.Rubric = &project.RubricSpec{Model: "gpt-4.1-nano"} + assert.Equal(t, "gpt-4.1-nano", generationModel(cfg)) +} + +// `tools` is accepted and ignored, so it has to be called out — the same +// reasoning as the trace fields it now shares a warning with. +func TestWarnIgnoredFields_CoversTools(t *testing.T) { + cases := []struct { + name string + build func(*project.GenerateConfig) + want []string + quiet bool + }{ + { + name: "nothing set stays silent", + build: func(*project.GenerateConfig) {}, + quiet: true, + }, + { + name: "tools alone", + build: func(c *project.GenerateConfig) { c.Agent.Context.Tools = "./agent/tools.json" }, + want: []string{"agent.context.tools", "has no effect"}, + }, + { + name: "tools and a trace field agree in number", + build: func(c *project.GenerateConfig) { + c.Agent.Context.Tools = "./agent/tools.json" + c.Agent.Context.Traces = &project.TraceSpec{Source: "app-insights"} + }, + want: []string{"agent.context.traces.source", "agent.context.tools", "have no effect"}, + }, + { + name: "a window alone is honored, so no warning", + build: func(c *project.GenerateConfig) { + c.Agent.Context.Traces = &project.TraceSpec{Window: "7d"} + }, + quiet: true, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + cfg := &project.GenerateConfig{} + tc.build(cfg) + + var buf bytes.Buffer + warnIgnoredTraceFields(cfg, &buf) + + if tc.quiet { + assert.Empty(t, buf.String()) + return + } + for _, want := range tc.want { + assert.Contains(t, buf.String(), want) + } + }) + } +} + +// init scaffolds only the context fields that are read. +func TestInitScaffold_OmitsToolsButKeepsInstructions(t *testing.T) { + cfg := buildGenerateScaffold("support-agent", "support-agent-quality", "gpt-4.1-nano") + assert.Equal(t, "./agent/instructions.md", cfg.Agent.Context.Instructions) + assert.Empty(t, cfg.Agent.Context.Tools, + "scaffolding a field nothing reads would warn on every default init") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go index eff801fc119..a3508ade263 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go @@ -6,6 +6,7 @@ package cmd import ( "context" "encoding/json" + "errors" "fmt" "io" "os" @@ -71,12 +72,30 @@ func newGenerateCommand() *cobra.Command { warnIgnoredTraceFields(cfg, out) } + // Both jobs are billed against a model deployment. Checking before + // any network work keeps the failure at the flag the caller can act + // on, instead of a service rejection partway through the command. + generatingRubric := len(evaluators) == 0 && cfg.Generate.Rubric != nil + generatingDataset := datasetFlag == "" && cfg.Generate.Dataset != nil + if (generatingRubric || generatingDataset) && generationModel(cfg) == "" { + return fmt.Errorf( + "a model deployment is required to generate: pass --eval-model, " + + "or set generate.rubric.model in the generation spec") + } + ec, err := newEvalContext(ctx, endpointFlg) if err != nil { return err } defer ec.Close() + instruction, err = ec.resolveGenerationInstruction( + ctx, cfg, instruction, configPath, out, isJSON(cmd), + ) + if err != nil { + return err + } + baseDir := filepath.Dir(deployPath) var datasetRefs, evaluatorRefs []project.ArtifactRef @@ -154,42 +173,39 @@ func newGenerateCommand() *cobra.Command { return cmd } -// warnIgnoredTraceFields reports trace settings that are accepted but have no -// effect yet. +// warnIgnoredTraceFields reports generation settings that are accepted but have +// no effect yet. // // The generation API takes a day window and nothing else, so `source` and // `sample` are parsed and dropped. Silently discarding them is worse than not // accepting them: the author believes they narrowed the trace selection when -// nothing changed. +// nothing changed. `agent.context.tools` is in the same position — nothing +// reads it, and only the instructions half of the agent's context is used. func warnIgnoredTraceFields(cfg *project.GenerateConfig, out io.Writer) { - traces := cfg.Agent.Context.Traces - if traces == nil { - return - } + var fields []string - var ignored []string - if traces.Source != "" { - ignored = append(ignored, "source") + if traces := cfg.Agent.Context.Traces; traces != nil { + if traces.Source != "" { + fields = append(fields, "agent.context.traces.source") + } + if traces.Sample > 0 { + fields = append(fields, "agent.context.traces.sample") + } } - if traces.Sample > 0 { - ignored = append(ignored, "sample") + if cfg.Agent.Context.Tools != "" { + fields = append(fields, "agent.context.tools") } - if len(ignored) == 0 { + if len(fields) == 0 { return } - fields := make([]string, 0, len(ignored)) - for _, name := range ignored { - fields = append(fields, "agent.context.traces."+name) - } - verb := "has" if len(fields) > 1 { verb = "have" } fmt.Fprintf(out, - "warning: %s %s no effect yet; trace seeding uses only `window`. "+ - "Trace selection lands with the trace scenarios.\n", + "warning: %s %s no effect yet; generation is seeded from the agent's "+ + "instructions and, when a window is set, its traces.\n", strings.Join(fields, " and "), verb) } @@ -214,6 +230,91 @@ func resolveInstruction(inline, path string) (string, error) { return text, nil } +// generationModel returns the deployment both generation jobs run against. +// +// Dataset generation has no model of its own: the spec carries one judge model +// and both jobs use it. +func generationModel(cfg *project.GenerateConfig) string { + if cfg.Generate.Rubric == nil { + return "" + } + return cfg.Generate.Rubric.Model +} + +// agentContextInstructions reads the instructions named by +// `agent.context.instructions`, relative to the spec that declared them. +// +// A missing file is not an error. `init` writes the field pointing at a +// conventional path before that file exists, so treating its absence as a +// failure would break the flow it scaffolds. +func agentContextInstructions(cfg *project.GenerateConfig, configPath string) (string, error) { + named := cfg.Agent.Context.Instructions + if named == "" { + return "", nil + } + + path := named + if !filepath.IsAbs(path) { + path = filepath.Join(filepath.Dir(configPath), filepath.FromSlash(named)) + } + raw, err := os.ReadFile(path) + if errors.Is(err, os.ErrNotExist) { + return "", nil + } + if err != nil { + return "", fmt.Errorf("reading agent.context.instructions %q: %w", named, err) + } + return strings.TrimSpace(string(raw)), nil +} + +// resolveGenerationInstruction decides what generation is seeded from. +// +// The service accepts an agent source that is meant to pull the agent's own +// instructions, but it fails for every agent, so the agent's context is read +// here instead. In precedence order: what the caller passed, then the +// instructions file the spec names, then the agent's published instructions. +// +// The last step is what makes `generate` work with no authored input at all, +// which is the flow `init` sets up. +func (ec *evalContext) resolveGenerationInstruction( + ctx context.Context, + cfg *project.GenerateConfig, + explicit, configPath string, + out io.Writer, + quiet bool, +) (string, error) { + if explicit != "" { + return explicit, nil + } + + fromFile, err := agentContextInstructions(cfg, configPath) + if err != nil { + return "", err + } + if fromFile != "" { + return fromFile, nil + } + + if cfg.Agent.Name == "" { + return "", nil + } + agent, err := ec.evalClient.GetAgent(ctx, cfg.Agent.Name, ProjectEndpointAPIVersion) + if err != nil { + // Generation can still proceed from the agent source alone, so a + // failure to read the agent is reported without stopping. + if !quiet { + fmt.Fprintf(out, " warning: could not read agent %q for generation context: %v\n", + cfg.Agent.Name, err) + } + return "", nil + } + instructions := agent.Instructions() + if instructions != "" && !quiet { + fmt.Fprintf(out, " Seeding generation from the instructions of agent %q.\n", cfg.Agent.Name) + } + return instructions, nil +} + // resolveGenerateConfig loads the spec when present, then layers flags on top. // A missing file is not an error: flags alone are sufficient. func resolveGenerateConfig( @@ -325,10 +426,7 @@ func (ec *evalContext) generateDataset( sources := eval_api.BuildGenerationSources( "agent", cfg.Agent.Name, "", instruction, traceOptions(cfg), ) - model := "" - if cfg.Generate.Rubric != nil { - model = cfg.Generate.Rubric.Model - } + model := generationModel(cfg) req := eval_api.NewDataGenerationJobRequest(spec.Name, model, spec.SampleSize, sources) job, err := ec.evalClient.CreateDataGenerationJob(ctx, req, DataGenerationAPIVersion) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go index 2905b7e42ca..1f62f8be3d2 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go @@ -110,8 +110,11 @@ func buildGenerateScaffold(target, rubricName, evalModel string) *project.Genera Agent: project.AgentSpec{ Name: target, Context: project.AgentContext{ + // Scaffolded even though the file does not exist yet: writing + // it overrides the agent's published instructions, which is the + // usual way to narrow what gets generated. `tools` is left out + // because nothing reads it yet. Instructions: "./agent/instructions.md", - Tools: "./agent/tools.json", }, }, Generate: project.GenerateSpec{ diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/models.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/models.go index cf51017746c..d5075eafbf1 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/models.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/models.go @@ -3,7 +3,10 @@ package eval_api -import "encoding/json" +import ( + "encoding/json" + "strings" +) // --------------------------------------------------------------------------- // Data Generation Jobs @@ -43,6 +46,35 @@ type GenerationSource struct { StartTime int64 `json:"start_time,omitempty"` } +// Agent is the part of a catalog agent that describes what it does. +// +// An agent is returned with its versions inlined rather than as a list, and +// only `latest` is populated on a plain read. +type Agent struct { + Name string `json:"name"` + Versions struct { + Latest *AgentVersion `json:"latest"` + } `json:"versions"` +} + +// AgentVersion is one published revision of an agent. +type AgentVersion struct { + Version string `json:"version"` + Definition struct { + Model string `json:"model"` + Instructions string `json:"instructions"` + } `json:"definition"` +} + +// Instructions returns the newest version's system prompt, or "" when the agent +// has no published version. +func (a *Agent) Instructions() string { + if a == nil || a.Versions.Latest == nil { + return "" + } + return strings.TrimSpace(a.Versions.Latest.Definition.Instructions) +} + // GenerationJob is the response for data and evaluator generation job operations. type GenerationJob struct { ID string `json:"id"` diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/operations.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/operations.go index d04a3104636..677e525f59f 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/operations.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/operations.go @@ -30,6 +30,7 @@ const ( pathEvaluators = "/evaluators" pathDatasets = "/datasets" pathOpenAIEvals = "/openai/v1/evals" + pathAgents = "/agents" ) // EvalClient provides methods for interacting with the Azure AI eval APIs. @@ -114,6 +115,19 @@ func (c *EvalClient) GetEvaluatorGenerationJob( return doRequestTyped[GenerationJob](c, ctx, http.MethodGet, path, nil, nil, apiVersion) } +// GetAgent reads an agent from the project's catalog. +// +// Only the newest version is returned, which is the one generation is seeded +// from: the point is to describe what the agent does now. +func (c *EvalClient) GetAgent( + ctx context.Context, + name string, + apiVersion string, +) (*Agent, error) { + path := pathAgents + "/" + url.PathEscape(name) + return doRequestTyped[Agent](c, ctx, http.MethodGet, path, nil, nil, apiVersion) +} + // CreateEvaluatorVersion creates a new version of a named evaluator. // The body should be the full evaluator JSON with the definition field updated. func (c *EvalClient) CreateEvaluatorVersion( From b9a30781f7ce329190fa8f272d7db4109d8f2530 Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 28 Jul 2026 11:20:42 +0300 Subject: [PATCH 037/320] feat(azure.ai.evaluations): run an eval group on a schedule MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes the scheduling half of M2. A schedule pairs a trigger with an eval group and the run to repeat, since the group holds only its testing criteria and the target and dataset travel with the run. `set` builds that run the same way `run` does, so a scheduled evaluation is the one already being run by hand. Triggers come from flags rather than raw JSON: --cron, or --every with the qualifier that period actually reads. Passing a qualifier from another period is rejected instead of dropped, because the service ignores what it does not recognise and the author would never learn the selection did not narrow. The route took finding. GET answers on both /schedules and /evaluations/schedules, but neither accepts a POST; creation is a PUT on a named resource, and everything else 404s. Three service behaviours shape the command, all measured rather than assumed: A project holds one schedule at a time. A second is refused with a 400 that carries no body at all, so the count is what explains it — the error now names the schedule already there and how to remove it. This also corrected an earlier reading of mine: a run of "required field" probes all failed for this reason, not the one I first recorded. A PUT over an existing schedule is accepted, echoes the new body, and changes nothing. Neither a new cron expression nor a new trigger type takes effect. So `set` refuses an existing name rather than reporting an edit that did not happen. Recreating under a name that was used before never leaves Creating and then cannot be deleted, so the refusal points at a different name rather than offering to replace in place. Deleting also needed care: a schedule mid-provision refuses, as 409 while busy or 404 because its trigger does not exist yet, so delete waits for it to settle. Also fixes a latent client bug this surfaced: 204 was not in the set of accepted statuses, so every successful delete was reported as a failure. --- .../azure.ai.evaluations/internal/cmd/root.go | 1 + .../internal/cmd/schedule.go | 595 ++++++++++++++++++ .../internal/cmd/schedule_test.go | 155 +++++ .../internal/pkg/eval_api/operations.go | 6 +- .../internal/pkg/eval_api/schedules.go | 175 ++++++ 5 files changed, 931 insertions(+), 1 deletion(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/schedule.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/schedule_test.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/schedules.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/root.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/root.go index 40724c5a0a7..118a831fdde 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/root.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/root.go @@ -51,6 +51,7 @@ func NewRootCommand() *cobra.Command { newRunCommand(), newEvaluatorCommand(), newResultsCommand(), + newScheduleCommand(), newListenCommand(), ) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/schedule.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/schedule.go new file mode 100644 index 00000000000..e0afd5c468b --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/schedule.go @@ -0,0 +1,595 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "context" + "fmt" + "strconv" + "strings" + "time" + + "azureaieval/internal/pkg/eval_api" + "azureaieval/internal/project" + + "github.com/spf13/cobra" +) + +// newScheduleCommand groups the recurring-evaluation commands. +func newScheduleCommand() *cobra.Command { + cmd := &cobra.Command{ + Use: "schedule", + Short: "Run an eval group on a schedule.", + } + cmd.AddCommand( + newScheduleSetCommand(), + newScheduleListCommand(), + newScheduleShowCommand(), + newScheduleDeleteCommand(), + ) + return cmd +} + +// newScheduleSetCommand creates the schedule that runs an eval group. +// +// It does not update. The service accepts a PUT over an existing schedule, +// echoes the new body and keeps the old trigger, so an in-place edit would +// report a change that did not happen. Recreating under the same name is not +// an escape either: the replacement never leaves Creating and cannot then be +// deleted. So an existing schedule is refused, and changing one means deleting +// it and creating another under a different name. +func newScheduleSetCommand() *cobra.Command { + var ( + configPath string + groupName string + evalID string + name string + description string + cron string + every string + interval int + atHours []int + onDays []string + onDaysOfMon []int + atTime string + timezone string + startTime string + endTime string + disabled bool + level string + maxSamples int + endpointFlg string + ) + + cmd := &cobra.Command{ + Use: "set [eval-id]", + Short: "Create the schedule that runs an eval group.", + Args: cobra.MaximumNArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + ctx := cmd.Context() + out := cmd.OutOrStdout() + + if len(args) == 1 { + evalID = args[0] + } + + trigger, err := buildTrigger(triggerFlags{ + cron: cron, + every: every, + interval: interval, + atHours: atHours, + onDays: onDays, + onDaysOfMon: onDaysOfMon, + atTime: atTime, + timezone: timezone, + startTime: startTime, + endTime: endTime, + }) + if err != nil { + return err + } + + ec, err := newEvalContext(ctx, endpointFlg) + if err != nil { + return err + } + defer ec.Close() + + // Same resolution as `run`: the config names the group unless + // --eval-id bypasses it, and the run payload carries the target + // and dataset because the group holds neither. + var group *project.EvalGroup + var dataSource *eval_api.EvalRunDataSource + if evalID == "" { + cfg, err := project.LoadEvalConfig(configPath) + if err != nil { + return err + } + if err := cfg.Validate(); err != nil { + return err + } + group, err = cfg.ResolveGroup(groupName) + if err != nil { + return err + } + if err := ec.checkDatasetRegistered(ctx, cfg, group, configPath); err != nil { + return err + } + evalID, err = ec.resolveEvalGroupID( + ctx, group, configPath, resolveLevel(level, group), out, isJSON(cmd)) + if err != nil { + return err + } + dataSource, err = buildRunDataSource( + group, configPath, resolveMaxSamples(maxSamples, group)) + if err != nil { + return err + } + } else { + dataSource, err = ec.reuseDataSourceFromLastRun(ctx, evalID) + if err != nil { + return err + } + } + + if name == "" { + name = defaultScheduleName(group) + } + if description == "" { + description = fmt.Sprintf("Scheduled evaluation of %s.", evalID) + } + + // An existing schedule cannot be edited: the service takes the PUT + // and ignores it. Recreating under the same name is worse — the + // replacement sticks in Creating and cannot be deleted — so the + // only safe answer is a different name. + if existing, err := ec.evalClient.GetSchedule( + ctx, name, ProjectEndpointAPIVersion); err == nil && existing != nil && existing.ID != "" { + return fmt.Errorf( + "schedule %q already exists, and the service ignores edits to it. "+ + "Delete it with `azd ai eval schedule delete %s` and create the new "+ + "one under a different name; reusing this one leaves it stuck", + name, name) + } + + metadata := map[string]string{} + if lvl := resolveLevel(level, group); lvl != "" { + metadata["evaluation_level"] = lvl + } + + schedule := &eval_api.Schedule{ + DisplayName: name, + Description: description, + Enabled: !disabled, + Trigger: trigger, + Task: &eval_api.ScheduleTask{ + Type: eval_api.ScheduleTaskEvaluation, + EvalID: evalID, + EvalRun: &eval_api.CreateOpenAIEvalRunRequest{ + Name: name, + DataSource: dataSource, + Metadata: metadata, + }, + }, + } + + saved, err := ec.evalClient.PutSchedule(ctx, name, schedule, ProjectEndpointAPIVersion) + if err != nil { + return explainScheduleFailure(ctx, ec, name, err) + } + + if isJSON(cmd) { + return emitJSON(out, saved) + } + state := "enabled" + if !saved.Enabled { + state = "disabled" + } + fmt.Fprintf(out, "Schedule %s (%s) runs %s on %s\n", + saved.ID, state, saved.Summary(), evalID) + return nil + }, + } + + cmd.Flags().StringVar(&configPath, "config", project.DefaultDeployConfig, + "Path to the eval deployment config.") + cmd.Flags().StringVar(&groupName, "eval-group", "", "Which evalGroups entry to schedule.") + cmd.Flags().StringVar(&evalID, "eval-id", "", "Schedule an existing eval group by id, ignoring config.") + cmd.Flags().StringVar(&name, "name", "", "Schedule name. Defaults to the group name.") + cmd.Flags().StringVar(&description, "description", "", "Schedule description.") + cmd.Flags().StringVar(&cron, "cron", "", `Cron expression, for example "0 9 * * *".`) + cmd.Flags().StringVar(&every, "every", "", + "Recur hourly, daily, weekly or monthly.") + cmd.Flags().IntVar(&interval, "interval", 0, "Repeat every N periods of --every. Defaults to 1.") + cmd.Flags().IntSliceVar(&atHours, "at", nil, "Hours of the day for --every daily, 0-23.") + cmd.Flags().StringSliceVar(&onDays, "on", nil, "Days of the week for --every weekly, for example Monday.") + cmd.Flags().IntSliceVar(&onDaysOfMon, "on-day", nil, "Days of the month for --every monthly, 1-31.") + cmd.Flags().StringVar(&atTime, "at-time", "", "Run once at this RFC3339 time.") + cmd.Flags().StringVar(&timezone, "timezone", "", "Timezone for the trigger. Defaults to UTC.") + cmd.Flags().StringVar(&startTime, "start-time", "", "RFC3339 time before which the schedule does not fire.") + cmd.Flags().StringVar(&endTime, "end-time", "", "RFC3339 time after which the schedule stops firing.") + cmd.Flags().BoolVar(&disabled, "disabled", false, "Create the schedule without enabling it.") + cmd.Flags().StringVar(&level, "level", "", "Evaluation level for the scheduled runs.") + cmd.Flags().IntVar(&maxSamples, "max-samples", 0, "Cap rows sent from a local dataset file.") + cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") + cmd.MarkFlagsMutuallyExclusive("cron", "every", "at-time") + + return cmd +} + +func newScheduleListCommand() *cobra.Command { + var endpointFlg string + + cmd := &cobra.Command{ + Use: "list", + Short: "List the project's schedules.", + RunE: func(cmd *cobra.Command, args []string) error { + ctx := cmd.Context() + out := cmd.OutOrStdout() + + ec, err := newEvalContext(ctx, endpointFlg) + if err != nil { + return err + } + defer ec.Close() + + list, err := ec.evalClient.ListSchedules(ctx, ProjectEndpointAPIVersion) + if err != nil { + return fmt.Errorf("listing schedules: %w", err) + } + if isJSON(cmd) { + return emitJSON(out, list) + } + if list == nil || len(list.Value) == 0 { + fmt.Fprintln(out, "No schedules.") + return nil + } + + rows := make([][]string, 0, len(list.Value)) + for i := range list.Value { + s := &list.Value[i] + evalGroup := "" + if s.Task != nil { + evalGroup = s.Task.EvalID + } + rows = append(rows, []string{ + s.ID, + strconv.FormatBool(s.Enabled), + s.ProvisioningStatus, + s.Summary(), + evalGroup, + }) + } + return emitTable(out, + []string{"NAME", "ENABLED", "STATUS", "TRIGGER", "EVAL GROUP"}, rows) + }, + } + cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") + return cmd +} + +func newScheduleShowCommand() *cobra.Command { + var ( + name string + endpointFlg string + ) + + cmd := &cobra.Command{ + Use: "show [name]", + Short: "Show one schedule.", + Args: cobra.MaximumNArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + ctx := cmd.Context() + out := cmd.OutOrStdout() + + if len(args) == 1 { + name = args[0] + } + if name == "" { + return requireFlag("name") + } + + ec, err := newEvalContext(ctx, endpointFlg) + if err != nil { + return err + } + defer ec.Close() + + s, err := ec.evalClient.GetSchedule(ctx, name, ProjectEndpointAPIVersion) + if err != nil { + return fmt.Errorf("reading schedule %q: %w", name, err) + } + if isJSON(cmd) { + return emitJSON(out, s) + } + + fmt.Fprintf(out, "Schedule %s\n", s.ID) + fmt.Fprintf(out, " enabled: %t\n", s.Enabled) + fmt.Fprintf(out, " status: %s\n", s.ProvisioningStatus) + fmt.Fprintf(out, " trigger: %s\n", s.Summary()) + if s.Trigger != nil && s.Trigger.Timezone != "" { + fmt.Fprintf(out, " timezone: %s\n", s.Trigger.Timezone) + } + if s.Task != nil { + fmt.Fprintf(out, " group: %s\n", s.Task.EvalID) + } + if s.Description != "" { + fmt.Fprintf(out, " about: %s\n", s.Description) + } + return nil + }, + } + cmd.Flags().StringVar(&name, "name", "", "Schedule name.") + cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") + return cmd +} + +func newScheduleDeleteCommand() *cobra.Command { + var ( + name string + endpointFlg string + ) + + cmd := &cobra.Command{ + Use: "delete [name]", + Short: "Delete a schedule.", + Args: cobra.MaximumNArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + ctx := cmd.Context() + out := cmd.OutOrStdout() + + if len(args) == 1 { + name = args[0] + } + if name == "" { + return requireFlag("name") + } + + ec, err := newEvalContext(ctx, endpointFlg) + if err != nil { + return err + } + defer ec.Close() + + if err := ec.deleteScheduleWhenSettled(ctx, name); err != nil { + return fmt.Errorf("deleting schedule %q: %w", name, err) + } + fmt.Fprintf(out, "Deleted schedule %s\n", name) + return nil + }, + } + cmd.Flags().StringVar(&name, "name", "", "Schedule name.") + cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") + return cmd +} + +// triggerFlags carries the schedule's timing flags so buildTrigger can be +// tested without a command. +type triggerFlags struct { + cron string + every string + interval int + atHours []int + onDays []string + onDaysOfMon []int + atTime string + timezone string + startTime string + endTime string +} + +// buildTrigger turns the timing flags into the trigger the API expects. +func buildTrigger(f triggerFlags) (*eval_api.ScheduleTrigger, error) { + tz := f.timezone + if tz == "" { + tz = "UTC" + } + + switch { + case f.cron != "": + return &eval_api.ScheduleTrigger{ + Type: eval_api.TriggerCron, + Expression: f.cron, + StartTime: f.startTime, + EndTime: f.endTime, + Timezone: tz, + }, nil + + case f.atTime != "": + if _, err := time.Parse(time.RFC3339, f.atTime); err != nil { + return nil, fmt.Errorf("--at-time %q is not an RFC3339 time", f.atTime) + } + return &eval_api.ScheduleTrigger{ + Type: eval_api.TriggerOneTime, + ScheduledTime: f.atTime, + Timezone: tz, + }, nil + + case f.every != "": + pattern, err := buildRecurrence(f) + if err != nil { + return nil, err + } + interval := f.interval + if interval <= 0 { + interval = 1 + } + return &eval_api.ScheduleTrigger{ + Type: eval_api.TriggerRecurrence, + Schedule: pattern, + Interval: interval, + StartTime: f.startTime, + EndTime: f.endTime, + Timezone: tz, + }, nil + } + + return nil, fmt.Errorf("a schedule needs a trigger: pass --cron, --every or --at-time") +} + +// buildRecurrence maps --every and its qualifiers onto a recurrence pattern. +// +// Each period reads only its own qualifier, so passing one that does not apply +// is rejected rather than dropped. +func buildRecurrence(f triggerFlags) (*eval_api.RecurrencePattern, error) { + period := strings.ToLower(strings.TrimSpace(f.every)) + + reject := func(flag, applies string) error { + return fmt.Errorf("--%s applies to --every %s, not %s", flag, applies, period) + } + + switch period { + case "hourly": + if len(f.atHours) > 0 { + return nil, reject("at", "daily") + } + if len(f.onDays) > 0 { + return nil, reject("on", "weekly") + } + if len(f.onDaysOfMon) > 0 { + return nil, reject("on-day", "monthly") + } + return &eval_api.RecurrencePattern{Type: eval_api.RecurrenceHourly}, nil + + case "daily": + if len(f.onDays) > 0 { + return nil, reject("on", "weekly") + } + if len(f.onDaysOfMon) > 0 { + return nil, reject("on-day", "monthly") + } + for _, h := range f.atHours { + if h < 0 || h > 23 { + return nil, fmt.Errorf("--at %d is not an hour of the day (0-23)", h) + } + } + return &eval_api.RecurrencePattern{Type: eval_api.RecurrenceDaily, Hours: f.atHours}, nil + + case "weekly": + if len(f.atHours) > 0 { + return nil, reject("at", "daily") + } + if len(f.onDaysOfMon) > 0 { + return nil, reject("on-day", "monthly") + } + days, err := normalizeDaysOfWeek(f.onDays) + if err != nil { + return nil, err + } + return &eval_api.RecurrencePattern{Type: eval_api.RecurrenceWeekly, DaysOfWeek: days}, nil + + case "monthly": + if len(f.atHours) > 0 { + return nil, reject("at", "daily") + } + if len(f.onDays) > 0 { + return nil, reject("on", "weekly") + } + for _, d := range f.onDaysOfMon { + if d < 1 || d > 31 { + return nil, fmt.Errorf("--on-day %d is not a day of the month (1-31)", d) + } + } + return &eval_api.RecurrencePattern{Type: eval_api.RecurrenceMonthly, DaysOfMonth: f.onDaysOfMon}, nil + } + + return nil, fmt.Errorf( + "--every %q is not a recurrence: use hourly, daily, weekly or monthly", f.every) +} + +// normalizeDaysOfWeek accepts day names in any casing and returns the spelling +// the service expects. +func normalizeDaysOfWeek(days []string) ([]string, error) { + if len(days) == 0 { + return nil, nil + } + canonical := map[string]string{} + for d := time.Sunday; d <= time.Saturday; d++ { + canonical[strings.ToLower(d.String())] = d.String() + } + + out := make([]string, 0, len(days)) + for _, raw := range days { + name, ok := canonical[strings.ToLower(strings.TrimSpace(raw))] + if !ok { + return nil, fmt.Errorf("--on %q is not a day of the week", raw) + } + out = append(out, name) + } + return out, nil +} + +// defaultScheduleName derives a schedule name from the group being scheduled. +func defaultScheduleName(group *project.EvalGroup) string { + if group != nil && group.Name != "" { + return group.Name + } + return "eval-" + strconv.FormatInt(time.Now().UTC().Unix(), 10) +} + +// deleteScheduleWhenSettled removes a schedule, waiting out the window where +// the service is still provisioning it. +// +// A schedule that is mid-provision refuses the delete, and does it two ways: +// 409 while it is busy, or 404 because the trigger behind it does not exist +// yet. Either way the caller neither caused it nor can see it, so the wait +// happens here. +func (ec *evalContext) deleteScheduleWhenSettled(ctx context.Context, name string) error { + const attempts = 30 + + for i := 0; i < attempts; i++ { + s, err := ec.evalClient.GetSchedule(ctx, name, ProjectEndpointAPIVersion) + if err != nil || s == nil || s.ID == "" { + // Nothing to wait for: let the delete report what it finds. + break + } + if s.Settled() { + break + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(4 * time.Second): + } + } + + for i := 0; ; i++ { + err := ec.evalClient.DeleteSchedule(ctx, name, ProjectEndpointAPIVersion) + if err == nil || !eval_api.IsConflict(err) || i == attempts-1 { + return err + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(4 * time.Second): + } + } +} + +// explainScheduleFailure turns the service's bodiless rejection into the reason +// it actually happened. +// +// The project accepts one schedule at a time and refuses a second with a 400 +// carrying no message, so the count is what explains it. +func explainScheduleFailure( + ctx context.Context, + ec *evalContext, + name string, + cause error, +) error { + list, listErr := ec.evalClient.ListSchedules(ctx, ProjectEndpointAPIVersion) + if listErr != nil || list == nil { + return fmt.Errorf("saving schedule %q: %w", name, cause) + } + + for i := range list.Value { + if other := list.Value[i].ID; other != "" && other != name { + return fmt.Errorf( + "saving schedule %q: the project already has a schedule, %q, and only one is "+ + "allowed at a time. Delete it first with "+ + "`azd ai eval schedule delete %s`", name, other, other) + } + } + return fmt.Errorf("saving schedule %q: %w", name, cause) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/schedule_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/schedule_test.go new file mode 100644 index 00000000000..9f93207950b --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/schedule_test.go @@ -0,0 +1,155 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "testing" + + "azureaieval/internal/pkg/eval_api" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestBuildTrigger_Cron(t *testing.T) { + got, err := buildTrigger(triggerFlags{cron: "0 9 * * *"}) + require.NoError(t, err) + assert.Equal(t, eval_api.TriggerCron, got.Type) + assert.Equal(t, "0 9 * * *", got.Expression) + assert.Equal(t, "UTC", got.Timezone, "UTC unless the caller says otherwise") +} + +func TestBuildTrigger_OneTime(t *testing.T) { + got, err := buildTrigger(triggerFlags{atTime: "2026-08-01T09:00:00Z", timezone: "Europe/Dublin"}) + require.NoError(t, err) + assert.Equal(t, eval_api.TriggerOneTime, got.Type) + assert.Equal(t, "2026-08-01T09:00:00Z", got.ScheduledTime) + assert.Equal(t, "Europe/Dublin", got.Timezone) + + _, err = buildTrigger(triggerFlags{atTime: "next tuesday"}) + require.ErrorContains(t, err, "RFC3339") +} + +func TestBuildTrigger_Recurrence(t *testing.T) { + cases := []struct { + name string + flags triggerFlags + wantType string + assert func(*testing.T, *eval_api.RecurrencePattern) + }{ + { + name: "hourly", + flags: triggerFlags{every: "hourly", interval: 6}, + wantType: eval_api.RecurrenceHourly, + }, + { + name: "daily with hours", + flags: triggerFlags{every: "Daily", atHours: []int{9, 17}}, + wantType: eval_api.RecurrenceDaily, + assert: func(t *testing.T, p *eval_api.RecurrencePattern) { + assert.Equal(t, []int{9, 17}, p.Hours) + }, + }, + { + name: "weekly normalizes day casing", + flags: triggerFlags{every: "weekly", onDays: []string{"monday", "THURSDAY"}}, + wantType: eval_api.RecurrenceWeekly, + assert: func(t *testing.T, p *eval_api.RecurrencePattern) { + assert.Equal(t, []string{"Monday", "Thursday"}, p.DaysOfWeek) + }, + }, + { + name: "monthly", + flags: triggerFlags{every: "monthly", onDaysOfMon: []int{1, 15}}, + wantType: eval_api.RecurrenceMonthly, + assert: func(t *testing.T, p *eval_api.RecurrencePattern) { + assert.Equal(t, []int{1, 15}, p.DaysOfMonth) + }, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got, err := buildTrigger(tc.flags) + require.NoError(t, err) + assert.Equal(t, eval_api.TriggerRecurrence, got.Type) + require.NotNil(t, got.Schedule) + assert.Equal(t, tc.wantType, got.Schedule.Type) + if tc.assert != nil { + tc.assert(t, got.Schedule) + } + }) + } +} + +// An interval is always sent, so the service never has to infer one. +func TestBuildTrigger_IntervalDefaultsToOne(t *testing.T) { + got, err := buildTrigger(triggerFlags{every: "daily"}) + require.NoError(t, err) + assert.Equal(t, 1, got.Interval) + + got, err = buildTrigger(triggerFlags{every: "daily", interval: 3}) + require.NoError(t, err) + assert.Equal(t, 3, got.Interval) +} + +// Each period reads only its own qualifier. Accepting one that does not apply +// would drop it silently, which is the failure mode the trace fields already +// taught us to avoid. +func TestBuildTrigger_RejectsQualifiersFromAnotherPeriod(t *testing.T) { + cases := []struct { + name string + flags triggerFlags + want string + }{ + {"hours on weekly", triggerFlags{every: "weekly", atHours: []int{9}}, "--at applies to --every daily"}, + {"days on daily", triggerFlags{every: "daily", onDays: []string{"Monday"}}, "--on applies to --every weekly"}, + {"month days on hourly", triggerFlags{every: "hourly", onDaysOfMon: []int{1}}, "--on-day applies to --every monthly"}, + {"hours on monthly", triggerFlags{every: "monthly", atHours: []int{9}}, "--at applies to --every daily"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + _, err := buildTrigger(tc.flags) + require.ErrorContains(t, err, tc.want) + }) + } +} + +func TestBuildTrigger_RejectsOutOfRangeValues(t *testing.T) { + _, err := buildTrigger(triggerFlags{every: "daily", atHours: []int{24}}) + require.ErrorContains(t, err, "hour of the day") + + _, err = buildTrigger(triggerFlags{every: "monthly", onDaysOfMon: []int{0}}) + require.ErrorContains(t, err, "day of the month") + + _, err = buildTrigger(triggerFlags{every: "weekly", onDays: []string{"Caturday"}}) + require.ErrorContains(t, err, "not a day of the week") + + _, err = buildTrigger(triggerFlags{every: "fortnightly"}) + require.ErrorContains(t, err, "hourly, daily, weekly or monthly") +} + +func TestBuildTrigger_NeedsATrigger(t *testing.T) { + _, err := buildTrigger(triggerFlags{}) + require.ErrorContains(t, err, "--cron, --every or --at-time") +} + +func TestScheduleSummary(t *testing.T) { + cron := &eval_api.Schedule{Trigger: &eval_api.ScheduleTrigger{ + Type: eval_api.TriggerCron, Expression: "0 9 * * *"}} + assert.Equal(t, "cron 0 9 * * *", cron.Summary()) + + weekly := &eval_api.Schedule{Trigger: &eval_api.ScheduleTrigger{ + Type: eval_api.TriggerRecurrence, + Schedule: &eval_api.RecurrencePattern{Type: eval_api.RecurrenceWeekly}}} + assert.Equal(t, "every Weekly", weekly.Summary()) + + once := &eval_api.Schedule{Trigger: &eval_api.ScheduleTrigger{ + Type: eval_api.TriggerOneTime, ScheduledTime: "2026-08-01T09:00:00Z"}} + assert.Equal(t, "once at 2026-08-01T09:00:00Z", once.Summary()) + + var nilSchedule *eval_api.Schedule + assert.Empty(t, nilSchedule.Summary()) + assert.Empty(t, (&eval_api.Schedule{}).Summary()) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/operations.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/operations.go index 677e525f59f..5b2a74923bf 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/operations.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/operations.go @@ -330,7 +330,11 @@ func (c *EvalClient) doRequest( log.Printf("[eval_api] response status: %d", resp.StatusCode) - if !runtime.HasStatusCode(resp, http.StatusOK, http.StatusCreated, http.StatusAccepted) { + // 204 belongs here: a delete that removed the resource answers No Content, + // and treating that as a failure reports every successful delete as an + // error. doRequestTyped already tolerates the empty body. + if !runtime.HasStatusCode(resp, + http.StatusOK, http.StatusCreated, http.StatusAccepted, http.StatusNoContent) { // Restore the body so runtime.NewResponseError can read it. resp.Body = io.NopCloser(bytes.NewReader(respBody)) return nil, runtime.NewResponseError(resp) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/schedules.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/schedules.go new file mode 100644 index 00000000000..ed205bf1f79 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/schedules.go @@ -0,0 +1,175 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package eval_api + +import ( + "context" + "errors" + "net/http" + "net/url" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore" +) + +const pathSchedules = "/schedules" + +// Trigger types accepted by the schedules API. +const ( + TriggerCron = "Cron" + TriggerRecurrence = "Recurrence" + TriggerOneTime = "OneTime" +) + +// Recurrence patterns accepted under a Recurrence trigger. +const ( + RecurrenceHourly = "Hourly" + RecurrenceDaily = "Daily" + RecurrenceWeekly = "Weekly" + RecurrenceMonthly = "Monthly" +) + +// ScheduleTaskEvaluation is the only task type the service accepts today; an +// Insight task is rejected on validation. +const ScheduleTaskEvaluation = "Evaluation" + +// RecurrencePattern is the shape of a Recurrence trigger's repeat rule. Which +// fields apply depends on Type: Daily reads Hours, Weekly reads DaysOfWeek, +// Monthly reads DaysOfMonth, and Hourly reads neither. +type RecurrencePattern struct { + Type string `json:"type"` + Hours []int `json:"hours,omitempty"` + DaysOfWeek []string `json:"daysOfWeek,omitempty"` + DaysOfMonth []int `json:"daysOfMonth,omitempty"` +} + +// ScheduleTrigger says when the task runs. The discriminator is Type; the +// other fields are per-type and only one set is ever populated. +type ScheduleTrigger struct { + Type string `json:"type"` + + // Cron + Expression string `json:"expression,omitempty"` + + // Recurrence + Schedule *RecurrencePattern `json:"schedule,omitempty"` + Interval int `json:"interval,omitempty"` + + // OneTime + ScheduledTime string `json:"scheduledTime,omitempty"` + + // Cron and Recurrence + StartTime string `json:"startTime,omitempty"` + EndTime string `json:"endTime,omitempty"` + + Timezone string `json:"timezone,omitempty"` +} + +// ScheduleTask is what the trigger fires. An evaluation task needs both the +// group and the run to repeat: the group holds only its testing criteria, so +// the target and dataset travel with the run. +type ScheduleTask struct { + Type string `json:"type"` + EvalID string `json:"evalId,omitempty"` + EvalRun *CreateOpenAIEvalRunRequest `json:"evalRun,omitempty"` +} + +// Schedule is a named, project-scoped recurring evaluation. +type Schedule struct { + ID string `json:"id,omitempty"` + DisplayName string `json:"displayName,omitempty"` + Description string `json:"description,omitempty"` + Enabled bool `json:"enabled"` + ProvisioningStatus string `json:"provisioningStatus,omitempty"` + Trigger *ScheduleTrigger `json:"trigger,omitempty"` + Task *ScheduleTask `json:"task,omitempty"` + Tags map[string]string `json:"tags,omitempty"` + Properties map[string]string `json:"properties,omitempty"` + Error *JobError `json:"error,omitempty"` +} + +// Summary renders the trigger as a single line for listings. +func (s *Schedule) Summary() string { + if s == nil || s.Trigger == nil { + return "" + } + switch s.Trigger.Type { + case TriggerCron: + return "cron " + s.Trigger.Expression + case TriggerOneTime: + return "once at " + s.Trigger.ScheduledTime + case TriggerRecurrence: + if s.Trigger.Schedule == nil { + return "recurrence" + } + return "every " + s.Trigger.Schedule.Type + } + return s.Trigger.Type +} + +// ScheduleList is the response for ListSchedules. +type ScheduleList struct { + Value []Schedule `json:"value"` +} + +// Settled reports whether the schedule has finished provisioning. +// +// A schedule that is still being created refuses a delete, and does it two +// different ways: 409 while it is busy, or 404 because the trigger behind it +// does not exist yet. Waiting for it to settle avoids both. +func (s *Schedule) Settled() bool { + if s == nil { + return true + } + switch s.ProvisioningStatus { + case "Creating", "Updating", "Deleting": + return false + } + return true +} + +// PutSchedule creates or replaces a schedule. The route is keyed by name, and +// the same call updates an existing schedule in place. +func (c *EvalClient) PutSchedule( + ctx context.Context, + name string, + schedule *Schedule, + apiVersion string, +) (*Schedule, error) { + path := pathSchedules + "/" + url.PathEscape(name) + return doRequestTyped[Schedule](c, ctx, http.MethodPut, path, nil, schedule, apiVersion) +} + +// GetSchedule reads one schedule by name. +func (c *EvalClient) GetSchedule( + ctx context.Context, + name string, + apiVersion string, +) (*Schedule, error) { + path := pathSchedules + "/" + url.PathEscape(name) + return doRequestTyped[Schedule](c, ctx, http.MethodGet, path, nil, nil, apiVersion) +} + +// ListSchedules returns the project's schedules. +func (c *EvalClient) ListSchedules(ctx context.Context, apiVersion string) (*ScheduleList, error) { + return doRequestTyped[ScheduleList](c, ctx, http.MethodGet, pathSchedules, nil, nil, apiVersion) +} + +// DeleteSchedule removes a schedule by name. +func (c *EvalClient) DeleteSchedule(ctx context.Context, name string, apiVersion string) error { + path := pathSchedules + "/" + url.PathEscape(name) + _, err := c.doRequest(ctx, http.MethodDelete, path, nil, nil, apiVersion) + return err +} + +// IsConflict reports whether the service refused because the resource is busy. +// +// A schedule that is still provisioning answers 409 to a delete, which is worth +// waiting out rather than reporting. +func IsConflict(err error) bool { + var respErr *azcore.ResponseError + if !errors.As(err, &respErr) { + return false + } + return respErr.StatusCode == http.StatusConflict +} From 1c42efab828bc34048af4c2d790ae27b839f5653 Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 28 Jul 2026 11:34:50 +0300 Subject: [PATCH 038/320] feat(azure.ai.evaluations): evaluate an agent's traces, and say why a run failed Traces as a run data source were recorded as awaiting service support. They are not: azure_ai_traces is in the run data-source discriminator and the service accepts and executes it. The note was an assumption I had not re-tested, so `run --from-traces` now evaluates what the agent already did rather than asking it fresh questions from a dataset. The agent comes from the group's target, or from whatever the group ran against last when --eval-id bypasses the config, so the flag needs no argument. A window is only sent when asked for: the service defaults it, and an unset bound sent as an epoch would quietly mean 1970. Verified against the live service, which stored the payload and normalised 7d into lookback_hours 168 while honouring max_traces. The other half of this was finding the run had failed and not being told why. A failed run carries the reason, and it is usually the only actionable thing in the response, but it was dropped: the trace case explains that the agent emitted no GenAI content and what to do about it, and all the caller saw was the word "failed". Every failure path was losing this, not just traces. --- .../azure.ai.evaluations/internal/cmd/run.go | 63 +++++++++++++- .../internal/cmd/traces_test.go | 85 +++++++++++++++++++ .../internal/pkg/eval_api/models.go | 44 +++++++++- 3 files changed, 189 insertions(+), 3 deletions(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/traces_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go index a90e4994745..ca2a16647f8 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go @@ -51,6 +51,9 @@ func buildRunCommand(use, short string) *cobra.Command { runName string level string maxSamples int + fromTraces bool + traceWindow string + maxTraces int wait bool endpointFlg string ) @@ -97,9 +100,12 @@ func buildRunCommand(use, short string) *cobra.Command { // With --eval-id there is no config to read, so the pairing of // target and dataset comes from the group's previous run. var dataSource *eval_api.EvalRunDataSource - if group == nil { + switch { + case fromTraces: + dataSource, err = buildTracesDataSource(ctx, ec, group, evalID, traceWindow, maxTraces) + case group == nil: dataSource, err = ec.reuseDataSourceFromLastRun(ctx, evalID) - } else { + default: dataSource, err = buildRunDataSource( group, configPath, resolveMaxSamples(maxSamples, group)) } @@ -170,6 +176,11 @@ func buildRunCommand(use, short string) *cobra.Command { "Scoring granularity: turn or conversation. Defaults to the service default (turn).") cmd.Flags().IntVar(&maxSamples, "max-samples", 0, "Cap the rows sent from a local dataset file. Ignored for registered datasets.") + cmd.Flags().BoolVar(&fromTraces, "from-traces", false, + "Evaluate the agent's recorded traces instead of the dataset.") + cmd.Flags().StringVar(&traceWindow, "trace-window", "", + "How far back to read traces, for example 7d. Defaults to the service's window.") + cmd.Flags().IntVar(&maxTraces, "max-traces", 0, "Cap the traces evaluated.") cmd.Flags().BoolVar(&wait, "wait", true, "Block until the run reaches a terminal state.") // The spec documents --no-wait, and cobra does not derive it from a bool. var noWait bool @@ -316,6 +327,48 @@ func (ec *evalContext) reuseDataSourceFromLastRun( return list.Data[0].DataSource, nil } +// buildTracesDataSource evaluates what the agent has already done, rather than +// asking it fresh questions from a dataset. +// +// The service reads the traces from Application Insights, so the agent has to +// be emitting gen_ai.input.messages / gen_ai.output.messages for anything to be +// found; when it is not, the run fails with the service saying so. +func buildTracesDataSource( + ctx context.Context, + ec *evalContext, + group *project.EvalGroup, + evalID, window string, + maxTraces int, +) (*eval_api.EvalRunDataSource, error) { + agent := "" + switch { + case group != nil && group.Target != nil: + agent = group.Target.Name + default: + // With --eval-id there is no config, so the agent comes from whatever + // the group ran against last. + last, err := ec.reuseDataSourceFromLastRun(ctx, evalID) + if err != nil { + return nil, err + } + if last != nil && last.Target != nil { + agent = last.Target.Name + } + } + if agent == "" { + return nil, fmt.Errorf( + "--from-traces needs to know whose traces to read, and the eval group does not " + + "name an agent. Declare target.type: agent on the group") + } + + var start, end time.Time + if days := parseWindowDays(window); days > 0 { + end = time.Now().UTC() + start = end.AddDate(0, 0, -days) + } + return eval_api.NewTracesDataSource(agent, start, end, maxTraces), nil +} + // buildRunDataSource binds the dataset to the run. The eval group carries no // dataset today, so it is supplied here. func buildRunDataSource( @@ -492,6 +545,12 @@ func (ec *evalContext) pollRun( func renderRun(out interface{ Write([]byte) (int, error) }, run *eval_api.OpenAIEvalRun) error { fmt.Fprintf(out, "\nRun %s finished with status %s\n", run.ID, run.Status) + // A run that failed carries why, and it is usually the only actionable + // thing in the response — dropping it leaves the caller with just the word + // "failed". + if why := run.Failure(); why != "" { + fmt.Fprintf(out, " %s\n", why) + } if run.ReportURL != "" { fmt.Fprintf(out, "Report: %s\n", run.ReportURL) } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/traces_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/traces_test.go new file mode 100644 index 00000000000..dacbdd02104 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/traces_test.go @@ -0,0 +1,85 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "bytes" + "encoding/json" + "testing" + "time" + + "azureaieval/internal/pkg/eval_api" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// A window is only sent when one was asked for; the service defaults it +// otherwise, and an epoch-zero bound would silently mean 1970. +func TestNewTracesDataSource_OmitsAnUnsetWindow(t *testing.T) { + ds := eval_api.NewTracesDataSource("support-agent", time.Time{}, time.Time{}, 0) + assert.Equal(t, eval_api.EvalRunDataSourceTypeTraces, ds.Type) + assert.Equal(t, "support-agent", ds.AgentName) + + raw, err := json.Marshal(ds) + require.NoError(t, err) + body := string(raw) + assert.NotContains(t, body, "start_time") + assert.NotContains(t, body, "end_time") + assert.NotContains(t, body, "max_traces") + assert.NotContains(t, body, "input_messages", "traces carry no template") +} + +func TestNewTracesDataSource_SendsTheWindowItWasGiven(t *testing.T) { + end := time.Date(2026, 7, 28, 12, 0, 0, 0, time.UTC) + start := end.AddDate(0, 0, -7) + + ds := eval_api.NewTracesDataSource("support-agent", start, end, 25) + assert.Equal(t, start.Unix(), ds.StartTime) + assert.Equal(t, end.Unix(), ds.EndTime) + assert.Equal(t, 25, ds.MaxTraces) +} + +// The reason a run failed is the only actionable part of the response, so it +// has to survive into the output. +func TestRunFailureMessage(t *testing.T) { + var run eval_api.OpenAIEvalRun + require.NoError(t, json.Unmarshal([]byte(`{ + "id": "evalrun_x", "status": "failed", + "error": { "code": "UserError", "message": " No trace data found for agent_name 'a'. " } + }`), &run)) + assert.Equal(t, "No trace data found for agent_name 'a'.", run.Failure()) + + // The field is present and null-valued on success, so presence alone + // must not read as failure. + var ok eval_api.OpenAIEvalRun + require.NoError(t, json.Unmarshal([]byte(`{ + "id": "evalrun_y", "status": "completed", + "error": { "code": null, "message": null } + }`), &ok)) + assert.Empty(t, ok.Failure()) + + var absent eval_api.OpenAIEvalRun + require.NoError(t, json.Unmarshal([]byte(`{"id":"evalrun_z","status":"completed"}`), &absent)) + assert.Empty(t, absent.Failure()) + + var nilRun *eval_api.OpenAIEvalRun + assert.Empty(t, nilRun.Failure()) +} + +func TestRenderRun_ShowsTheFailureReason(t *testing.T) { + var buf bytes.Buffer + run := &eval_api.OpenAIEvalRun{ + ID: "evalrun_x", + Status: "failed", + Error: &eval_api.JobError{Code: "UserError", Message: "No trace data found."}, + } + require.NoError(t, renderRun(&buf, run)) + assert.Contains(t, buf.String(), "failed") + assert.Contains(t, buf.String(), "No trace data found.") + + var clean bytes.Buffer + require.NoError(t, renderRun(&clean, &eval_api.OpenAIEvalRun{ID: "evalrun_y", Status: "completed"})) + assert.NotContains(t, clean.String(), " \n", "a successful run gains no blank reason line") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/models.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/models.go index d5075eafbf1..92b7f2817df 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/models.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/models.go @@ -6,6 +6,7 @@ package eval_api import ( "encoding/json" "strings" + "time" ) // --------------------------------------------------------------------------- @@ -315,6 +316,11 @@ type EvalRunDataSourceType string const ( // EvalRunDataSourceTypeAgentTarget is the data source type for agent target completions. EvalRunDataSourceTypeAgentTarget EvalRunDataSourceType = "azure_ai_target_completions" + + // EvalRunDataSourceTypeTraces evaluates an agent's recorded traces instead of + // a dataset. The service reads them from Application Insights, so the agent + // must be emitting gen_ai.input.messages / gen_ai.output.messages. + EvalRunDataSourceTypeTraces EvalRunDataSourceType = "azure_ai_traces" ) // EvalRunDataContentType defines the source type for eval run data content. @@ -331,6 +337,12 @@ type EvalRunDataSource struct { InputMessages *EvalRunInputMessages `json:"input_messages,omitempty"` Source *EvalRunDataContent `json:"source,omitempty"` Target *EvalRunTarget `json:"target,omitempty"` + + // Traces only. The window defaults to the last seven days when left unset. + AgentName string `json:"agent_name,omitempty"` + StartTime int64 `json:"start_time,omitempty"` + EndTime int64 `json:"end_time,omitempty"` + MaxTraces int `json:"max_traces,omitempty"` } // EvalRunInputMessages describes how input messages are constructed from dataset items. @@ -385,6 +397,25 @@ func NewAgentTargetDataSource(agentName string, agentVersion *string) *EvalRunDa } } +// NewTracesDataSource evaluates an agent's recorded traces instead of a dataset. +// +// The service reads them from Application Insights and defaults to the last +// seven days, so a zero window is left off rather than sent as an epoch. +func NewTracesDataSource(agentName string, start, end time.Time, maxTraces int) *EvalRunDataSource { + ds := &EvalRunDataSource{ + Type: EvalRunDataSourceTypeTraces, + AgentName: agentName, + MaxTraces: maxTraces, + } + if !start.IsZero() { + ds.StartTime = start.Unix() + } + if !end.IsZero() { + ds.EndTime = end.Unix() + } + return ds +} + // SetFileContent sets the data source to use inline file content. func (ds *EvalRunDataSource) SetFileContent(items []map[string]any) { ds.Source = &EvalRunDataContent{ @@ -417,7 +448,18 @@ type OpenAIEvalRun struct { // Result summary ResultCounts *EvalRunResultCounts `json:"result_counts,omitempty"` PerTestingCriteria []EvalRunCriteriaResult `json:"per_testing_criteria_results,omitempty"` - Error any `json:"error,omitempty"` + Error *JobError `json:"error,omitempty"` +} + +// Failure returns why the run failed, or "" when it did not. +// +// The field is always present and its members are null on success, so its +// presence says nothing on its own. +func (r *OpenAIEvalRun) Failure() string { + if r == nil || r.Error == nil { + return "" + } + return strings.TrimSpace(r.Error.Message) } // EvalRunResultCounts holds pass/fail/error/skip counts for a run. From 03c7026d24d5041337e994f1d21b27435ec0fe21 Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 28 Jul 2026 11:40:57 +0300 Subject: [PATCH 039/320] fix(azure.ai.evaluations): send the trace window in the field the service reads --trace-window was being dropped. The traces data source has no start bound: its window is lookback_hours, and a start_time is accepted and discarded, leaving the default seven days in place. It looked like it worked because the value I first tested with, 7d, is the default. Asking for 30d stored lookback_hours 168 and queried a week, which is the failure mode this extension has been fixing everywhere else: a field accepted, ignored, and never reported. Now sends lookback_hours, verified against the service, which stored 720 and queried 2026-06-28 to 2026-07-28. Found by reading the service's own contract rather than the response echo. The echo was no help: it reflected a well-formed request whose window had already been replaced by the default. --- .../azure.ai.evaluations/internal/cmd/run.go | 7 ++--- .../internal/cmd/traces_test.go | 26 +++++++++------- .../internal/pkg/eval_api/models.go | 30 ++++++++++--------- 3 files changed, 35 insertions(+), 28 deletions(-) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go index ca2a16647f8..436b94ae456 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go @@ -361,12 +361,11 @@ func buildTracesDataSource( "name an agent. Declare target.type: agent on the group") } - var start, end time.Time + var lookbackHours int if days := parseWindowDays(window); days > 0 { - end = time.Now().UTC() - start = end.AddDate(0, 0, -days) + lookbackHours = days * 24 } - return eval_api.NewTracesDataSource(agent, start, end, maxTraces), nil + return eval_api.NewTracesDataSource(agent, lookbackHours, time.Time{}, maxTraces), nil } // buildRunDataSource binds the dataset to the run. The eval group carries no diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/traces_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/traces_test.go index dacbdd02104..75a175d6d3f 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/traces_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/traces_test.go @@ -16,29 +16,35 @@ import ( ) // A window is only sent when one was asked for; the service defaults it -// otherwise, and an epoch-zero bound would silently mean 1970. +// otherwise. func TestNewTracesDataSource_OmitsAnUnsetWindow(t *testing.T) { - ds := eval_api.NewTracesDataSource("support-agent", time.Time{}, time.Time{}, 0) + ds := eval_api.NewTracesDataSource("support-agent", 0, time.Time{}, 0) assert.Equal(t, eval_api.EvalRunDataSourceTypeTraces, ds.Type) assert.Equal(t, "support-agent", ds.AgentName) raw, err := json.Marshal(ds) require.NoError(t, err) body := string(raw) - assert.NotContains(t, body, "start_time") + assert.NotContains(t, body, "lookback_hours") assert.NotContains(t, body, "end_time") assert.NotContains(t, body, "max_traces") assert.NotContains(t, body, "input_messages", "traces carry no template") } -func TestNewTracesDataSource_SendsTheWindowItWasGiven(t *testing.T) { - end := time.Date(2026, 7, 28, 12, 0, 0, 0, time.UTC) - start := end.AddDate(0, 0, -7) - - ds := eval_api.NewTracesDataSource("support-agent", start, end, 25) - assert.Equal(t, start.Unix(), ds.StartTime) - assert.Equal(t, end.Unix(), ds.EndTime) +// The service reads `lookback_hours` and has no start bound. Sending a +// start_time is accepted and dropped, which silently leaves the default seven +// days in place, so the window has to travel as hours. +func TestNewTracesDataSource_SendsAWindowTheServiceReads(t *testing.T) { + ds := eval_api.NewTracesDataSource("support-agent", 30*24, time.Time{}, 25) + assert.Equal(t, 720, ds.LookbackHours) assert.Equal(t, 25, ds.MaxTraces) + + raw, err := json.Marshal(ds) + require.NoError(t, err) + body := string(raw) + assert.Contains(t, body, `"lookback_hours":720`) + assert.NotContains(t, body, "start_time", + "the service drops start_time and falls back to its default window") } // The reason a run failed is the only actionable part of the response, so it diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/models.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/models.go index 92b7f2817df..6d2f4c2c1c7 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/models.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/models.go @@ -338,11 +338,13 @@ type EvalRunDataSource struct { Source *EvalRunDataContent `json:"source,omitempty"` Target *EvalRunTarget `json:"target,omitempty"` - // Traces only. The window defaults to the last seven days when left unset. - AgentName string `json:"agent_name,omitempty"` - StartTime int64 `json:"start_time,omitempty"` - EndTime int64 `json:"end_time,omitempty"` - MaxTraces int `json:"max_traces,omitempty"` + // Traces only. The window is expressed as a lookback in hours, not as a + // start bound: the service has no start_time on this data source and + // silently falls back to its default when one is sent. + AgentName string `json:"agent_name,omitempty"` + LookbackHours int `json:"lookback_hours,omitempty"` + EndTime int64 `json:"end_time,omitempty"` + MaxTraces int `json:"max_traces,omitempty"` } // EvalRunInputMessages describes how input messages are constructed from dataset items. @@ -399,16 +401,16 @@ func NewAgentTargetDataSource(agentName string, agentVersion *string) *EvalRunDa // NewTracesDataSource evaluates an agent's recorded traces instead of a dataset. // -// The service reads them from Application Insights and defaults to the last -// seven days, so a zero window is left off rather than sent as an epoch. -func NewTracesDataSource(agentName string, start, end time.Time, maxTraces int) *EvalRunDataSource { +// The window is a lookback in hours. The service's own field is +// `lookback_hours` and it has no start bound: a `start_time` is accepted and +// dropped, leaving the default seven days in place, so the conversion happens +// here rather than being left to look like it worked. +func NewTracesDataSource(agentName string, lookbackHours int, end time.Time, maxTraces int) *EvalRunDataSource { ds := &EvalRunDataSource{ - Type: EvalRunDataSourceTypeTraces, - AgentName: agentName, - MaxTraces: maxTraces, - } - if !start.IsZero() { - ds.StartTime = start.Unix() + Type: EvalRunDataSourceTypeTraces, + AgentName: agentName, + LookbackHours: lookbackHours, + MaxTraces: maxTraces, } if !end.IsZero() { ds.EndTime = end.Unix() From 7905c97ff2f02666a433a75289264bebb4079285 Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 28 Jul 2026 11:47:59 +0300 Subject: [PATCH 040/320] feat(azure.ai.evaluations): evaluate stored responses by id The second M4 scenario listed as awaiting service support and reachable today. `run --response-id` evaluates responses the project already holds, so a run that already happened can be scored without replaying it. The shape is not the obvious one. There is no list of ids on the data source: the ids travel as ordinary JSONL rows and a data_mapping points the service at the field holding each one, which is what lets it retrieve the chat history behind the response. Every guess at a plainer shape was rejected with the same unhelpful "Item generation source content is empty", so this came from reading the service contract instead. Verified live: three stored responses evaluated, three passed, and the stored payload matched what was sent field for field. --- .../azure.ai.evaluations/internal/cmd/run.go | 9 ++++ .../internal/cmd/traces_test.go | 28 +++++++++++++ .../internal/pkg/eval_api/models.go | 41 +++++++++++++++++++ 3 files changed, 78 insertions(+) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go index 436b94ae456..355bc06b678 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go @@ -54,6 +54,8 @@ func buildRunCommand(use, short string) *cobra.Command { fromTraces bool traceWindow string maxTraces int + responseIDs []string + maxTurns int wait bool endpointFlg string ) @@ -101,6 +103,8 @@ func buildRunCommand(use, short string) *cobra.Command { // target and dataset comes from the group's previous run. var dataSource *eval_api.EvalRunDataSource switch { + case len(responseIDs) > 0: + dataSource = eval_api.NewResponsesDataSource(responseIDs, maxTurns) case fromTraces: dataSource, err = buildTracesDataSource(ctx, ec, group, evalID, traceWindow, maxTraces) case group == nil: @@ -181,6 +185,11 @@ func buildRunCommand(use, short string) *cobra.Command { cmd.Flags().StringVar(&traceWindow, "trace-window", "", "How far back to read traces, for example 7d. Defaults to the service's window.") cmd.Flags().IntVar(&maxTraces, "max-traces", 0, "Cap the traces evaluated.") + cmd.Flags().StringSliceVar(&responseIDs, "response-id", nil, + "Evaluate stored responses by id instead of the dataset; repeatable.") + cmd.Flags().IntVar(&maxTurns, "max-turns", 0, + "Turns of chat history to pull back per response. Defaults to the service's limit.") + cmd.MarkFlagsMutuallyExclusive("from-traces", "response-id") cmd.Flags().BoolVar(&wait, "wait", true, "Block until the run reaches a terminal state.") // The spec documents --no-wait, and cobra does not derive it from a bool. var noWait bool diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/traces_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/traces_test.go index 75a175d6d3f..078a62435b9 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/traces_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/traces_test.go @@ -74,6 +74,34 @@ func TestRunFailureMessage(t *testing.T) { assert.Empty(t, nilRun.Failure()) } +// The ids travel as ordinary JSONL rows with a mapping pointing at the field +// that holds each one; that is how the service finds the chat history. +func TestNewResponsesDataSource(t *testing.T) { + ds := eval_api.NewResponsesDataSource([]string{"resp_a", "resp_b"}, 10) + assert.Equal(t, eval_api.EvalRunDataSourceTypeResponses, ds.Type) + require.NotNil(t, ds.ItemGenerationParams) + assert.Equal(t, "response_retrieval", ds.ItemGenerationParams.Type) + assert.Equal(t, 10, ds.ItemGenerationParams.MaxNumTurns) + assert.Equal(t, + map[string]string{"response_id": "{{item.response_id}}"}, + ds.ItemGenerationParams.DataMapping) + + raw, err := json.Marshal(ds) + require.NoError(t, err) + body := string(raw) + assert.Contains(t, body, `"response_id":"resp_a"`) + assert.Contains(t, body, `"response_id":"resp_b"`) + assert.NotContains(t, body, "agent_name", "responses carry no agent") +} + +// An unset turn limit is left to the service rather than sent as zero. +func TestNewResponsesDataSource_OmitsAnUnsetTurnLimit(t *testing.T) { + ds := eval_api.NewResponsesDataSource([]string{"resp_a"}, 0) + raw, err := json.Marshal(ds) + require.NoError(t, err) + assert.NotContains(t, string(raw), "max_num_turns") +} + func TestRenderRun_ShowsTheFailureReason(t *testing.T) { var buf bytes.Buffer run := &eval_api.OpenAIEvalRun{ diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/models.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/models.go index 6d2f4c2c1c7..af5f880e268 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/models.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/models.go @@ -321,6 +321,10 @@ const ( // a dataset. The service reads them from Application Insights, so the agent // must be emitting gen_ai.input.messages / gen_ai.output.messages. EvalRunDataSourceTypeTraces EvalRunDataSourceType = "azure_ai_traces" + + // EvalRunDataSourceTypeResponses evaluates responses the project already + // stored, addressed by id. + EvalRunDataSourceTypeResponses EvalRunDataSourceType = "azure_ai_responses" ) // EvalRunDataContentType defines the source type for eval run data content. @@ -345,6 +349,18 @@ type EvalRunDataSource struct { LookbackHours int `json:"lookback_hours,omitempty"` EndTime int64 `json:"end_time,omitempty"` MaxTraces int `json:"max_traces,omitempty"` + + // Responses only. + ItemGenerationParams *ItemGenerationParams `json:"item_generation_params,omitempty"` +} + +// ItemGenerationParams says how the service should turn a source into the items +// it evaluates. +type ItemGenerationParams struct { + Type string `json:"type"` + MaxNumTurns int `json:"max_num_turns,omitempty"` + DataMapping map[string]string `json:"data_mapping,omitempty"` + Source *EvalRunDataContent `json:"source,omitempty"` } // EvalRunInputMessages describes how input messages are constructed from dataset items. @@ -418,6 +434,31 @@ func NewTracesDataSource(agentName string, lookbackHours int, end time.Time, max return ds } +// NewResponsesDataSource evaluates responses the project already stored. +// +// The ids travel as ordinary JSONL rows and a data_mapping points the service +// at the field holding each one, which is how it retrieves the chat history +// behind the response. +func NewResponsesDataSource(responseIDs []string, maxTurns int) *EvalRunDataSource { + rows := make([]map[string]any, 0, len(responseIDs)) + for _, id := range responseIDs { + rows = append(rows, map[string]any{"item": map[string]any{"response_id": id}}) + } + + return &EvalRunDataSource{ + Type: EvalRunDataSourceTypeResponses, + ItemGenerationParams: &ItemGenerationParams{ + Type: "response_retrieval", + MaxNumTurns: maxTurns, + DataMapping: map[string]string{"response_id": "{{item.response_id}}"}, + Source: &EvalRunDataContent{ + Type: EvalRunDataContentTypeFileContent, + Content: rows, + }, + }, + } +} + // SetFileContent sets the data source to use inline file content. func (ds *EvalRunDataSource) SetFileContent(items []map[string]any) { ds.Source = &EvalRunDataContent{ From 1d195f56a82fb40a93ce177ec2565b43d0edef82 Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 28 Jul 2026 12:01:03 +0300 Subject: [PATCH 041/320] feat(azure.ai.evaluations): evaluate a model deployment directly MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit target.type: model evaluates a deployment with no agent in front of it. The config already rejected it by name, and the test asserting that used "model" as its example of an unsupported type, so the gap was recorded twice as deliberate. The service supports it. What makes it more than a new enum value is that the criteria have to change with it. An agent returns output_items and tool calls; a model answers as plain text under output_text. Binding the agent shape to a model run does not fail at validation, it fails per row at execution with "Missing inputs for line 1: data.sample.tool_definitions, data.sample.output_items" — the group looks well formed and every row errors. So the sample bindings are chosen from the target kind rather than from whether a target exists at all, and a model target binds response alone. Verified end to end: a model-target group deployed with response bound to {{sample.output_text}}, ran, and scored 2 passed / 1 failed / 0 errored across coherence and fluency. --- .../internal/cmd/build.go | 31 +++++- .../internal/cmd/model_target_test.go | 98 +++++++++++++++++++ .../azure.ai.evaluations/internal/cmd/run.go | 9 +- .../internal/pkg/eval_api/models.go | 36 ++++++- .../internal/project/eval_config.go | 12 ++- .../internal/project/eval_config_test.go | 2 +- 6 files changed, 175 insertions(+), 13 deletions(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/model_target_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build.go index 803eed1c4e0..2fe71d974a5 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build.go @@ -35,6 +35,25 @@ var sampleBindings = map[string]string{ "tool_definitions": "{{sample.tool_definitions}}", } +// modelSampleBindings are what a model target produces. A model answers as +// plain text and calls no tools, so binding an agent's richer output would +// leave the evaluator waiting on fields the run never produces. +var modelSampleBindings = map[string]string{ + "response": "{{sample.output_text}}", +} + +// sampleBindingsFor returns the run-time bindings a target of this kind can +// satisfy. An empty target kind means nothing is invoked, so nothing is bound. +func sampleBindingsFor(targetType string) map[string]string { + switch targetType { + case project.TargetTypeAgent: + return sampleBindings + case project.TargetTypeModel: + return modelSampleBindings + } + return nil +} + // legacyInputs is the mapping used when the service publishes no schema for an // evaluator, which is the case for freshly uploaded custom evaluators. It // matches the agent-target shape. @@ -116,7 +135,7 @@ func selectLevelFields(accepted, required []string, level string) []string { func planCriterion( ref evalcore.EvaluatorRef, schema *eval_api.EvaluatorSummary, - hasTarget bool, + targetBindings map[string]string, datasetColumns map[string]bool, evalModel string, level string, @@ -138,7 +157,7 @@ func planCriterion( } for _, field := range accepted { - if binding, ok := sampleBindings[field]; ok && hasTarget { + if binding, ok := targetBindings[field]; ok { plan.dataMapping[field] = binding continue } @@ -233,9 +252,15 @@ func buildEvalGroupRequest( ) (*eval_api.CreateOpenAIEvalRequest, error) { metadata := map[string]string{} hasTarget := group.Target != nil && group.Target.Name != "" + targetType := "" if hasTarget { metadata["azd_agent"] = group.Target.Name + targetType = group.Target.Type + if targetType == "" { + targetType = project.TargetTypeAgent + } } + targetBindings := sampleBindingsFor(targetType) metadata["azd_eval_group"] = group.Name // The create request has no description field, so the group's own // description rides in metadata rather than being dropped. @@ -263,7 +288,7 @@ func buildEvalGroupRequest( schema = &eval_api.EvaluatorSummary{Name: ref.Name} } - plan, err := planCriterion(ref, schema, hasTarget, datasetColumns, evalModel, level) + plan, err := planCriterion(ref, schema, targetBindings, datasetColumns, evalModel, level) if err != nil { return nil, err } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/model_target_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/model_target_test.go new file mode 100644 index 00000000000..2a458a5778c --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/model_target_test.go @@ -0,0 +1,98 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "encoding/json" + "testing" + + "azureaieval/internal/pkg/eval_api" + "azureaieval/internal/pkg/evalcore" + "azureaieval/internal/project" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// A model answers as plain text and calls no tools. Binding an agent's richer +// output would leave the evaluator waiting on fields the run never produces, +// which the service reports as a missing input rather than a mapping mistake. +func TestSampleBindingsFor(t *testing.T) { + agent := sampleBindingsFor(project.TargetTypeAgent) + assert.Equal(t, "{{sample.output_items}}", agent["response"]) + assert.Contains(t, agent, "tool_calls") + assert.Contains(t, agent, "tool_definitions") + + model := sampleBindingsFor(project.TargetTypeModel) + assert.Equal(t, "{{sample.output_text}}", model["response"]) + assert.NotContains(t, model, "tool_calls", "a model calls no tools") + assert.NotContains(t, model, "tool_definitions") + + assert.Nil(t, sampleBindingsFor(""), "with nothing invoked, nothing is bound") +} + +// The criteria a group sends depend on what it targets. +func TestBuildEvalGroupRequest_BindsByTargetKind(t *testing.T) { + schemas := map[string]*eval_api.EvaluatorSummary{ + "builtin.coherence": { + Name: "builtin.coherence", + Definition: &eval_api.EvaluatorContract{ + DataSchema: &eval_api.JSONSchema{ + Required: []string{"query", "response"}, + Properties: map[string]any{ + "query": map[string]any{"type": "string"}, + "response": map[string]any{"type": "string"}, + }, + }, + }, + }, + } + + for _, tc := range []struct { + targetType string + want string + }{ + {project.TargetTypeAgent, "{{sample.output_items}}"}, + {project.TargetTypeModel, "{{sample.output_text}}"}, + } { + t.Run(tc.targetType, func(t *testing.T) { + group := &project.EvalGroup{ + Name: "quality", + Evaluators: []evalcore.EvaluatorRef{{Name: "builtin.coherence"}}, + Target: &project.Target{Type: tc.targetType, Name: "thing"}, + } + req, err := buildEvalGroupRequest(group, schemas, map[string]bool{"query": true}) + require.NoError(t, err) + require.Len(t, req.TestingCriteria, 1) + assert.Equal(t, tc.want, req.TestingCriteria[0].DataMapping["response"]) + assert.Equal(t, "{{item.query}}", req.TestingCriteria[0].DataMapping["query"]) + }) + } +} + +// The target the run posts has to match what the group's criteria expect. +func TestNewModelTargetDataSource(t *testing.T) { + ds := eval_api.NewModelTargetDataSource("gpt-4.1-nano") + require.NotNil(t, ds.Target) + assert.Equal(t, "azure_ai_model", ds.Target.Type) + assert.Equal(t, "gpt-4.1-nano", ds.Target.Model) + + raw, err := json.Marshal(ds) + require.NoError(t, err) + body := string(raw) + assert.Contains(t, body, `"model":"gpt-4.1-nano"`) + assert.NotContains(t, body, `"name"`, "a model target is addressed by deployment, not name") + assert.NotContains(t, body, "tool_descriptions", "a model calls no tools") +} + +func TestNewAgentTargetDataSource_StillSendsAgentFields(t *testing.T) { + ds := eval_api.NewAgentTargetDataSource("support-agent", nil) + require.NotNil(t, ds.Target) + assert.Equal(t, "azure_ai_agent", ds.Target.Type) + assert.Equal(t, "support-agent", ds.Target.Name) + + raw, err := json.Marshal(ds) + require.NoError(t, err) + assert.NotContains(t, string(raw), `"model"`) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go index 355bc06b678..1d3d20b53cb 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go @@ -386,10 +386,15 @@ func buildRunDataSource( ) (*eval_api.EvalRunDataSource, error) { if group == nil || group.Target == nil { return nil, fmt.Errorf( - "the eval group must declare target.type: agent so the run knows what to invoke") + "the eval group must declare a target so the run knows what to invoke") } - ds := eval_api.NewAgentTargetDataSource(group.Target.Name, nil) + var ds *eval_api.EvalRunDataSource + if group.Target.Type == project.TargetTypeModel { + ds = eval_api.NewModelTargetDataSource(group.Target.Name) + } else { + ds = eval_api.NewAgentTargetDataSource(group.Target.Name, nil) + } if group.Dataset == "" { return nil, fmt.Errorf("eval group %q does not reference a dataset", group.Name) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/models.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/models.go index af5f880e268..8e1ba82f1bf 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/models.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/models.go @@ -376,12 +376,14 @@ type EvalRunMessageTemplate struct { Type string `json:"type"` } -// EvalRunTarget describes the agent target for completions. +// EvalRunTarget describes what the run invokes: an agent by name, or a model +// deployment directly. Only the fields belonging to Type are sent. type EvalRunTarget struct { Type string `json:"type"` - Name string `json:"name"` - Version *string `json:"version"` - ToolDescriptions []string `json:"tool_descriptions"` + Name string `json:"name,omitempty"` + Version *string `json:"version,omitempty"` + ToolDescriptions []string `json:"tool_descriptions,omitempty"` + Model string `json:"model,omitempty"` } // EvalRunDataContent holds the source reference within an EvalRunDataSource. @@ -434,6 +436,32 @@ func NewTracesDataSource(agentName string, lookbackHours int, end time.Time, max return ds } +// NewModelTargetDataSource sends the dataset's questions straight to a model +// deployment, with no agent in front of it. +// +// The model answers as plain text, so a group evaluating one has to bind its +// response to {{sample.output_text}} rather than the richer output an agent +// produces. +func NewModelTargetDataSource(model string) *EvalRunDataSource { + return &EvalRunDataSource{ + Type: EvalRunDataSourceTypeAgentTarget, + InputMessages: &EvalRunInputMessages{ + Type: "template", + Template: []EvalRunMessageTemplate{ + { + Role: "user", + Content: "{{item.query}}", + Type: "message", + }, + }, + }, + Target: &EvalRunTarget{ + Type: "azure_ai_model", + Model: model, + }, + } +} + // NewResponsesDataSource evaluates responses the project already stored. // // The ids travel as ordinary JSONL rows and a data_mapping points the service diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go index 37a0777448a..13b1eba685f 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go @@ -58,6 +58,11 @@ type Target struct { const TargetTypeAgent = "agent" +// TargetTypeModel evaluates a model deployment directly, with no agent in +// front of it. A model answers as plain text, so a group targeting one binds +// its response differently from a group targeting an agent. +const TargetTypeModel = "model" + // Options are run settings carried on the group. type Options struct { EvalModel string `yaml:"eval_model,omitempty" json:"eval_model,omitempty"` @@ -145,10 +150,11 @@ func (c *EvalConfig) Validate() error { i, g.Name, ref.Name, evalcore.BuiltinPrefix) } } - if g.Target != nil && g.Target.Type != "" && g.Target.Type != TargetTypeAgent { + if g.Target != nil && g.Target.Type != "" && + g.Target.Type != TargetTypeAgent && g.Target.Type != TargetTypeModel { return fmt.Errorf( - "evalGroups[%d] (%s): target.type %q is not supported; only %q is available today", - i, g.Name, g.Target.Type, TargetTypeAgent) + "evalGroups[%d] (%s): target.type %q is not supported; use %q or %q", + i, g.Name, g.Target.Type, TargetTypeAgent, TargetTypeModel) } if g.Options != nil { switch g.Options.EvaluationLevel { diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_test.go index 172154ce714..177638d5507 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_test.go @@ -148,7 +148,7 @@ func TestValidate_Rejects(t *testing.T) { }, { name: "unsupported target type", - body: "evalGroups:\n - name: g\n evaluators: [builtin.relevance]\n target:\n type: model\n", + body: "evalGroups:\n - name: g\n evaluators: [builtin.relevance]\n target:\n type: prompt\n", wantErr: "is not supported", }, { From fbf532fdc2bdccab20d25c10161aa69f0ba547f6 Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 28 Jul 2026 12:16:31 +0300 Subject: [PATCH 042/320] fix(azure.ai.evaluations): run the datasets that are already registered A group whose dataset has no local source could not run at all. The run sent the dataset name as a file_id, and a file_id means an uploaded file, so the service answered "invalid data source file ids". Every test until now used a local source, which is why the path stayed unexercised. Registered datasets are now fetched and their rows sent inline, the same as a local file. That also gives --max-samples the meaning it always claimed: it was documented as ignored for registered datasets, because a file reference carries no row limit and the service cannot narrow one. Fetching the rows is what makes narrowing possible, so the flag now behaves the same either way and the caveat is gone from its help. Found while implementing the last M4 scenario, subsetting a registered dataset. The subsetting worked on the first try; the run underneath it did not, which is the part worth having found. Verified live against a published dataset: the whole set scores 2 passed / 1 failed, and --max-samples 2 scores 2 rows. --- .../internal/cmd/dataset_rows_test.go | 53 +++++++++++ .../azure.ai.evaluations/internal/cmd/run.go | 88 +++++++++++++++++-- .../internal/cmd/schedule.go | 4 +- 3 files changed, 134 insertions(+), 11 deletions(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset_rows_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset_rows_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset_rows_test.go new file mode 100644 index 00000000000..dd25bcf022b --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset_rows_test.go @@ -0,0 +1,53 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestReadJSONLBytes(t *testing.T) { + content := []byte( + "{\"query\":\"a\"}\n" + + "\n" + // blank lines are skipped, not treated as rows + "{\"query\":\"b\"}\n" + + " {\"query\":\"c\"} \n") + + all, err := readJSONLBytes(content, 0) + require.NoError(t, err) + require.Len(t, all, 3) + assert.Equal(t, "a", all[0]["query"]) + assert.Equal(t, "c", all[2]["query"], "surrounding whitespace is not part of the row") +} + +// The limit is what makes --max-samples mean the same thing for a published +// dataset as for a local file. +func TestReadJSONLBytes_StopsAtTheLimit(t *testing.T) { + content := []byte("{\"n\":1}\n{\"n\":2}\n{\"n\":3}\n") + + two, err := readJSONLBytes(content, 2) + require.NoError(t, err) + require.Len(t, two, 2) + assert.EqualValues(t, 1, two[0]["n"]) + assert.EqualValues(t, 2, two[1]["n"]) + + // A limit larger than the file is not an error. + more, err := readJSONLBytes(content, 99) + require.NoError(t, err) + assert.Len(t, more, 3) +} + +func TestReadJSONLBytes_ReportsTheOffendingLine(t *testing.T) { + _, err := readJSONLBytes([]byte("{\"n\":1}\nnot json\n"), 0) + require.ErrorContains(t, err, "line 2") +} + +func TestReadJSONLBytes_EmptyIsNotAnError(t *testing.T) { + items, err := readJSONLBytes([]byte("\n\n"), 0) + require.NoError(t, err) + assert.Empty(t, items, "the caller decides whether no rows is a problem") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go index 1d3d20b53cb..39f98b811d1 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go @@ -5,15 +5,18 @@ package cmd import ( "bufio" + "bytes" "context" "encoding/json" "errors" "fmt" + "io" "os" "path/filepath" "strings" "time" + "azureaieval/internal/pkg/dataset_api" "azureaieval/internal/pkg/eval_api" "azureaieval/internal/project" @@ -110,8 +113,8 @@ func buildRunCommand(use, short string) *cobra.Command { case group == nil: dataSource, err = ec.reuseDataSourceFromLastRun(ctx, evalID) default: - dataSource, err = buildRunDataSource( - group, configPath, resolveMaxSamples(maxSamples, group)) + dataSource, err = ec.buildRunDataSource( + ctx, group, configPath, resolveMaxSamples(maxSamples, group)) } if err != nil { return err @@ -179,7 +182,7 @@ func buildRunCommand(use, short string) *cobra.Command { cmd.Flags().StringVar(&level, "level", "", "Scoring granularity: turn or conversation. Defaults to the service default (turn).") cmd.Flags().IntVar(&maxSamples, "max-samples", 0, - "Cap the rows sent from a local dataset file. Ignored for registered datasets.") + "Cap the rows sent from the dataset.") cmd.Flags().BoolVar(&fromTraces, "from-traces", false, "Evaluate the agent's recorded traces instead of the dataset.") cmd.Flags().StringVar(&traceWindow, "trace-window", "", @@ -379,7 +382,8 @@ func buildTracesDataSource( // buildRunDataSource binds the dataset to the run. The eval group carries no // dataset today, so it is supplied here. -func buildRunDataSource( +func (ec *evalContext) buildRunDataSource( + ctx context.Context, group *project.EvalGroup, configPath string, maxSamples int, @@ -400,10 +404,17 @@ func buildRunDataSource( return nil, fmt.Errorf("eval group %q does not reference a dataset", group.Name) } - // A local source is sent inline; anything else is a registered dataset. + // A local source is read from disk; anything else is already registered and + // has to be fetched. Either way the rows are sent inline, because a run's + // file_id means an uploaded file and a dataset name is not one: sending the + // name is rejected with "invalid data source file ids". localPath := localDatasetPath(configPath, group) if localPath == "" { - ds.SetFileID(group.Dataset) + items, err := ec.readRegisteredDataset(ctx, group.Dataset, maxSamples) + if err != nil { + return nil, err + } + ds.SetFileContent(items) return ds, nil } @@ -418,6 +429,49 @@ func buildRunDataSource( return ds, nil } +// readRegisteredDataset fetches a published dataset's rows, optionally keeping +// only the first n. +// +// The rows have to be fetched because a run cannot reference a dataset by +// name: `file_id` means an uploaded file, and passing a dataset name there is +// rejected. Fetching also makes --max-samples mean the same thing whether the +// dataset is local or published, which a file reference could not — that +// source carries no row limit. +func (ec *evalContext) readRegisteredDataset( + ctx context.Context, + name string, + maxSamples int, +) ([]map[string]any, error) { + version := ec.getEnvValue(ctx, envKeyDatasetVersion) + if version == "" { + versions, err := ec.datasetClient.ListDatasetVersions(ctx, name, ProjectEndpointAPIVersion) + if err != nil { + return nil, fmt.Errorf("reading dataset %q: %w", name, err) + } + if versions != nil { + version = dataset_api.LatestVersion(versions.Value) + } + } + if version == "" { + return nil, fmt.Errorf("dataset %q has no versions to read", name) + } + + content, err := ec.datasetClient.DownloadDatasetContent( + ctx, name, version, ProjectEndpointAPIVersion) + if err != nil { + return nil, fmt.Errorf("reading dataset %q version %s: %w", name, version, err) + } + + items, err := readJSONLBytes(content, maxSamples) + if err != nil { + return nil, fmt.Errorf("reading dataset %q version %s: %w", name, version, err) + } + if len(items) == 0 { + return nil, fmt.Errorf("dataset %q version %s has no rows", name, version) + } + return items, nil +} + // datasetColumns reports the columns a group's dataset provides, so criteria // bind only to fields that exist and a missing required field is caught // locally rather than as a service rejection. @@ -472,8 +526,24 @@ func readJSONL(path string, limit int) ([]map[string]any, error) { } defer f.Close() + items, err := scanJSONL(f, limit) + if err != nil { + return nil, fmt.Errorf("reading dataset %q: %w", path, err) + } + return items, nil +} + +// readJSONLBytes parses JSONL already in memory, which is how a registered +// dataset arrives. +func readJSONLBytes(content []byte, limit int) ([]map[string]any, error) { + return scanJSONL(bytes.NewReader(content), limit) +} + +// scanJSONL reads rows until the limit is reached, so a subset costs only the +// rows it needs to parse. +func scanJSONL(r io.Reader, limit int) ([]map[string]any, error) { var items []map[string]any - scanner := bufio.NewScanner(f) + scanner := bufio.NewScanner(r) scanner.Buffer(make([]byte, 0, 64*1024), 8*1024*1024) line := 0 for scanner.Scan() { @@ -484,7 +554,7 @@ func readJSONL(path string, limit int) ([]map[string]any, error) { } var row map[string]any if err := json.Unmarshal([]byte(text), &row); err != nil { - return nil, fmt.Errorf("%s line %d is not valid JSON: %w", path, line, err) + return nil, fmt.Errorf("line %d is not valid JSON: %w", line, err) } items = append(items, row) if limit > 0 && len(items) >= limit { @@ -492,7 +562,7 @@ func readJSONL(path string, limit int) ([]map[string]any, error) { } } if err := scanner.Err(); err != nil { - return nil, fmt.Errorf("reading dataset %q: %w", path, err) + return nil, err } return items, nil } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/schedule.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/schedule.go index e0afd5c468b..96f5429e060 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/schedule.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/schedule.go @@ -121,8 +121,8 @@ func newScheduleSetCommand() *cobra.Command { if err != nil { return err } - dataSource, err = buildRunDataSource( - group, configPath, resolveMaxSamples(maxSamples, group)) + dataSource, err = ec.buildRunDataSource( + ctx, group, configPath, resolveMaxSamples(maxSamples, group)) if err != nil { return err } From 09b7e5f965cfae62887d29abc8eb3454eae9d7ec Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 28 Jul 2026 12:27:05 +0300 Subject: [PATCH 043/320] fix(azure.ai.evaluations): stop a second eval group from taking the first one's id A config with two eval groups aliased them. Resolved ids were kept under one shared EVAL_GROUP_ID, so on the second deploy the first group read the id the second had left there, confirmed that group existed, and adopted it. Both declarations then pointed at the same group, and running one scored the other's criteria against its own dataset. Nothing failed; the results were just wrong. The first deploy hid it, because the env is not readable back within the same deploy, so both groups were created correctly and only the cache was left crossed. It needed a second deploy to surface, which is why the single-group testing everything else used never reached it. Ids are now keyed by declaration name, as the fingerprints already were. EVAL_GROUP_ID is still written as the last-deployed group, which is what the commands fall back to when a config names only one. Same fix for the dataset version read while narrowing a registered dataset: it was reading the shared EVAL_DATASET_VERSION, which with two datasets is whichever was published last. Verified live: two groups keep distinct ids across repeated deploys, and each runs its own criteria. --- .../internal/cmd/envkeys_test.go | 37 +++++++++++++++++++ .../internal/cmd/reconciler.go | 15 +++++++- .../azure.ai.evaluations/internal/cmd/run.go | 7 ++-- 3 files changed, 55 insertions(+), 4 deletions(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/envkeys_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/envkeys_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/envkeys_test.go new file mode 100644 index 00000000000..be8090b9d06 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/envkeys_test.go @@ -0,0 +1,37 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +// Ids are per declaration. A shared key works only while a config has one +// group: with two, the second deploy finds the first's id cached, confirms it +// exists, and hands it back for the wrong group — so group A silently scores +// group B's criteria. +func TestIDKey_IsPerName(t *testing.T) { + a := idKey("evalgroup", "quality-a") + b := idKey("evalgroup", "quality-b") + + assert.NotEqual(t, a, b, "two groups must not share an id key") + assert.Contains(t, a, "QUALITY_A") + assert.True(t, len(a) > 3 && a[len(a)-3:] == "_ID") +} + +// Names that are not valid env identifiers still have to produce distinct, +// stable keys. +func TestIDKey_NormalizesNames(t *testing.T) { + assert.Equal(t, idKey("evalgroup", "my group"), idKey("evalgroup", "my-group"), + "characters that cannot appear in an env name normalize the same way") + assert.NotEqual(t, idKey("evalgroup", "a"), idKey("dataset", "a"), + "the kind keeps different resources apart") +} + +// The id and version keys for the same declaration must not collide. +func TestIDKey_DoesNotCollideWithVersionKey(t *testing.T) { + assert.NotEqual(t, idKey("dataset", "golden"), versionKey("dataset", "golden")) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go index a00ce08711b..b645c8ce6ee 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go @@ -220,13 +220,14 @@ func (r *evalReconciler) EnsureEvalGroup( recreate = true } - cached := r.ec.getEnvValue(ctx, envKeyEvalGroupID) + cached := r.ec.getEnvValue(ctx, idKey("evalgroup", group.Name)) if cached != "" && !recreate { if _, err := r.ec.evalClient.GetOpenAIEval(ctx, cached); err == nil { // Record the digest on reuse as well, otherwise a group deployed // before fingerprinting existed never establishes a baseline and // later edits go undetected. _ = r.ec.setEnvValue(ctx, key, digest) + _ = r.ec.setEnvValue(ctx, envKeyEvalGroupID, cached) return cached, nil } } @@ -244,6 +245,9 @@ func (r *evalReconciler) EnsureEvalGroup( return "", err } _ = r.ec.setEnvValue(ctx, key, digest) + _ = r.ec.setEnvValue(ctx, idKey("evalgroup", group.Name), created.ID) + // EVAL_GROUP_ID stays the last-deployed group, which is what the commands + // fall back to when a config names only one. _ = r.ec.setEnvValue(ctx, envKeyEvalGroupID, created.ID) return created.ID, nil } @@ -314,3 +318,12 @@ func versionFromRaw(raw []byte, fallback string) string { func versionKey(kind, name string) string { return project.FingerprintKey(kind, name) + "_VERSION" } + +// idKey names the env entry holding a resolved id. +// +// Ids are per declaration. A single shared key works only while a config has +// one group: with two, the second deploy finds the first's id cached, confirms +// it exists, and hands it back for the wrong group. +func idKey(kind, name string) string { + return project.FingerprintKey(kind, name) + "_ID" +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go index 39f98b811d1..e0d187856b5 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go @@ -223,7 +223,7 @@ func (ec *evalContext) resolveEvalGroupID( return group.ID, nil } - if cached := ec.getEnvValue(ctx, envKeyEvalGroupID); cached != "" { + if cached := ec.getEnvValue(ctx, idKey("evalgroup", group.Name)); cached != "" { // Confirm it still exists; a deleted group should fall through to create. if _, err := ec.evalClient.GetOpenAIEval(ctx, cached); err == nil { return cached, nil @@ -258,9 +258,10 @@ func (ec *evalContext) resolveEvalGroupID( if err != nil { return "", fmt.Errorf("creating eval group %q: %w", group.Name, err) } - if err := ec.setEnvValue(ctx, envKeyEvalGroupID, created.ID); err != nil { + if err := ec.setEnvValue(ctx, idKey("evalgroup", group.Name), created.ID); err != nil { fmt.Fprintf(out, "warning: %v\n", err) } + _ = ec.setEnvValue(ctx, envKeyEvalGroupID, created.ID) return created.ID, nil } @@ -442,7 +443,7 @@ func (ec *evalContext) readRegisteredDataset( name string, maxSamples int, ) ([]map[string]any, error) { - version := ec.getEnvValue(ctx, envKeyDatasetVersion) + version := ec.getEnvValue(ctx, versionKey("dataset", name)) if version == "" { versions, err := ec.datasetClient.ListDatasetVersions(ctx, name, ProjectEndpointAPIVersion) if err != nil { From d961d3bb0773f19a7912b8ba55c4000d38e633b5 Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 28 Jul 2026 12:37:45 +0300 Subject: [PATCH 044/320] feat(azure.ai.evaluations): reach a group by name, and remember its run separately MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Finishes what the group-id fix started. Two more places assumed one group. Every command taking an eval-id could only be pointed at a group by its service id. With one group the cached id made that invisible; with two there was no way to ask for the other except by looking its id up. They now accept --eval-group, naming the group the way the config does. That exposed the next layer: the remembered run was also a single key, so it belonged to whichever group ran last. Asking group A for its latest results fetched group B's run id inside group A and returned 404 — the group resolved correctly and the run did not. Runs are now remembered per group. A remembered run that no longer resolves also stops being fatal. It is a convenience, so a stale one falls through to the group's current latest rather than failing; an explicit --run-id still reports what went wrong. Verified live with two groups: each resolves to its own run and its own criteria, and an undeployed name is refused by name. --- .../internal/cmd/compare.go | 4 +- .../internal/cmd/results.go | 59 +++++++++++++++---- .../azure.ai.evaluations/internal/cmd/run.go | 4 ++ .../internal/cmd/run_ops.go | 16 +++-- 4 files changed, 67 insertions(+), 16 deletions(-) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/compare.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/compare.go index 500f135b3fd..815f4ab47c6 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/compare.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/compare.go @@ -26,6 +26,7 @@ func newResultsCompareCommand() *cobra.Command { treatments []string displayName string endpointFlg string + groupName string ) cmd := &cobra.Command{ @@ -42,7 +43,7 @@ func newResultsCompareCommand() *cobra.Command { } defer ec.Close() - evalID, err := resolveEvalID(cmd, ec, args) + evalID, err := resolveEvalID(cmd, ec, args, groupName) if err != nil { return err } @@ -89,6 +90,7 @@ func newResultsCompareCommand() *cobra.Command { cmd.Flags().StringArrayVar(&treatments, "treatment", nil, "Run to measure, repeatable. Defaults to the most recent completed run.") cmd.Flags().StringVar(&displayName, "name", "", "Name for this comparison.") + addEvalGroupFlag(cmd, &groupName) cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") return cmd } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/results.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/results.go index 985a4b7b461..3a2f4d87f5a 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/results.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/results.go @@ -31,6 +31,7 @@ func newResultsShowCommand() *cobra.Command { failedOnly bool outFile string endpointFlg string + groupName string ) cmd := &cobra.Command{ @@ -45,7 +46,7 @@ func newResultsShowCommand() *cobra.Command { } defer ec.Close() - evalID, err := resolveEvalID(cmd, ec, args) + evalID, err := resolveEvalID(cmd, ec, args, groupName) if err != nil { return err } @@ -73,6 +74,7 @@ func newResultsShowCommand() *cobra.Command { cmd.Flags().StringVar(&runID, "run-id", "", "Run to show. Defaults to the most recent run.") cmd.Flags().BoolVar(&failedOnly, "failed-only", false, "Show only criteria with failures.") cmd.Flags().StringVarP(&outFile, "out-file", "O", "", "Write JSON results to this path.") + addEvalGroupFlag(cmd, &groupName) cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") return cmd } @@ -83,6 +85,7 @@ func newResultsExportCommand() *cobra.Command { format string outFile string endpointFlg string + groupName string ) cmd := &cobra.Command{ @@ -102,7 +105,7 @@ func newResultsExportCommand() *cobra.Command { } defer ec.Close() - evalID, err := resolveEvalID(cmd, ec, args) + evalID, err := resolveEvalID(cmd, ec, args, groupName) if err != nil { return err } @@ -132,24 +135,53 @@ func newResultsExportCommand() *cobra.Command { cmd.Flags().StringVar(&runID, "run-id", "", "Run to export. Defaults to the most recent run.") cmd.Flags().StringVar(&format, "format", "json", "Output format: json or csv.") cmd.Flags().StringVarP(&outFile, "out-file", "O", "", "Write to this path instead of stdout.") + addEvalGroupFlag(cmd, &groupName) cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") return cmd } -// resolveEvalID takes the eval group id from the argument, falling back to the -// id cached in the azd environment. -func resolveEvalID(cmd *cobra.Command, ec *evalContext, args []string) (string, error) { +// resolveEvalID takes the eval group id from the argument, from a group named +// with --eval-group, or from the id cached in the azd environment. +// +// The cached id is the last group deployed, which is unambiguous only while a +// config declares one. --eval-group is how the others are reached without +// having to know their service ids. +func resolveEvalID( + cmd *cobra.Command, + ec *evalContext, + args []string, + groupName string, +) (string, error) { if len(args) > 0 && args[0] != "" { return args[0], nil } + + if groupName != "" { + if id := ec.getEnvValue(cmd.Context(), idKey("evalgroup", groupName)); id != "" { + return id, nil + } + return "", fmt.Errorf( + "eval group %q has no id recorded in this environment; deploy it first, "+ + "or pass its id directly", groupName) + } + if cached := ec.getEnvValue(cmd.Context(), envKeyEvalGroupID); cached != "" { return cached, nil } return "", fmt.Errorf( - "no eval group id given; pass it as an argument or set %s in the azd environment", + "no eval group id given; pass it as an argument, name one with --eval-group, "+ + "or set %s in the azd environment", envKeyEvalGroupID) } +// addEvalGroupFlag registers the flag that names a group from the config, so +// every command taking an eval-id can reach a group by the name its author +// used. +func addEvalGroupFlag(cmd *cobra.Command, target *string) { + cmd.Flags().StringVar(target, "eval-group", "", + "Name a group from the config instead of passing its id.") +} + // latestOrNamedRun returns the named run, or the most recent one for the group. func (ec *evalContext) latestOrNamedRun( cmd *cobra.Command, @@ -157,17 +189,22 @@ func (ec *evalContext) latestOrNamedRun( ) (*eval_api.OpenAIEvalRun, error) { ctx := cmd.Context() + // The remembered run is per group. A single shared one belongs to whichever + // group ran last, and asking another group for it returns 404 rather than + // that group's own latest run. if runID == "" { - if cached := ec.getEnvValue(ctx, envKeyEvalRunID); cached != "" { - runID = cached - } + runID = ec.getEnvValue(ctx, idKey("evalrun", evalID)) } if runID != "" { run, err := ec.evalClient.GetOpenAIEvalRun(ctx, evalID, runID) - if err != nil { + if err == nil { + return run, nil + } + // A remembered run that no longer resolves is not worth failing on: + // fall through to whatever the group has now. + if cmd.Flag("run-id") != nil && cmd.Flag("run-id").Changed { return nil, fmt.Errorf("reading run %s: %w", runID, err) } - return run, nil } list, err := ec.evalClient.ListOpenAIEvalRuns(ctx, evalID, 1) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go index e0d187856b5..31299be89de 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go @@ -142,6 +142,10 @@ func buildRunCommand(use, short string) *cobra.Command { return fmt.Errorf("starting the evaluation run: %w", err) } + // Remembered per group as well as globally: a single shared key + // belongs to whichever group ran last, so another group asking for + // "the last run" would be handed one that is not its own. + _ = ec.setEnvValue(ctx, idKey("evalrun", evalID), run.ID) if err := ec.setEnvValue(ctx, envKeyEvalRunID, run.ID); err != nil { // Persisting the run id is a convenience for later commands. // Reported on stdout because azd does not surface an diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go index c0eee5226cc..196a989a3bc 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go @@ -25,7 +25,10 @@ func addRunSubcommands(cmd *cobra.Command) { } func newRunListCommand() *cobra.Command { - var endpointFlg string + var ( + endpointFlg string + groupName string + ) cmd := &cobra.Command{ Use: "list [eval-id]", @@ -39,7 +42,7 @@ func newRunListCommand() *cobra.Command { } defer ec.Close() - evalID, err := resolveEvalID(cmd, ec, args) + evalID, err := resolveEvalID(cmd, ec, args, groupName) if err != nil { return err } @@ -64,6 +67,7 @@ func newRunListCommand() *cobra.Command { []string{"RUN ID", "NAME", "STATUS", "RESULTS"}, rows) }, } + addEvalGroupFlag(cmd, &groupName) cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") return cmd } @@ -72,6 +76,7 @@ func newRunShowCommand() *cobra.Command { var ( runID string endpointFlg string + groupName string ) cmd := &cobra.Command{ @@ -86,7 +91,7 @@ func newRunShowCommand() *cobra.Command { } defer ec.Close() - evalID, err := resolveEvalID(cmd, ec, args) + evalID, err := resolveEvalID(cmd, ec, args, groupName) if err != nil { return err } @@ -113,6 +118,7 @@ func newRunShowCommand() *cobra.Command { }, } cmd.Flags().StringVar(&runID, "run-id", "", "Run to show. Defaults to the most recent run.") + addEvalGroupFlag(cmd, &groupName) cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") return cmd } @@ -121,6 +127,7 @@ func newRunCancelCommand() *cobra.Command { var ( runID string endpointFlg string + groupName string ) cmd := &cobra.Command{ @@ -135,7 +142,7 @@ func newRunCancelCommand() *cobra.Command { } defer ec.Close() - evalID, err := resolveEvalID(cmd, ec, args) + evalID, err := resolveEvalID(cmd, ec, args, groupName) if err != nil { return err } @@ -167,6 +174,7 @@ func newRunCancelCommand() *cobra.Command { }, } cmd.Flags().StringVar(&runID, "run-id", "", "Run to cancel. Defaults to the most recent run.") + addEvalGroupFlag(cmd, &groupName) cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") return cmd } From ca09620770a72d1bf74edbf7814f5e73fc61f3b0 Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 28 Jul 2026 12:46:56 +0300 Subject: [PATCH 045/320] fix(azure.ai.evaluations): publish the dataset version the config declares `version:` on a dataset was being passed to the upload helper whose argument is the version to count from, not the one to write. So a config declaring version "1.0" published 2.0, and every later edit climbed again. The author never got the version they asked for and nothing said so. The field also meant two different things depending on the branch taken: unchanged content resolved to the declared version, changed content published one above it. A declared version is now the version published. If it already exists and the local file differs, that is the author's decision to make, so the deploy stops and says to raise the pin or drop it. Without a pin nothing changes: each deploy still takes the next version. Verified live: version "1.0" on a new dataset publishes 1.0, and editing the file while still pinned to 1.0 fails with that instruction instead of quietly publishing 2.0. --- .../internal/cmd/reconciler.go | 22 +++++++++++++++ .../internal/pkg/dataset_api/operations.go | 22 ++++++++++++--- .../internal/pkg/dataset_api/version_test.go | 27 +++++++++++++++++++ 3 files changed, 67 insertions(+), 4 deletions(-) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go index b645c8ce6ee..4b2c52401c8 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go @@ -94,6 +94,28 @@ func (r *evalReconciler) EnsureDataset( dir = filepath.Dir(localPath) } + // A declared version is the version to publish, not one to count from. + // Reaching here means the content differs from what that version holds, so + // republishing over it would change a version the author pinned. + if decl.Version != "" { + ds, err := r.ec.datasetClient.UploadVersion( + ctx, decl.Name, decl.Version, dir, ProjectEndpointAPIVersion, + ) + if err != nil { + if dataset_api.IsVersionConflict(err) { + return "", false, fmt.Errorf( + "dataset %q version %s already exists and the local file differs from it. "+ + "Raise `version:` to publish the change, or drop it to let each "+ + "deploy take the next version", + decl.Name, decl.Version) + } + return "", false, err + } + _ = r.ec.setEnvValue(ctx, key, digest) + _ = r.ec.setEnvValue(ctx, versionKey("dataset", decl.Name), ds.Version) + return ds.Version, true, nil + } + // UploadNextVersion discovers the currently registered version when none is // declared, so the upload does not restart at 1.0 and collide. ds, err := r.ec.datasetClient.UploadNextVersion( diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/operations.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/operations.go index be1e3e03e5f..c285c19a450 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/operations.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/operations.go @@ -107,9 +107,8 @@ func (c *DatasetClient) UploadNextVersion( if currentVersion == "" { currentVersion = c.latestRegisteredVersion(ctx, name, apiVersion) } - ds, err := c.UploadNewVersion(ctx, name, currentVersion, localDir, apiVersion) - if err == nil || !isVersionConflict(err) { + if err == nil || !IsVersionConflict(err) { return ds, err } @@ -136,7 +135,7 @@ func (c *DatasetClient) latestRegisteredVersion( // isVersionConflict reports whether the service refused the upload because the // target version already exists. -func isVersionConflict(err error) bool { +func IsVersionConflict(err error) bool { var respErr *azcore.ResponseError if !errors.As(err, &respErr) { return false @@ -156,13 +155,28 @@ func (c *DatasetClient) UploadNewVersion( currentVersion string, localDir string, apiVersion string, +) (*Dataset, error) { + return c.UploadVersion(ctx, name, NextVersion(currentVersion), localDir, apiVersion) +} + +// UploadVersion publishes the dataset at exactly this version. +// +// Separate from UploadNewVersion because its parameter is the version to +// count from, not the one to write: passing "1.0" there publishes 2.0. An +// author who declares a version means that version. +func (c *DatasetClient) UploadVersion( + ctx context.Context, + name string, + version string, + localDir string, + apiVersion string, ) (*Dataset, error) { content, err := ReadFirstJSONLFile(localDir) if err != nil { return nil, fmt.Errorf("reading dataset from %s: %w", localDir, err) } - newVersion := NextVersion(currentVersion) + newVersion := version // Step 1: Start pending upload to get a SAS URI. pending, err := c.StartPendingUpload(ctx, name, newVersion, apiVersion) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/version_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/version_test.go index de822def3cd..052a63ae504 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/version_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/version_test.go @@ -39,3 +39,30 @@ func TestVersionGreaterIgnoresUnorderable(t *testing.T) { require.False(t, VersionGreater("", "1.0")) require.False(t, VersionGreater("1.0", "")) } + +// The two upload entry points read their version argument differently, and the +// difference is the whole point: UploadNewVersion counts from it, UploadVersion +// writes it. Passing "1.0" to the counting one publishes 2.0, which is not what +// an author who wrote version: "1.0" asked for. +func TestNextVersionCountsFromTheArgument(t *testing.T) { + if got := NextVersion("1.0"); got != "2.0" { + t.Fatalf("NextVersion(1.0) = %q, want 2.0", got) + } + if got := NextVersion("1"); got != "2.0" { + t.Fatalf("NextVersion(1) = %q, want 2.0", got) + } + // An unknown current version starts the sequence rather than guessing. + if got := NextVersion(""); got != "1.0" { + t.Fatalf("NextVersion(empty) = %q, want 1.0", got) + } +} + +func TestLatestVersionOrdersNumerically(t *testing.T) { + got := LatestVersion([]Dataset{{Version: "1.0"}, {Version: "10.0"}, {Version: "2.0"}}) + if got != "10.0" { + t.Fatalf("LatestVersion = %q, want 10.0 (numeric, not lexical)", got) + } + if LatestVersion(nil) != "" { + t.Fatal("LatestVersion(nil) should be empty") + } +} From 131f172043a4053d8eb43111ce239c4affaddbec Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 28 Jul 2026 12:55:38 +0300 Subject: [PATCH 046/320] fix(azure.ai.evaluations): refuse an evaluator version the service will not honour An evaluator declaring both a source and a version published at whatever version came next and the group bound that, so the pin described a version nothing used. A config asking for version 7 deployed version 1 and said nothing. Unlike a dataset, the version here is not ours to choose: the service assigns it on publish. So there is nothing to honour, and the field is refused alongside a source rather than accepted and ignored. The message names both ways out, since either is a coherent thing to have meant: drop version to publish the file, or drop source to reference a version already published. Both remaining forms verified live: source alone publishes and then reports unchanged, version alone references what is on the project. --- .../internal/project/eval_config.go | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go index 13b1eba685f..2a12cb6bfab 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go @@ -118,6 +118,17 @@ func (c *EvalConfig) Validate() error { if evaluators[e.Name] { return fmt.Errorf("evaluators[%d]: duplicate evaluator name %q", i, e.Name) } + // The service assigns an evaluator's version on publish, so a declared + // one cannot be honoured alongside a source: the upload lands on + // whatever comes next and the group binds that, leaving the pin + // describing a version nothing uses. + if e.Source != "" && e.Version != "" { + return fmt.Errorf( + "evaluators[%d] (%s): `version` cannot be set with `source`, because the "+ + "service assigns the version when it publishes. Drop `version` to "+ + "publish this file, or drop `source` to reference a version already "+ + "on the project", i, e.Name) + } evaluators[e.Name] = true } From 1c42eb2032d55e50b2566806815b41c9e8824e68 Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 28 Jul 2026 13:08:13 +0300 Subject: [PATCH 047/320] fix(azure.ai.evaluations): read the built-in evaluators' published schemas Criteria are meant to be shaped from each evaluator's own contract. For built-ins they never were: the schemas were fetched with an unfiltered list, which returns only the project's own evaluators. Built-ins have to be asked for by type, so every one of them fell back to the legacy field list instead. Nothing looked wrong because that list is query, response, tool_calls, tool_definitions, which is exactly what the common evaluators want. It fails where an evaluator wants anything else. builtin.task_completion at conversation level wants messages, so its criterion was published with an empty data_mapping and no evaluation_level: a group that could not score a single row, and no error anywhere. Also drops the requirement that a group declare a target. A dataset holding both sides of an exchange has nothing to invoke, and the service runs it happily; refusing it was ours alone. Verified live: the same group now publishes messages bound to {{item.messages}} with evaluation_level conversation, and runs to 1 passed / 1 failed / 0 errored. Note for anyone reading the earlier verification: groups built from these schemas were confirmed accepted by the service, and that is still true. The service accepts an empty mapping. Accepted was never the same as correct. --- .../internal/cmd/build.go | 24 ++++++++++++++++--- .../azure.ai.evaluations/internal/cmd/run.go | 15 ++++++++---- .../internal/pkg/eval_api/models.go | 11 +++++++++ 3 files changed, 42 insertions(+), 8 deletions(-) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build.go index 2fe71d974a5..c92ff0bccf3 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build.go @@ -18,12 +18,30 @@ import ( // // A failure is deliberately not fatal: without schemas the builder falls back // to the agent-target shape, which is what it always used to send. +// evaluatorSchemas indexes the published contract of every evaluator a group +// can reference. +// +// Built-ins have to be asked for separately. An unfiltered list returns only +// the project's own evaluators, so relying on it leaves every built-in without +// a schema and falling back to legacyInputs — which happens to match +// query/response and so looks right for the common evaluators while quietly +// dropping the fields anything else needs. func (ec *evalContext) evaluatorSchemas(ctx context.Context) map[string]*eval_api.EvaluatorSummary { - list, err := ec.evalClient.ListEvaluators(ctx, "", ProjectEndpointAPIVersion) - if err != nil { + index := map[string]*eval_api.EvaluatorSummary{} + + for _, filter := range []string{"", eval_api.EvaluatorTypeBuiltin} { + list, err := ec.evalClient.ListEvaluators(ctx, filter, ProjectEndpointAPIVersion) + if err != nil { + continue + } + for name, summary := range list.ByName() { + index[name] = summary + } + } + if len(index) == 0 { return nil } - return list.ByName() + return index } // sampleBindings are the fields an agent target produces at run time. Anything diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go index 31299be89de..d7a3bb0e3eb 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go @@ -393,15 +393,20 @@ func (ec *evalContext) buildRunDataSource( configPath string, maxSamples int, ) (*eval_api.EvalRunDataSource, error) { - if group == nil || group.Target == nil { - return nil, fmt.Errorf( - "the eval group must declare a target so the run knows what to invoke") + if group == nil { + return nil, fmt.Errorf("no eval group to run") } + // A group with no target scores a dataset that already holds the exchange, + // so there is nothing to invoke. That is how recorded conversations are + // evaluated. var ds *eval_api.EvalRunDataSource - if group.Target.Type == project.TargetTypeModel { + switch { + case group.Target == nil || group.Target.Name == "": + ds = eval_api.NewDatasetOnlyDataSource() + case group.Target.Type == project.TargetTypeModel: ds = eval_api.NewModelTargetDataSource(group.Target.Name) - } else { + default: ds = eval_api.NewAgentTargetDataSource(group.Target.Name, nil) } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/models.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/models.go index 8e1ba82f1bf..506fefd9578 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/models.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/models.go @@ -325,6 +325,9 @@ const ( // EvalRunDataSourceTypeResponses evaluates responses the project already // stored, addressed by id. EvalRunDataSourceTypeResponses EvalRunDataSourceType = "azure_ai_responses" + + // EvalRunDataSourceTypeJSONL scores the rows as they are, invoking nothing. + EvalRunDataSourceTypeJSONL EvalRunDataSourceType = "jsonl" ) // EvalRunDataContentType defines the source type for eval run data content. @@ -436,6 +439,14 @@ func NewTracesDataSource(agentName string, lookbackHours int, end time.Time, max return ds } +// NewDatasetOnlyDataSource scores the dataset as it stands, invoking nothing. +// +// Used when a group declares no target: the rows already hold both sides of +// the exchange, which is how a recorded conversation is evaluated. +func NewDatasetOnlyDataSource() *EvalRunDataSource { + return &EvalRunDataSource{Type: EvalRunDataSourceTypeJSONL} +} + // NewModelTargetDataSource sends the dataset's questions straight to a model // deployment, with no agent in front of it. // From ed62aeed1f824778f3ade07d90ba5af5f2012df7 Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 28 Jul 2026 13:15:02 +0300 Subject: [PATCH 048/320] test(azure.ai.evaluations): cover the lookup that supplies evaluator schemas The built-in schemas were never reaching the builder in the shipping path, and the test that walks every built-in could not see it: it fetched the schemas itself with the Builtin filter and handed them in. It was building the input the product was failing to build, so the builder was verified against every published contract while the code that supplies them resolved none. That test now takes its schemas from the production lookup, so a break in supply breaks the test. Two more cover the lookup directly: one asserts built-ins resolve at all and reports how many the unfiltered listing returns, the other that a conversation-level evaluator binds its conversation field rather than producing the empty mapping that scores nothing. Checked by reverting the fix: the new test fails with "the unfiltered listing returns 0 of them", and all ten built-ins pass again with it restored. --- .../internal/cmd/build_live_test.go | 11 ++- .../internal/cmd/schemas_live_test.go | 96 +++++++++++++++++++ 2 files changed, 106 insertions(+), 1 deletion(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/schemas_live_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build_live_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build_live_test.go index 0eefc25a34c..124b56c0249 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build_live_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build_live_test.go @@ -64,11 +64,20 @@ func TestLiveBuildAcceptedForEveryBuiltin(t *testing.T) { listed, err := client.ListEvaluators(ctx, eval_api.EvaluatorTypeBuiltin, ProjectEndpointAPIVersion) require.NoError(t, err) require.NotEmpty(t, listed.Value) - schemas := listed.ByName() + + // Deliberately the production lookup rather than the listing above. Taking + // the schemas straight from a filtered list is what let this test pass + // while the shipping path resolved none of them: it built the input the + // product was failing to build. + ec := &evalContext{evalClient: client} + schemas := ec.evaluatorSchemas(ctx) + require.NotEmpty(t, schemas) for _, summary := range listed.Value { summary := summary t.Run(summary.Name, func(t *testing.T) { + require.NotNil(t, schemas[summary.Name], + "the shipping lookup did not resolve %s", summary.Name) // Give the builder a dataset carrying every column the evaluator // accepts, so a rejection means the request shape is wrong rather // than the data being genuinely absent. diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/schemas_live_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/schemas_live_test.go new file mode 100644 index 00000000000..dd852e0a778 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/schemas_live_test.go @@ -0,0 +1,96 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +//go:build live + +package cmd + +import ( + "context" + "testing" + + "azureaieval/internal/pkg/eval_api" + "azureaieval/internal/pkg/evalcore" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestLiveEvaluatorSchemasIncludesBuiltins covers the function that supplies +// the schemas, rather than the builder that consumes them. +// +// The builder was already tested against every built-in, but the test fetched +// them itself with the Builtin filter. Production did not: it listed +// unfiltered, which returns only the project's own evaluators, so every +// built-in reached the builder with no schema at all. The builder was correct +// and the criteria were still wrong, and no test could see it because each one +// constructed the input production was failing to construct. +func TestLiveEvaluatorSchemasIncludesBuiltins(t *testing.T) { + client, _ := liveEvalClient(t) + ctx := context.Background() + + // The listing production used to rely on, to show what it omits. + unfiltered, err := client.ListEvaluators(ctx, "", ProjectEndpointAPIVersion) + require.NoError(t, err) + builtinsInUnfiltered := 0 + for _, e := range unfiltered.Value { + if eval_api.IsBuiltinEvaluator(e.Name) { + builtinsInUnfiltered++ + } + } + + ec := &evalContext{evalClient: client} + schemas := ec.evaluatorSchemas(ctx) + require.NotEmpty(t, schemas, "no evaluator schemas were resolved at all") + + builtins := 0 + for name, summary := range schemas { + if !eval_api.IsBuiltinEvaluator(name) { + continue + } + builtins++ + assert.NotNil(t, summary.DataSchema(), + "%s resolved without the contract the criteria are shaped from", name) + } + + require.NotZero(t, builtins, + "built-ins must be resolvable; the unfiltered listing returns %d of them, "+ + "so they have to be asked for by type", builtinsInUnfiltered) +} + +// The fields an evaluator declares are the ones its criterion has to bind, so +// a conversation-level evaluator must resolve to its conversation field. +func TestLiveConversationEvaluatorBindsMessages(t *testing.T) { + client, judge := liveEvalClient(t) + ctx := context.Background() + + ec := &evalContext{evalClient: client} + schemas := ec.evaluatorSchemas(ctx) + require.NotEmpty(t, schemas) + + var name string + for n, summary := range schemas { + if eval_api.IsBuiltinEvaluator(n) && summary.SupportsLevel("conversation") { + if ds := summary.DataSchema(); ds != nil && ds.Accepts(conversationField) { + name = n + break + } + } + } + if name == "" { + t.Skip("no built-in advertises a conversation contract on this project") + } + + plan, err := planCriterion( + evalcore.EvaluatorRef{Name: name}, + schemas[name], + nil, // no target: the dataset holds both sides of the exchange + map[string]bool{conversationField: true}, + judge, + "conversation", + ) + require.NoError(t, err) + assert.Equal(t, "{{item."+conversationField+"}}", plan.dataMapping[conversationField], + "%s must bind its conversation field", name) + assert.NotEmpty(t, plan.dataMapping, "an empty mapping scores nothing") +} From 36b51c9a8b994b9dd21b8981bc3469b90347e48b Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 28 Jul 2026 13:21:12 +0300 Subject: [PATCH 049/320] test(azure.ai.evaluations): pin the decisions that fail quietly Coverage across the decision-making code was thin in exactly the places this extension has been getting wrong: precedence rules and URI resolution, where a wrong answer produces no error at all. Precedence: which of a flag and a config option wins, for the sample cap and the evaluation level. options.max_samples was already parsed and dropped once, and nothing failed when it was; a test would have said so. Level filtering: that a conversation evaluator is sent messages and a turn one query and response, in both directions, and that a required field survives the filter so a real conflict still surfaces as a missing field rather than being reshaped away. URI resolution: the service spells these fields several ways and a URI read from the wrong spelling comes back empty rather than wrong, which is how the dataset URI went unbound the first time. Also that an upload's SAS-bearing URI and its plain finalize URI are read from their own places, since confusing them fails at different stages. --- .../internal/cmd/resolution_test.go | 74 ++++++++++++++++++ .../internal/pkg/dataset_api/uri_test.go | 75 +++++++++++++++++++ 2 files changed, 149 insertions(+) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/resolution_test.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/uri_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/resolution_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/resolution_test.go new file mode 100644 index 00000000000..9bd441b0f4b --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/resolution_test.go @@ -0,0 +1,74 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "testing" + + "azureaieval/internal/project" + + "github.com/stretchr/testify/assert" +) + +// Precedence decides behaviour without announcing it, so a wrong answer here +// is silent. options.max_samples was parsed and dropped once already, which is +// what these lock down. +func TestResolveMaxSamples_Precedence(t *testing.T) { + withOptions := &project.EvalGroup{Options: &project.Options{MaxSamples: 25}} + + assert.Equal(t, 5, resolveMaxSamples(5, withOptions), "the flag wins over the config") + assert.Equal(t, 25, resolveMaxSamples(0, withOptions), "the config is used when no flag is given") + assert.Equal(t, 0, resolveMaxSamples(0, &project.EvalGroup{}), "neither means no cap") + assert.Equal(t, 0, resolveMaxSamples(0, nil)) + assert.Equal(t, 7, resolveMaxSamples(7, nil), "a flag stands on its own") + + // Zero in config is absent, not a cap of zero: a cap of zero would send + // nothing at all. + assert.Equal(t, 0, resolveMaxSamples(0, &project.EvalGroup{Options: &project.Options{MaxSamples: 0}})) +} + +func TestResolveLevel_Precedence(t *testing.T) { + withOptions := &project.EvalGroup{ + Options: &project.Options{EvaluationLevel: project.EvaluationLevelConversation}, + } + + assert.Equal(t, project.EvaluationLevelTurn, resolveLevel(project.EvaluationLevelTurn, withOptions), + "the flag wins over the config") + assert.Equal(t, project.EvaluationLevelConversation, resolveLevel("", withOptions)) + assert.Empty(t, resolveLevel("", &project.EvalGroup{}), "unset defers to the service default") + assert.Empty(t, resolveLevel("", nil)) +} + +// A group's target decides which run-time fields its criteria can bind. Getting +// this wrong passes validation and then errors on every row. +func TestSampleBindingsFor_UnknownTargetBindsNothing(t *testing.T) { + assert.Nil(t, sampleBindingsFor("prompt"), + "an unrecognised target must bind nothing rather than guess at agent fields") +} + +// The level filter is what keeps a conversation evaluator from being sent turn +// fields and the reverse. Both directions matter. +func TestSelectLevelFields_KeepsOnlyTheLevelsShape(t *testing.T) { + accepted := []string{"query", "response", "messages", "tool_definitions"} + + conv := selectLevelFields(accepted, nil, project.EvaluationLevelConversation) + assert.Contains(t, conv, "messages") + assert.NotContains(t, conv, "query") + assert.NotContains(t, conv, "response") + assert.Contains(t, conv, "tool_definitions", "fields outside the split are untouched") + + turn := selectLevelFields(accepted, nil, project.EvaluationLevelTurn) + assert.Contains(t, turn, "query") + assert.Contains(t, turn, "response") + assert.NotContains(t, turn, "messages") + + // An evaluator offering only one shape is left alone, whatever the level. + only := []string{"query", "response"} + assert.Equal(t, only, selectLevelFields(only, nil, project.EvaluationLevelConversation)) + + // A required field is never dropped: a genuine conflict has to surface as a + // missing-field error rather than being reshaped away. + kept := selectLevelFields(accepted, []string{"query"}, project.EvaluationLevelConversation) + assert.Contains(t, kept, "query", "a required field survives the level filter") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/uri_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/uri_test.go new file mode 100644 index 00000000000..e6cc46c2e3d --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/uri_test.go @@ -0,0 +1,75 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package dataset_api + +import ( + "encoding/json" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// The service spells these fields inconsistently, and a URI read from the +// wrong spelling comes back empty rather than wrong — which is how the dataset +// URI went unbound the first time. +func TestDatasetResolvedBlobURI_AcceptsEitherSpelling(t *testing.T) { + cases := map[string]string{ + `{"dataUri":"https://x/y.jsonl"}`: "https://x/y.jsonl", + `{"data_uri":"https://x/y.jsonl"}`: "https://x/y.jsonl", + `{"blobUri":"https://x/b.jsonl"}`: "https://x/b.jsonl", + `{"contentUri":"https://x/c.jsonl"}`: "https://x/c.jsonl", + } + for body, want := range cases { + var ds Dataset + require.NoError(t, json.Unmarshal([]byte(body), &ds), body) + assert.Equal(t, want, ds.ResolvedBlobURI(), body) + } + + var none Dataset + require.NoError(t, json.Unmarshal([]byte(`{"name":"x"}`), &none)) + assert.Empty(t, none.ResolvedBlobURI(), + "no URI means the caller has to fetch a credential, not that the dataset is unreadable") +} + +// An upload needs the SAS-bearing URI to write to and the plain one to +// finalize with. Confusing them fails at different stages, so both are read +// from their own place. +func TestPendingUploadURIs(t *testing.T) { + var p PendingUploadResponse + require.NoError(t, json.Unmarshal([]byte(`{ + "blobReference": { + "blobUri": "https://acct.blob.core.windows.net/container", + "credential": { "sasUri": "https://acct.blob.core.windows.net/container?sig=abc" } + } + }`), &p)) + + assert.Equal(t, "https://acct.blob.core.windows.net/container?sig=abc", p.ResolvedUploadURI(), + "the upload target carries the SAS") + assert.Equal(t, "https://acct.blob.core.windows.net/container", p.ResolvedBlobURI(), + "the finalize URI does not") + + var empty PendingUploadResponse + assert.Empty(t, empty.ResolvedUploadURI()) + assert.Empty(t, empty.ResolvedBlobURI()) +} + +// Credentials arrive in two shapes and the consumption one takes precedence, +// because that is the one scoped for reading. +func TestCredentialResolvedDownloadURI(t *testing.T) { + var c DatasetCredential + require.NoError(t, json.Unmarshal([]byte(`{ + "blobReferenceForConsumption": { "credential": { "sasUri": "https://acct/read?sig=r" } }, + "blobReference": { "credential": { "sasUri": "https://acct/write?sig=w" } } + }`), &c)) + assert.Equal(t, "https://acct/read?sig=r", c.ResolvedDownloadURI()) + + var legacy DatasetCredential + require.NoError(t, json.Unmarshal([]byte(`{"sas_uri":"https://acct/legacy?sig=l"}`), &legacy)) + assert.Equal(t, "https://acct/legacy?sig=l", legacy.ResolvedDownloadURI(), + "the flat spelling is still honoured") + + var none DatasetCredential + assert.Empty(t, none.ResolvedDownloadURI()) +} From b28dfaa7f142d73864ac8f84a86eda2f50b286ac Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 28 Jul 2026 13:51:43 +0300 Subject: [PATCH 050/320] Honour a hand-set EVAL_GROUP_ID again for single-group configs Giving each eval group its own env entry stopped a second group from adopting the first one's id, but it also removed the only read of EVAL_GROUP_ID on the run path. Setting that key by hand is the documented way to point a config at a group created in the portal or by another tool, so the documented behaviour quietly stopped working: the override was ignored and a new group created. The shared key is read again, but only when the config declares a single group. With more than one there is no way to tell which group it refers to, which is what caused the aliasing in the first place. --- .../internal/cmd/envkeys_test.go | 17 ++++++++++++ .../azure.ai.evaluations/internal/cmd/run.go | 27 +++++++++++++++++-- 2 files changed, 42 insertions(+), 2 deletions(-) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/envkeys_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/envkeys_test.go index be8090b9d06..dbda6180b23 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/envkeys_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/envkeys_test.go @@ -35,3 +35,20 @@ func TestIDKey_NormalizesNames(t *testing.T) { func TestIDKey_DoesNotCollideWithVersionKey(t *testing.T) { assert.NotEqual(t, idKey("dataset", "golden"), versionKey("dataset", "golden")) } + +// Setting EVAL_GROUP_ID by hand is the documented way to point a config at a +// group that already exists. It is also the key the extension writes itself, +// which is what let a second group adopt the first one's id — so it stays +// readable only where it cannot be ambiguous. Fixing the aliasing dropped this +// fallback entirely once, silently breaking the documented behaviour. +func TestGroupIDKeys_SharedKeyReadOnlyWhenUnambiguous(t *testing.T) { + sole := groupIDKeys("quality", true) + assert.Equal(t, idKey("evalgroup", "quality"), sole[0], + "a group's own entry is preferred over the shared one") + assert.Contains(t, sole, envKeyEvalGroupID, + "a single-group config honours an id set by hand") + + assert.Equal(t, []string{idKey("evalgroup", "quality")}, groupIDKeys("quality", false), + "with several groups the shared entry cannot say which group it means") +} + diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go index d7a3bb0e3eb..3f5d81ebb0f 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go @@ -96,7 +96,8 @@ func buildRunCommand(use, short string) *cobra.Command { } evalID, err = ec.resolveEvalGroupID( - ctx, group, configPath, resolveLevel(level, group), out, isJSON(cmd)) + ctx, group, configPath, resolveLevel(level, group), + len(cfg.EvalGroups) == 1, out, isJSON(cmd)) if err != nil { return err } @@ -220,6 +221,7 @@ func (ec *evalContext) resolveEvalGroupID( group *project.EvalGroup, configPath string, level string, + soleGroup bool, out interface{ Write([]byte) (int, error) }, jsonMode bool, ) (string, error) { @@ -227,7 +229,11 @@ func (ec *evalContext) resolveEvalGroupID( return group.ID, nil } - if cached := ec.getEnvValue(ctx, idKey("evalgroup", group.Name)); cached != "" { + for _, key := range groupIDKeys(group.Name, soleGroup) { + cached := ec.getEnvValue(ctx, key) + if cached == "" { + continue + } // Confirm it still exists; a deleted group should fall through to create. if _, err := ec.evalClient.GetOpenAIEval(ctx, cached); err == nil { return cached, nil @@ -269,6 +275,23 @@ func (ec *evalContext) resolveEvalGroupID( return created.ID, nil } +// groupIDKeys lists the env entries that may hold this group's id, most +// specific first. +// +// The per-name entry is what the extension writes. EVAL_GROUP_ID is also the +// documented way to point a config at a group that already exists, created in +// the portal or by another tool, so it stays readable — but only when the +// config declares a single group. With more than one there is no way to tell +// which group a shared entry refers to, and reading it anyway is what let a +// second group adopt the first one's id. +func groupIDKeys(name string, soleGroup bool) []string { + keys := []string{idKey("evalgroup", name)} + if soleGroup { + keys = append(keys, envKeyEvalGroupID) + } + return keys +} + // checkDatasetRegistered fails when the group's local dataset has edits that // were never deployed. // From 69a6ec8527a4b0a5e5be1910f90acb92e0f6550c Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 28 Jul 2026 13:51:53 +0300 Subject: [PATCH 051/320] Emit a JSON array from every list command The list commands passed the service's own envelope straight through, and the two services behind them do not agree on one: runs came back as {"data": [..]} and datasets, evaluators and schedules as {"value": [..]}. A script had to know which API happened to back each command to read its output, and the envelopes also carry paging fields the extension never follows, implying there is more to fetch when there is not. Each list now emits a bare array. A nil slice is normalized to [] so that an empty listing is still something a caller can iterate rather than null. --- .../internal/cmd/dataset.go | 2 +- .../internal/cmd/evaluator.go | 2 +- .../internal/cmd/output.go | 18 +++++++++++ .../internal/cmd/output_test.go | 31 +++++++++++++++++++ .../internal/cmd/run_ops.go | 6 +++- .../internal/cmd/schedule.go | 9 ++++-- 6 files changed, 63 insertions(+), 5 deletions(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/output_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset.go index 5c18f453297..3e4513b07c3 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset.go @@ -138,7 +138,7 @@ func newDatasetListCommand() *cobra.Command { } if isJSON(cmd) { - return emitJSON(cmd.OutOrStdout(), list) + return emitJSONList(cmd.OutOrStdout(), list.Value) } rows := make([][]string, 0, len(list.Value)) for _, d := range list.Value { diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator.go index ca1d597eaa3..4ec85fca946 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator.go @@ -227,7 +227,7 @@ func newEvaluatorBuiltinsCommand() *cobra.Command { func renderEvaluators(cmd *cobra.Command, list *eval_api.EvaluatorListResponse) error { if isJSON(cmd) { - return emitJSON(cmd.OutOrStdout(), list) + return emitJSONList(cmd.OutOrStdout(), list.Value) } if len(list.Value) == 0 { fmt.Fprintln(cmd.OutOrStdout(), "No evaluators found.") diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/output.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/output.go index ee6a1928fdb..319ed2565f0 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/output.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/output.go @@ -39,6 +39,24 @@ func emitJSON(w io.Writer, v any) error { return enc.Encode(v) } +// emitJSONList writes items as a JSON array. +// +// List commands emit a bare array rather than the envelope the service replied +// with. The envelopes disagree with each other — the OpenAI-shaped APIs wrap +// results in `data`, the ARM-shaped ones in `value` — so passing them through +// would make a caller's parsing depend on which service happens to back a given +// command. They also carry paging fields that this extension does not follow, +// which would suggest there is more to fetch when there is not. +// +// A nil slice encodes as `null`, so it is normalized to an empty array: a +// caller iterating the result should see no elements, not a type error. +func emitJSONList[T any](w io.Writer, items []T) error { + if items == nil { + items = []T{} + } + return emitJSON(w, items) +} + // emitTable writes a simple aligned table. Rows must match the header width. func emitTable(w io.Writer, headers []string, rows [][]string) error { tw := tabwriter.NewWriter(w, 0, 0, 3, ' ', 0) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/output_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/output_test.go new file mode 100644 index 00000000000..2914e5c3fbd --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/output_test.go @@ -0,0 +1,31 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "bytes" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// The list commands are backed by two different services whose envelopes +// disagree — `data` on one side, `value` on the other. Emitting whichever one +// came back would make a caller's parsing depend on that accident, so every +// list emits a bare array instead. +func TestEmitJSONList_EmitsAnArrayNotAnEnvelope(t *testing.T) { + var buf bytes.Buffer + require.NoError(t, emitJSONList(&buf, []string{"a", "b"})) + assert.Equal(t, "[\n \"a\",\n \"b\"\n]\n", buf.String()) +} + +// A nil slice marshals to `null`, which a caller iterating the output cannot +// range over. An empty listing has to come back as an empty array. +func TestEmitJSONList_NilBecomesEmptyArray(t *testing.T) { + var buf bytes.Buffer + var none []string + require.NoError(t, emitJSONList(&buf, none)) + assert.Equal(t, "[]\n", buf.String()) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go index 196a989a3bc..509f7f48026 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go @@ -52,7 +52,11 @@ func newRunListCommand() *cobra.Command { return fmt.Errorf("listing runs for %q: %w", evalID, err) } if isJSON(cmd) { - return emitJSON(cmd.OutOrStdout(), list) + var runs []eval_api.OpenAIEvalRun + if list != nil { + runs = list.Data + } + return emitJSONList(cmd.OutOrStdout(), runs) } if list == nil || len(list.Data) == 0 { fmt.Fprintf(cmd.OutOrStdout(), "Eval group %s has no runs yet.\n", evalID) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/schedule.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/schedule.go index 96f5429e060..ae86cb60dc8 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/schedule.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/schedule.go @@ -117,7 +117,8 @@ func newScheduleSetCommand() *cobra.Command { return err } evalID, err = ec.resolveEvalGroupID( - ctx, group, configPath, resolveLevel(level, group), out, isJSON(cmd)) + ctx, group, configPath, resolveLevel(level, group), + len(cfg.EvalGroups) == 1, out, isJSON(cmd)) if err != nil { return err } @@ -239,7 +240,11 @@ func newScheduleListCommand() *cobra.Command { return fmt.Errorf("listing schedules: %w", err) } if isJSON(cmd) { - return emitJSON(out, list) + var schedules []eval_api.Schedule + if list != nil { + schedules = list.Value + } + return emitJSONList(out, schedules) } if list == nil || len(list.Value) == 0 { fmt.Fprintln(out, "No schedules.") From ab06f367184f6abf96260c6a0491a0859ebe9f7f Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 28 Jul 2026 14:12:13 +0300 Subject: [PATCH 052/320] Accept --eval-id on every command that takes an eval group `run start --eval-id ` is the form the CI example uses, but the sibling commands accepted the id only as a positional argument, so `run list --eval-id` failed with "unknown flag". Writing the verification script for the spec examples is what surfaced it: the flag learned in one command does not work in the next one a script reaches for. --eval-id now sits alongside --eval-group on run list/show/cancel and results show/export/compare, resolved with the same precedence as before. The positional argument keeps working and still wins. --- .../internal/cmd/compare.go | 2 +- .../internal/cmd/results.go | 19 ++++++++++++--- .../internal/cmd/run_ops.go | 6 ++--- .../internal/cmd/run_ops_test.go | 24 +++++++++++++++++++ 4 files changed, 44 insertions(+), 7 deletions(-) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/compare.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/compare.go index 815f4ab47c6..05775d3a1e4 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/compare.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/compare.go @@ -90,7 +90,7 @@ func newResultsCompareCommand() *cobra.Command { cmd.Flags().StringArrayVar(&treatments, "treatment", nil, "Run to measure, repeatable. Defaults to the most recent completed run.") cmd.Flags().StringVar(&displayName, "name", "", "Name for this comparison.") - addEvalGroupFlag(cmd, &groupName) + addEvalGroupFlags(cmd, &groupName) cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") return cmd } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/results.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/results.go index 3a2f4d87f5a..1fc0ee4157b 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/results.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/results.go @@ -74,7 +74,7 @@ func newResultsShowCommand() *cobra.Command { cmd.Flags().StringVar(&runID, "run-id", "", "Run to show. Defaults to the most recent run.") cmd.Flags().BoolVar(&failedOnly, "failed-only", false, "Show only criteria with failures.") cmd.Flags().StringVarP(&outFile, "out-file", "O", "", "Write JSON results to this path.") - addEvalGroupFlag(cmd, &groupName) + addEvalGroupFlags(cmd, &groupName) cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") return cmd } @@ -135,7 +135,7 @@ func newResultsExportCommand() *cobra.Command { cmd.Flags().StringVar(&runID, "run-id", "", "Run to export. Defaults to the most recent run.") cmd.Flags().StringVar(&format, "format", "json", "Output format: json or csv.") cmd.Flags().StringVarP(&outFile, "out-file", "O", "", "Write to this path instead of stdout.") - addEvalGroupFlag(cmd, &groupName) + addEvalGroupFlags(cmd, &groupName) cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") return cmd } @@ -156,6 +156,10 @@ func resolveEvalID( return args[0], nil } + if flag, err := cmd.Flags().GetString("eval-id"); err == nil && flag != "" { + return flag, nil + } + if groupName != "" { if id := ec.getEnvValue(cmd.Context(), idKey("evalgroup", groupName)); id != "" { return id, nil @@ -177,9 +181,18 @@ func resolveEvalID( // addEvalGroupFlag registers the flag that names a group from the config, so // every command taking an eval-id can reach a group by the name its author // used. -func addEvalGroupFlag(cmd *cobra.Command, target *string) { +// addEvalGroupFlags registers the two ways to say which group a command acts +// on: --eval-group names one from the config, --eval-id gives its service id. +// +// The id is also accepted as a positional argument. The flag exists because +// `run start --eval-id` already spells it that way, and a script that learned +// it there should not have to find out that the sibling commands take only a +// positional. +func addEvalGroupFlags(cmd *cobra.Command, target *string) { cmd.Flags().StringVar(target, "eval-group", "", "Name a group from the config instead of passing its id.") + cmd.Flags().String("eval-id", "", + "Id of the eval group. Same as passing the id as an argument.") } // latestOrNamedRun returns the named run, or the most recent one for the group. diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go index 509f7f48026..1ff2ee7cebb 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go @@ -71,7 +71,7 @@ func newRunListCommand() *cobra.Command { []string{"RUN ID", "NAME", "STATUS", "RESULTS"}, rows) }, } - addEvalGroupFlag(cmd, &groupName) + addEvalGroupFlags(cmd, &groupName) cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") return cmd } @@ -122,7 +122,7 @@ func newRunShowCommand() *cobra.Command { }, } cmd.Flags().StringVar(&runID, "run-id", "", "Run to show. Defaults to the most recent run.") - addEvalGroupFlag(cmd, &groupName) + addEvalGroupFlags(cmd, &groupName) cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") return cmd } @@ -178,7 +178,7 @@ func newRunCancelCommand() *cobra.Command { }, } cmd.Flags().StringVar(&runID, "run-id", "", "Run to cancel. Defaults to the most recent run.") - addEvalGroupFlag(cmd, &groupName) + addEvalGroupFlags(cmd, &groupName) cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") return cmd } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops_test.go index 63ef3344e2e..edc4d99ccef 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops_test.go @@ -62,6 +62,30 @@ func TestRunStartMirrorsCompositeFlags(t *testing.T) { } } +// Every command that acts on an eval group takes the id the same two ways. +// `run start --eval-id` is the form the CI example uses, and `run list` used to +// reject that flag and accept only a positional, so a script that worked for +// one sibling failed on the next. +func TestEvalGroupCommandsAcceptIDAsAFlag(t *testing.T) { + subs := map[string]*cobra.Command{} + for _, sub := range newRunCommand().Commands() { + subs["run "+sub.Name()] = sub + } + for _, sub := range newResultsCommand().Commands() { + subs["results "+sub.Name()] = sub + } + + for _, name := range []string{ + "run list", "run show", "run cancel", + "results show", "results export", "results compare", + } { + cmd := subs[name] + require.NotNil(t, cmd, "%s should exist", name) + require.NotNil(t, cmd.Flags().Lookup("eval-id"), "%s should accept --eval-id", name) + require.NotNil(t, cmd.Flags().Lookup("eval-group"), "%s should accept --eval-group", name) + } +} + // --no-wait is documented in the spec, and cobra does not derive it from the // --wait bool. func TestRunCommandAcceptsNoWait(t *testing.T) { From 3a2fb1eae23085569a6755bf3dcd0a098f0c2683 Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 28 Jul 2026 14:20:22 +0300 Subject: [PATCH 053/320] Survive the quoted NaN the service sends for an undefined statistic MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A run with a single sample has no standard deviation, and the service reports it as the string "NaN" because JSON has no NaN literal. The model declared float64, so decoding failed and the whole comparison was thrown away with "cannot unmarshal string into Go struct field ... of type float64" — including the TooFewSamples verdict that exists to explain that exact situation. Comparing a one-sample gate, which is the cheapest thing a pipeline does, could not work. The statistics are now a LenientFloat that decodes a number or any of the quoted forms, and marshals non-finite values as null, because encoding/json refuses NaN outright and would have broken -o json instead. The table prints an undefined statistic as a dash rather than the literal NaN. The earlier live check passed only because every run it compared had enough samples for the deviation to be a real number. --- .../internal/cmd/compare.go | 19 ++- .../internal/cmd/envkeys_test.go | 1 - .../internal/pkg/eval_api/insights.go | 67 +++++++++-- .../internal/pkg/eval_api/insights_test.go | 108 ++++++++++++++++++ 4 files changed, 183 insertions(+), 12 deletions(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/insights_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/compare.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/compare.go index 05775d3a1e4..f90ecfb5ef8 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/compare.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/compare.go @@ -167,6 +167,17 @@ func (ec *evalContext) pollInsight(ctx context.Context, insightID string) (*eval } // renderComparison prints one row per criterion per treatment run. +// formatStat renders a statistic, showing an undefined one as a dash. A +// standard deviation over a single sample has no value, and printing the +// literal "NaN" in a results table reads like a failure rather than the +// arithmetic it is. +func formatStat(verb string, v eval_api.LenientFloat) string { + if !v.Defined() { + return "-" + } + return fmt.Sprintf(verb, float64(v)) +} + func renderComparison(w interface{ Write([]byte) (int, error) }, insight *eval_api.Insight) error { if insight.Result == nil || len(insight.Result.Comparisons) == 0 { fmt.Fprintln(w, "The comparison produced no metrics.") @@ -181,13 +192,13 @@ func renderComparison(w interface{ Write([]byte) (int, error) }, insight *eval_a for _, c := range insight.Result.Comparisons { baseAvg := "-" if c.BaselineRunSummary != nil { - baseAvg = fmt.Sprintf("%.3f", c.BaselineRunSummary.Average) + baseAvg = formatStat("%.3f", c.BaselineRunSummary.Average) } for _, item := range c.CompareItems { treatAvg := "-" runID := "-" if item.TreatmentRunSummary != nil { - treatAvg = fmt.Sprintf("%.3f", item.TreatmentRunSummary.Average) + treatAvg = formatStat("%.3f", item.TreatmentRunSummary.Average) runID = item.TreatmentRunSummary.RunID } rows = append(rows, []string{ @@ -195,8 +206,8 @@ func renderComparison(w interface{ Write([]byte) (int, error) }, insight *eval_a runID, baseAvg, treatAvg, - fmt.Sprintf("%+.3f", item.DeltaEstimate), - fmt.Sprintf("%.3f", item.PValue), + formatStat("%+.3f", item.DeltaEstimate), + formatStat("%.3f", item.PValue), item.TreatmentEffect, }) } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/envkeys_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/envkeys_test.go index dbda6180b23..b78bd8debf9 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/envkeys_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/envkeys_test.go @@ -51,4 +51,3 @@ func TestGroupIDKeys_SharedKeyReadOnlyWhenUnambiguous(t *testing.T) { assert.Equal(t, []string{idKey("evalgroup", "quality")}, groupIDKeys("quality", false), "with several groups the shared entry cannot say which group it means") } - diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/insights.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/insights.go index 8296fd21992..0335f951d62 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/insights.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/insights.go @@ -5,9 +5,13 @@ package eval_api import ( "context" + "encoding/json" "fmt" + "math" "net/http" "net/url" + "strconv" + "strings" ) // InsightTypeEvaluationComparison compares evaluation runs. The service also @@ -34,19 +38,68 @@ type CreateInsightRequest struct { Request *InsightRequest `json:"request"` } +// LenientFloat is a float64 that also decodes the quoted forms the service +// uses for values JSON cannot express. +// +// A run with a single sample has an undefined standard deviation, and the +// service sends it as the string "NaN" because JSON has no NaN literal. +// Decoding that into a plain float64 fails the entire comparison — including +// the TooFewSamples verdict that exists to explain exactly this case — so a +// one-sample gate reported a parse error instead of its result. +type LenientFloat float64 + +func (f *LenientFloat) UnmarshalJSON(data []byte) error { + s := strings.TrimSpace(string(data)) + if s == "null" { + *f = LenientFloat(math.NaN()) + return nil + } + // "NaN", "Infinity", "-Infinity" and ordinary numbers arrive quoted; + // ParseFloat accepts all of them once the quotes are gone. + if unquoted, err := strconv.Unquote(s); err == nil { + s = strings.TrimSpace(unquoted) + if s == "" { + *f = LenientFloat(math.NaN()) + return nil + } + } + v, err := strconv.ParseFloat(s, 64) + if err != nil { + return fmt.Errorf("parsing number %s: %w", data, err) + } + *f = LenientFloat(v) + return nil +} + +// MarshalJSON writes non-finite values as null. encoding/json refuses to +// marshal NaN or ±Inf at all, which would turn `-o json` into an error the +// moment a comparison contained one; null is valid JSON and reads as the +// "undefined" that a one-sample standard deviation actually is. +func (f LenientFloat) MarshalJSON() ([]byte, error) { + if math.IsNaN(float64(f)) || math.IsInf(float64(f), 0) { + return []byte("null"), nil + } + return json.Marshal(float64(f)) +} + +// Defined reports whether the value is a real number that can be shown. +func (f LenientFloat) Defined() bool { + return !math.IsNaN(float64(f)) && !math.IsInf(float64(f), 0) +} + // RunSummary is one run's aggregate for a single metric. type RunSummary struct { - RunID string `json:"runId"` - SampleCount int `json:"sampleCount"` - Average float64 `json:"average"` - StandardDeviation float64 `json:"standardDeviation"` + RunID string `json:"runId"` + SampleCount int `json:"sampleCount"` + Average LenientFloat `json:"average"` + StandardDeviation LenientFloat `json:"standardDeviation"` } // CompareItem is one treatment run measured against the baseline. type CompareItem struct { - TreatmentRunSummary *RunSummary `json:"treatmentRunSummary,omitempty"` - DeltaEstimate float64 `json:"deltaEstimate"` - PValue float64 `json:"pValue"` + TreatmentRunSummary *RunSummary `json:"treatmentRunSummary,omitempty"` + DeltaEstimate LenientFloat `json:"deltaEstimate"` + PValue LenientFloat `json:"pValue"` // TreatmentEffect classifies the result, e.g. TooFewSamples when the // sample count cannot support a conclusion. TreatmentEffect string `json:"treatmentEffect,omitempty"` diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/insights_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/insights_test.go new file mode 100644 index 00000000000..b8db134dc79 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/insights_test.go @@ -0,0 +1,108 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package eval_api + +import ( + "encoding/json" + "math" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// The exact body the service returned for a comparison of two single-sample +// runs. `standardDeviation` is the string "NaN" because JSON has no NaN +// literal, and decoding it into a float64 failed the whole comparison — losing +// the TooFewSamples verdict that explains the very situation that produced it. +const oneSampleComparison = `{ + "comparisons": [ + { + "testingCriteria": "task_adherence", + "metric": "task_adherence", + "evaluator": "builtin.task_adherence", + "baselineRunSummary": { + "runId": "evalrun_base", + "sampleCount": 1, + "average": 1.0, + "standardDeviation": "NaN" + }, + "compareItems": [ + { + "treatmentRunSummary": { + "runId": "evalrun_treat", + "sampleCount": 1, + "average": 1.0, + "standardDeviation": "NaN" + }, + "deltaEstimate": 0.0, + "pValue": 1.0, + "treatmentEffect": "TooFewSamples" + } + ] + } + ], + "method": "TTest", + "type": "EvaluationComparison" +}` + +func TestInsightResult_DecodesQuotedNaN(t *testing.T) { + var got InsightResult + require.NoError(t, json.Unmarshal([]byte(oneSampleComparison), &got)) + + require.Len(t, got.Comparisons, 1) + c := got.Comparisons[0] + require.NotNil(t, c.BaselineRunSummary) + + assert.Equal(t, 1.0, float64(c.BaselineRunSummary.Average)) + assert.False(t, c.BaselineRunSummary.StandardDeviation.Defined(), + "a single sample has no standard deviation") + + require.Len(t, c.CompareItems, 1) + assert.Equal(t, "TooFewSamples", c.CompareItems[0].TreatmentEffect, + "the verdict survives, which is the whole point of not failing the parse") + assert.Equal(t, 1.0, float64(c.CompareItems[0].PValue)) +} + +func TestLenientFloat_AcceptsBothShapes(t *testing.T) { + cases := map[string]func(LenientFloat) bool{ + `0.75`: func(f LenientFloat) bool { return float64(f) == 0.75 }, + `"0.75"`: func(f LenientFloat) bool { return float64(f) == 0.75 }, + `"NaN"`: func(f LenientFloat) bool { return !f.Defined() }, + `"Infinity"`: func(f LenientFloat) bool { return !f.Defined() }, + `"-Infinity"`: func(f LenientFloat) bool { return !f.Defined() }, + `null`: func(f LenientFloat) bool { return !f.Defined() }, + `""`: func(f LenientFloat) bool { return !f.Defined() }, + } + + for raw, ok := range cases { + var f LenientFloat + require.NoError(t, json.Unmarshal([]byte(raw), &f), "decoding %s", raw) + assert.True(t, ok(f), "unexpected value decoding %s", raw) + } + + var f LenientFloat + assert.Error(t, json.Unmarshal([]byte(`"not a number"`), &f), + "genuine garbage must still be reported") +} + +// encoding/json refuses to marshal NaN, so `-o json` would fail on any +// comparison holding one unless it is written as null. +func TestLenientFloat_MarshalsNonFiniteAsNull(t *testing.T) { + b, err := json.Marshal(LenientFloat(math.NaN())) + require.NoError(t, err) + assert.Equal(t, "null", string(b)) + + b, err = json.Marshal(LenientFloat(0.5)) + require.NoError(t, err) + assert.Equal(t, "0.5", string(b)) + + // The whole result has to survive a round trip, since that is what + // `results compare -o json` emits. + var res InsightResult + require.NoError(t, json.Unmarshal([]byte(oneSampleComparison), &res)) + out, err := json.Marshal(res) + require.NoError(t, err, "a comparison containing NaN must still emit JSON") + assert.Contains(t, string(out), `"standardDeviation":null`) +} From fe590edb259e650a8e15d80db72477b1c4d603ea Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 28 Jul 2026 14:49:21 +0300 Subject: [PATCH 054/320] Explain the two schedule failures the E2E scenarios turned up A schedule repeats the eval group's most recent run, so scheduling a group whose last run came from --from-traces makes it a trace evaluation, and the service then allows only an hourly trigger. Asking for a daily one produced a raw 400 saying "Scheduled trace evaluations only support hourly recurrence triggers", which is baffling when a trigger was the only thing asked for and traces were never mentioned. Confirmed by experiment: daily was accepted after an agent run, refused after a traces run on the same group, and hourly accepted for that same traces run. The error now names the cause and both ways out. Deleting a schedule that does not exist printed 2364 characters of service error document wrapping an inner 404 from the trigger service. A missing name is the ordinary typo, so it now says so in one line and points at schedule list. The live run test also asserted only that the run reached a terminal state. A run whose every sample errors still reports "completed", so it now requires no errored samples and at least one scored one - otherwise a broken target or evaluator would leave the suite green. --- .../internal/cmd/schedule.go | 29 +++++++++++++++++++ .../internal/cmd/schedule_test.go | 13 +++++++++ .../internal/pkg/eval_api/schedules.go | 9 ++++++ .../tests/live/live_test.go | 14 +++++++++ 4 files changed, 65 insertions(+) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/schedule.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/schedule.go index ae86cb60dc8..b2972b29ac9 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/schedule.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/schedule.go @@ -358,6 +358,15 @@ func newScheduleDeleteCommand() *cobra.Command { defer ec.Close() if err := ec.deleteScheduleWhenSettled(ctx, name); err != nil { + // A name that was never there is the common typo, and the + // service answers it with a full error document wrapping an + // inner 404 from the trigger service. Saying so in one line is + // more use than reproducing that. + if eval_api.IsNotFound(err) { + return fmt.Errorf( + "no schedule named %q in this project; "+ + "`azd ai eval schedule list` shows the ones that exist", name) + } return fmt.Errorf("deleting schedule %q: %w", name, err) } fmt.Fprintf(out, "Deleted schedule %s\n", name) @@ -583,6 +592,19 @@ func explainScheduleFailure( name string, cause error, ) error { + // A schedule repeats the group's most recent run, so scheduling a group + // whose last run came from --from-traces creates a trace evaluation, and + // the service allows only an hourly trigger for those. The message it + // returns says so without saying why it thinks the schedule is one, which + // is bewildering when the trigger was the only thing asked for. + if isTracesHourlyOnly(cause) { + return fmt.Errorf( + "saving schedule %q: this group's most recent run read from traces, and a schedule "+ + "repeats that run, so the service treats it as a scheduled trace evaluation "+ + "and allows only `--every hourly`. Use `--every hourly`, or run the group "+ + "once against its dataset first so the schedule repeats that instead", name) + } + list, listErr := ec.evalClient.ListSchedules(ctx, ProjectEndpointAPIVersion) if listErr != nil || list == nil { return fmt.Errorf("saving schedule %q: %w", name, cause) @@ -598,3 +620,10 @@ func explainScheduleFailure( } return fmt.Errorf("saving schedule %q: %w", name, cause) } + +// isTracesHourlyOnly matches the service's refusal of a non-hourly trigger on a +// schedule it considers a trace evaluation. +func isTracesHourlyOnly(err error) bool { + return err != nil && + strings.Contains(err.Error(), "trace evaluations only support hourly") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/schedule_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/schedule_test.go index 9f93207950b..f518397d843 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/schedule_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/schedule_test.go @@ -4,6 +4,7 @@ package cmd import ( + "errors" "testing" "azureaieval/internal/pkg/eval_api" @@ -20,6 +21,18 @@ func TestBuildTrigger_Cron(t *testing.T) { assert.Equal(t, "UTC", got.Timezone, "UTC unless the caller says otherwise") } +// A schedule repeats the group's most recent run. Scheduling a group whose last +// run came from --from-traces therefore creates a trace evaluation, and the +// service allows only an hourly trigger for those. Confirmed live: a daily +// trigger was accepted after an agent run, refused after a traces run on the +// same group, and hourly was accepted for that same traces run. +func TestIsTracesHourlyOnly(t *testing.T) { + assert.True(t, isTracesHourlyOnly( + errors.New(`{"message": "Scheduled trace evaluations only support hourly recurrence triggers. is invalid"}`))) + assert.False(t, isTracesHourlyOnly(errors.New("some other 400"))) + assert.False(t, isTracesHourlyOnly(nil)) +} + func TestBuildTrigger_OneTime(t *testing.T) { got, err := buildTrigger(triggerFlags{atTime: "2026-08-01T09:00:00Z", timezone: "Europe/Dublin"}) require.NoError(t, err) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/schedules.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/schedules.go index ed205bf1f79..a6ab65746a4 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/schedules.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/schedules.go @@ -173,3 +173,12 @@ func IsConflict(err error) bool { } return respErr.StatusCode == http.StatusConflict } + +// IsNotFound reports whether the service answered 404. +func IsNotFound(err error) bool { + var respErr *azcore.ResponseError + if !errors.As(err, &respErr) { + return false + } + return respErr.StatusCode == http.StatusNotFound +} diff --git a/cli/azd/extensions/azure.ai.evaluations/tests/live/live_test.go b/cli/azd/extensions/azure.ai.evaluations/tests/live/live_test.go index 9a4a209bffb..2ecceff6f7e 100644 --- a/cli/azd/extensions/azure.ai.evaluations/tests/live/live_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/tests/live/live_test.go @@ -320,6 +320,20 @@ func TestLiveRun(t *testing.T) { } body, _ := json.MarshalIndent(current.PerTestingCriteria, "", " ") t.Logf("per-criteria results: %s", string(body)) + + // Reaching a terminal state is not the same as having evaluated + // anything. A run whose every sample errors still reports + // "completed", so asserting only on the status would let the target + // or the evaluator break without the test noticing. + require.Equal(t, "completed", strings.ToLower(current.Status), + "the run must complete rather than fail or cancel") + require.NotNil(t, current.ResultCounts, "a completed run must report counts") + require.Zero(t, current.ResultCounts.Errored, + "an errored sample means the target or the evaluator did not run") + require.Positive(t, + current.ResultCounts.Passed+current.ResultCounts.Failed, + "the run must score at least one sample; a pass or a fail are both fine, "+ + "but scoring nothing means the data never reached the evaluator") return } if time.Now().After(deadline) { From 42f8fe120a5816591a1c39b07bd1260ac458b044 Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 28 Jul 2026 18:06:55 +0300 Subject: [PATCH 055/320] Declare the eval service in the root azure.yaml instead of describing it init scaffolded evals/azure.yaml and then printed a services block for the reader to paste into the root azure.yaml. azd acts on nothing until that reference exists, so the documented flow stopped between init and azd up, and every scenario run so far had passed only because the harness wrote that file itself before calling init. The reference is mechanical, so init writes it: creating the project file when the folder has none, and otherwise adding the service through the YAML node tree so the project keeps its name, comments, key order and other services. A project already declaring an azure.ai.eval service is left alone - matched on the host rather than the service name, so running init twice cannot deploy the same evals twice. Anything that cannot be edited safely still prints the block. The next steps now suggest azd deploy rather than azd up when the project has no infra to provision, since azd up stops at a missing infra/main.bicep without saying that provisioning is what it wanted. --- .../azure.ai.evaluations/internal/cmd/init.go | 184 +++++++++++++++++- .../internal/cmd/init_wiring_test.go | 126 ++++++++++++ 2 files changed, 301 insertions(+), 9 deletions(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_wiring_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go index 1f62f8be3d2..f4de5319b79 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go @@ -4,6 +4,7 @@ package cmd import ( + "errors" "fmt" "os" "path/filepath" @@ -69,27 +70,59 @@ func newInitCommand() *cobra.Command { return err } + // Scaffolding a config azd cannot see is half a step: the eval + // service has to be referenced from the root config before any of + // `azd up`, `azd deploy` or `azd ai eval run` will act on it. + // Printing the block and leaving the edit to the reader was enough + // to make the documented flow stop working between `init` and + // `azd up`. + rootWiring, err := ensureRootEvalService(rootConfigName, depPath) + if err != nil { + return err + } + if isJSON(cmd) { return emitJSON(out, map[string]any{ "generateConfig": genPath, "deployConfig": depPath, "datasetsDir": filepath.Join(outDir, project.DefaultDatasetsDir), "evaluatorsDir": filepath.Join(outDir, project.DefaultEvaluatorsDir), + "rootConfig": rootWiring, }) } fmt.Fprintf(out, "Wrote %s\n", genPath) fmt.Fprintf(out, "Wrote %s\n", depPath) + fmt.Fprintln(out, "\nNext:") - fmt.Fprintf(out, " 1. Reference %s from your root azure.yaml:\n", depPath) - fmt.Fprintln(out, " services:") - fmt.Fprintln(out, " evals:") - fmt.Fprintln(out, " host: azure.ai.eval") - fmt.Fprintln(out, " uses: [ai-project]") - fmt.Fprintf(out, " $ref: ./%s\n", filepath.ToSlash(depPath)) - fmt.Fprintln(out, " 2. azd ai eval generate (or supply your own dataset)") - fmt.Fprintln(out, " 3. azd up") - fmt.Fprintln(out, " 4. azd ai eval run") + step := 1 + if rootWiring == wiringManual { + // Only reached when the root config could not be read or has a + // shape this cannot safely edit, so the wiring is the caller's + // to do. + fmt.Fprintf(out, " %d. Reference %s from your root azure.yaml:\n", step, depPath) + fmt.Fprintln(out, " services:") + fmt.Fprintln(out, " evals:") + fmt.Fprintln(out, " host: azure.ai.eval") + fmt.Fprintln(out, " uses: [ai-project]") + fmt.Fprintf(out, " $ref: ./%s\n", filepath.ToSlash(depPath)) + step++ + } else { + fmt.Fprintf(out, " (%s references %s)\n", rootConfigName, depPath) + } + fmt.Fprintf(out, " %d. azd ai eval generate (or supply your own dataset)\n", step) + step++ + // azd up provisions before it deploys, which needs a bicep template. + // An eval-only project has none, and the failure names a missing + // infra/main.bicep rather than the reason, so it is only suggested + // where it can work. + if hasInfra() { + fmt.Fprintf(out, " %d. azd up\n", step) + } else { + fmt.Fprintf(out, " %d. azd deploy evals (azd up once the project has infra to provision)\n", step) + } + step++ + fmt.Fprintf(out, " %d. azd ai eval run\n", step) return nil }, } @@ -105,6 +138,139 @@ func newInitCommand() *cobra.Command { return cmd } +// rootConfigName is azd's project file, which the eval service is declared in. +const rootConfigName = "azure.yaml" + +// How the root config ended up referencing the eval service. +const ( + wiringCreated = "created" // there was no root config, so one was written + wiringAdded = "added" // the service was added to an existing config + wiringPresent = "present" // an eval service was already declared + wiringManual = "manual" // the caller has to do it; the block is printed +) + +// ensureRootEvalService declares the eval service in azd's project file. +// +// A config azd cannot see does nothing, and the reference is mechanical, so it +// is written rather than described. An existing project file is edited in place +// through the YAML node tree, which keeps its comments and key order; anything +// that cannot be edited safely falls back to printing the block. +func ensureRootEvalService(rootPath, depPath string) (string, error) { + ref := "./" + filepath.ToSlash(depPath) + + raw, err := os.ReadFile(rootPath) + if errors.Is(err, os.ErrNotExist) { + name := filepath.Base(mustAbs(filepath.Dir(rootPath))) + body := fmt.Sprintf(""+ + "name: %s\n"+ + "services:\n"+ + " evals:\n"+ + " host: azure.ai.eval\n"+ + " $ref: %s\n", name, ref) + if err := os.WriteFile(rootPath, []byte(body), 0o600); err != nil { + return "", fmt.Errorf("writing %s: %w", rootPath, err) + } + return wiringCreated, nil + } + if err != nil { + return wiringManual, nil + } + + var doc yaml.Node + if err := yaml.Unmarshal(raw, &doc); err != nil || len(doc.Content) == 0 { + return wiringManual, nil + } + root := doc.Content[0] + if root.Kind != yaml.MappingNode { + return wiringManual, nil + } + + services := mappingValue(root, "services") + if services != nil && services.Kind == yaml.MappingNode { + // A service already pointing at an eval config is left alone, whatever + // it is called: adding a second would deploy the same evals twice. + for i := 0; i+1 < len(services.Content); i += 2 { + if host := mappingValue(services.Content[i+1], "host"); host != nil && + host.Value == project.EvalHost { + return wiringPresent, nil + } + } + } + if services == nil { + root.Content = append(root.Content, + &yaml.Node{Kind: yaml.ScalarNode, Tag: "!!str", Value: "services"}, + &yaml.Node{Kind: yaml.MappingNode, Tag: "!!map"}) + services = root.Content[len(root.Content)-1] + } + if services.Kind != yaml.MappingNode { + return wiringManual, nil + } + + entry := &yaml.Node{Kind: yaml.MappingNode, Tag: "!!map"} + entry.Content = append(entry.Content, + &yaml.Node{Kind: yaml.ScalarNode, Tag: "!!str", Value: "host"}, + &yaml.Node{Kind: yaml.ScalarNode, Tag: "!!str", Value: project.EvalHost}, + &yaml.Node{Kind: yaml.ScalarNode, Tag: "!!str", Value: "$ref"}, + &yaml.Node{Kind: yaml.ScalarNode, Tag: "!!str", Value: ref}) + services.Content = append(services.Content, + &yaml.Node{Kind: yaml.ScalarNode, Tag: "!!str", Value: uniqueServiceName(services)}, + entry) + + out, err := yaml.Marshal(&doc) + if err != nil { + return wiringManual, nil + } + if err := os.WriteFile(rootPath, out, 0o600); err != nil { + return "", fmt.Errorf("updating %s: %w", rootPath, err) + } + return wiringAdded, nil +} + +// mappingValue returns the value node for key, or nil. +func mappingValue(m *yaml.Node, key string) *yaml.Node { + if m == nil || m.Kind != yaml.MappingNode { + return nil + } + for i := 0; i+1 < len(m.Content); i += 2 { + if m.Content[i].Value == key { + return m.Content[i+1] + } + } + return nil +} + +// uniqueServiceName avoids colliding with a service the project already has. +func uniqueServiceName(services *yaml.Node) string { + taken := map[string]bool{} + for i := 0; i+1 < len(services.Content); i += 2 { + taken[services.Content[i].Value] = true + } + if !taken["evals"] { + return "evals" + } + for i := 2; ; i++ { + candidate := fmt.Sprintf("evals%d", i) + if !taken[candidate] { + return candidate + } + } +} + +func mustAbs(p string) string { + abs, err := filepath.Abs(p) + if err != nil { + return p + } + return abs +} + +// hasInfra reports whether azd has a template to provision, which decides +// whether `azd up` can work here. +func hasInfra() bool { + _, err := os.Stat(filepath.Join("infra", "main.bicep")) + return err == nil +} + func buildGenerateScaffold(target, rubricName, evalModel string) *project.GenerateConfig { return &project.GenerateConfig{ Agent: project.AgentSpec{ diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_wiring_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_wiring_test.go new file mode 100644 index 00000000000..86288bdf578 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_wiring_test.go @@ -0,0 +1,126 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// init scaffolds a config that azd only acts on once the root project file +// references it. Printing the block and leaving the edit to the reader is what +// made the documented flow stop between `init` and `azd up`. +func TestEnsureRootEvalService_CreatesTheProjectFileWhenAbsent(t *testing.T) { + dir := t.TempDir() + t.Chdir(dir) + + action, err := ensureRootEvalService(rootConfigName, filepath.Join("evals", "azure.yaml")) + require.NoError(t, err) + assert.Equal(t, wiringCreated, action) + + body, err := os.ReadFile(rootConfigName) + require.NoError(t, err) + assert.Contains(t, string(body), "host: azure.ai.eval") + assert.Contains(t, string(body), "$ref: ./evals/azure.yaml") + assert.Contains(t, string(body), "name: "+filepath.Base(dir), + "azd needs a project name, taken from the directory") +} + +// An existing project file belongs to the caller, so the service is added +// without disturbing what is already declared. +func TestEnsureRootEvalService_AddsToAnExistingProject(t *testing.T) { + dir := t.TempDir() + t.Chdir(dir) + existing := "name: my-app\n" + + "services:\n" + + " api:\n" + + " host: containerapp\n" + + " language: python\n" + require.NoError(t, os.WriteFile(rootConfigName, []byte(existing), 0o600)) + + action, err := ensureRootEvalService(rootConfigName, filepath.Join("evals", "azure.yaml")) + require.NoError(t, err) + assert.Equal(t, wiringAdded, action) + + body, err := os.ReadFile(rootConfigName) + require.NoError(t, err) + assert.Contains(t, string(body), "host: azure.ai.eval") + assert.Contains(t, string(body), "host: containerapp", "the existing service survives") + assert.Contains(t, string(body), "language: python") + assert.Contains(t, string(body), "name: my-app", "the project keeps its name") +} + +// Running init twice must not declare the evals twice, which would deploy them +// twice. The name is not what identifies it — the host is. +func TestEnsureRootEvalService_LeavesAnExistingEvalServiceAlone(t *testing.T) { + dir := t.TempDir() + t.Chdir(dir) + existing := "name: my-app\n" + + "services:\n" + + " quality:\n" + + " host: azure.ai.eval\n" + + " $ref: ./evals/azure.yaml\n" + require.NoError(t, os.WriteFile(rootConfigName, []byte(existing), 0o600)) + + action, err := ensureRootEvalService(rootConfigName, filepath.Join("evals", "azure.yaml")) + require.NoError(t, err) + assert.Equal(t, wiringPresent, action) + + body, err := os.ReadFile(rootConfigName) + require.NoError(t, err) + assert.Equal(t, existing, string(body), "an already-wired project is untouched") +} + +// A project declaring no services at all still needs the key adding. +func TestEnsureRootEvalService_AddsTheServicesKeyWhenMissing(t *testing.T) { + dir := t.TempDir() + t.Chdir(dir) + require.NoError(t, os.WriteFile(rootConfigName, []byte("name: my-app\n"), 0o600)) + + action, err := ensureRootEvalService(rootConfigName, filepath.Join("evals", "azure.yaml")) + require.NoError(t, err) + assert.Equal(t, wiringAdded, action) + + body, err := os.ReadFile(rootConfigName) + require.NoError(t, err) + assert.Contains(t, string(body), "services:") + assert.Contains(t, string(body), "host: azure.ai.eval") +} + +// A file that is not a YAML mapping is someone else's to fix; the block is +// printed instead of guessing at an edit. +func TestEnsureRootEvalService_FallsBackWhenTheProjectFileIsNotAMapping(t *testing.T) { + dir := t.TempDir() + t.Chdir(dir) + require.NoError(t, os.WriteFile(rootConfigName, []byte("- not\n- a mapping\n"), 0o600)) + + action, err := ensureRootEvalService(rootConfigName, filepath.Join("evals", "azure.yaml")) + require.NoError(t, err) + assert.Equal(t, wiringManual, action) +} + +// A service called "evals" already existing for something else must not be +// overwritten. +func TestEnsureRootEvalService_DoesNotClobberAnUnrelatedEvalsService(t *testing.T) { + dir := t.TempDir() + t.Chdir(dir) + existing := "name: my-app\n" + + "services:\n" + + " evals:\n" + + " host: containerapp\n" + require.NoError(t, os.WriteFile(rootConfigName, []byte(existing), 0o600)) + + action, err := ensureRootEvalService(rootConfigName, filepath.Join("evals", "azure.yaml")) + require.NoError(t, err) + assert.Equal(t, wiringAdded, action) + + body, err := os.ReadFile(rootConfigName) + require.NoError(t, err) + assert.Contains(t, string(body), "host: containerapp", "the unrelated service is intact") + assert.Contains(t, string(body), "evals2:", "the eval service takes a free name") +} From 2713a8e3e74171ee4202f930af1e51d4fb8dda1d Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 28 Jul 2026 20:01:12 +0300 Subject: [PATCH 056/320] Add the eval service through azd, and catch two things the corner cases found init hand-edited the root azure.yaml through the YAML node tree. No other extension does that - the agents extension adds its service with azd's own Project().AddService, and the only azure.yaml any extension writes is a staged copy in a temp directory. So the eval service goes through AddService too, and azd owns the edit. A folder with no project is no longer given one: evals attach to a project, so init says to run azd init. That also removes the azd up divergence. init suggested azd deploy when the project had no infra, which quietly moved people off the flow the spec describes; it says azd up again, and a project without infra fails in provisioning where the reason belongs. Two defects the corner cases turned up: A malformed dataset row was published without complaint. {not json at all} became version 1.0 with an eval group bound to it, and would have failed much later on a row nobody had looked at. Every row is now checked before upload and the offending line is named. The scan buffer is 8MB because a conversation-level row runs well past bufio's 64KB default. Asking for runs of an eval group that does not exist printed 1682 characters of HTTP error. A mistyped or undeployed id is ordinary, so it now says so in one line. --- .../azure.ai.evaluations/internal/cmd/init.go | 191 ++++++------------ .../internal/cmd/init_wiring_test.go | 126 ++---------- .../internal/cmd/jsonl_validation_test.go | 75 +++++++ .../internal/cmd/reconciler.go | 53 +++++ .../internal/cmd/results.go | 5 + .../internal/cmd/run_ops.go | 5 + 6 files changed, 214 insertions(+), 241 deletions(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/jsonl_validation_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go index f4de5319b79..be2d29941e2 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go @@ -4,7 +4,7 @@ package cmd import ( - "errors" + "context" "fmt" "os" "path/filepath" @@ -13,8 +13,10 @@ import ( "azureaieval/internal/pkg/evalcore" "azureaieval/internal/project" + "github.com/azure/azure-dev/cli/azd/pkg/azdext" "github.com/spf13/cobra" "go.yaml.in/yaml/v3" + "google.golang.org/protobuf/types/known/structpb" ) // newInitCommand scaffolds the eval configuration. It makes no service calls at @@ -76,7 +78,7 @@ func newInitCommand() *cobra.Command { // Printing the block and leaving the edit to the reader was enough // to make the documented flow stop working between `init` and // `azd up`. - rootWiring, err := ensureRootEvalService(rootConfigName, depPath) + rootWiring, err := ensureRootEvalService(cmd.Context(), depPath) if err != nil { return err } @@ -93,36 +95,17 @@ func newInitCommand() *cobra.Command { fmt.Fprintf(out, "Wrote %s\n", genPath) fmt.Fprintf(out, "Wrote %s\n", depPath) + switch rootWiring { + case wiringAdded: + fmt.Fprintf(out, "Added the evals service to %s\n", rootConfigName) + case wiringPresent: + fmt.Fprintf(out, "%s already declares an eval service\n", rootConfigName) + } fmt.Fprintln(out, "\nNext:") - step := 1 - if rootWiring == wiringManual { - // Only reached when the root config could not be read or has a - // shape this cannot safely edit, so the wiring is the caller's - // to do. - fmt.Fprintf(out, " %d. Reference %s from your root azure.yaml:\n", step, depPath) - fmt.Fprintln(out, " services:") - fmt.Fprintln(out, " evals:") - fmt.Fprintln(out, " host: azure.ai.eval") - fmt.Fprintln(out, " uses: [ai-project]") - fmt.Fprintf(out, " $ref: ./%s\n", filepath.ToSlash(depPath)) - step++ - } else { - fmt.Fprintf(out, " (%s references %s)\n", rootConfigName, depPath) - } - fmt.Fprintf(out, " %d. azd ai eval generate (or supply your own dataset)\n", step) - step++ - // azd up provisions before it deploys, which needs a bicep template. - // An eval-only project has none, and the failure names a missing - // infra/main.bicep rather than the reason, so it is only suggested - // where it can work. - if hasInfra() { - fmt.Fprintf(out, " %d. azd up\n", step) - } else { - fmt.Fprintf(out, " %d. azd deploy evals (azd up once the project has infra to provision)\n", step) - } - step++ - fmt.Fprintf(out, " %d. azd ai eval run\n", step) + fmt.Fprintln(out, " 1. azd ai eval generate (or supply your own dataset)") + fmt.Fprintln(out, " 2. azd up") + fmt.Fprintln(out, " 3. azd ai eval run") return nil }, } @@ -143,134 +126,80 @@ const rootConfigName = "azure.yaml" // How the root config ended up referencing the eval service. const ( - wiringCreated = "created" // there was no root config, so one was written - wiringAdded = "added" // the service was added to an existing config + wiringAdded = "added" // the service was added to the project wiringPresent = "present" // an eval service was already declared - wiringManual = "manual" // the caller has to do it; the block is printed ) // ensureRootEvalService declares the eval service in azd's project file. // -// A config azd cannot see does nothing, and the reference is mechanical, so it -// is written rather than described. An existing project file is edited in place -// through the YAML node tree, which keeps its comments and key order; anything -// that cannot be edited safely falls back to printing the block. -func ensureRootEvalService(rootPath, depPath string) (string, error) { - ref := "./" + filepath.ToSlash(depPath) - - raw, err := os.ReadFile(rootPath) - if errors.Is(err, os.ErrNotExist) { - name := filepath.Base(mustAbs(filepath.Dir(rootPath))) - body := fmt.Sprintf(""+ - "name: %s\n"+ - "services:\n"+ - " evals:\n"+ - " host: azure.ai.eval\n"+ - " $ref: %s\n", name, ref) - if err := os.WriteFile(rootPath, []byte(body), 0o600); err != nil { - return "", fmt.Errorf("writing %s: %w", rootPath, err) - } - return wiringCreated, nil - } +// azd acts on nothing until the service exists, so the reference is made rather +// than described. It goes through azd's own Project().AddService, the same call +// the agents extension uses, so azd owns the edit and the project file keeps +// whatever shape azd gives it. +// +// The eval config itself stays in evals/azure.yaml and is referenced with +// `$ref`. azd carries unknown keys through AdditionalProperties untouched, +// which is how the extension gets it back at deploy time. +func ensureRootEvalService(ctx context.Context, depPath string) (string, error) { + azdClient, err := azdext.NewAzdClient() if err != nil { - return wiringManual, nil - } - - var doc yaml.Node - if err := yaml.Unmarshal(raw, &doc); err != nil || len(doc.Content) == 0 { - return wiringManual, nil + return "", fmt.Errorf("connecting to azd: %w", err) } - root := doc.Content[0] - if root.Kind != yaml.MappingNode { - return wiringManual, nil + defer azdClient.Close() + + resp, err := azdClient.Project().Get(ctx, &azdext.EmptyRequest{}) + if err != nil || resp.GetProject() == nil { + // Evals attach to a project; they do not create one. Saying which + // command makes one is more use than a gRPC error. + return "", fmt.Errorf( + "no azd project found in this directory. Run `azd init` first, "+ + "or run this from the root of an existing one; the eval service is "+ + "added to its %s", rootConfigName) } - services := mappingValue(root, "services") - if services != nil && services.Kind == yaml.MappingNode { - // A service already pointing at an eval config is left alone, whatever - // it is called: adding a second would deploy the same evals twice. - for i := 0; i+1 < len(services.Content); i += 2 { - if host := mappingValue(services.Content[i+1], "host"); host != nil && - host.Value == project.EvalHost { - return wiringPresent, nil - } + // A service already pointing at an eval config is left alone, whatever it + // is called: a second one would deploy the same evals twice. + for _, svc := range resp.GetProject().GetServices() { + if svc.GetHost() == project.EvalHost { + return wiringPresent, nil } } - if services == nil { - root.Content = append(root.Content, - &yaml.Node{Kind: yaml.ScalarNode, Tag: "!!str", Value: "services"}, - &yaml.Node{Kind: yaml.MappingNode, Tag: "!!map"}) - services = root.Content[len(root.Content)-1] - } - if services.Kind != yaml.MappingNode { - return wiringManual, nil - } - entry := &yaml.Node{Kind: yaml.MappingNode, Tag: "!!map"} - entry.Content = append(entry.Content, - &yaml.Node{Kind: yaml.ScalarNode, Tag: "!!str", Value: "host"}, - &yaml.Node{Kind: yaml.ScalarNode, Tag: "!!str", Value: project.EvalHost}, - &yaml.Node{Kind: yaml.ScalarNode, Tag: "!!str", Value: "$ref"}, - &yaml.Node{Kind: yaml.ScalarNode, Tag: "!!str", Value: ref}) - services.Content = append(services.Content, - &yaml.Node{Kind: yaml.ScalarNode, Tag: "!!str", Value: uniqueServiceName(services)}, - entry) - - out, err := yaml.Marshal(&doc) + props, err := structpb.NewStruct(map[string]any{ + "$ref": "./" + filepath.ToSlash(depPath), + }) if err != nil { - return wiringManual, nil + return "", fmt.Errorf("building the eval service entry: %w", err) } - if err := os.WriteFile(rootPath, out, 0o600); err != nil { - return "", fmt.Errorf("updating %s: %w", rootPath, err) - } - return wiringAdded, nil -} -// mappingValue returns the value node for key, or nil. -func mappingValue(m *yaml.Node, key string) *yaml.Node { - if m == nil || m.Kind != yaml.MappingNode { - return nil - } - for i := 0; i+1 < len(m.Content); i += 2 { - if m.Content[i].Value == key { - return m.Content[i+1] - } + _, err = azdClient.Project().AddService(ctx, &azdext.AddServiceRequest{ + Service: &azdext.ServiceConfig{ + Name: evalServiceName(resp.GetProject()), + Host: project.EvalHost, + AdditionalProperties: props, + }, + }) + if err != nil { + return "", fmt.Errorf("adding the eval service to %s: %w", rootConfigName, err) } - return nil + return wiringAdded, nil } -// uniqueServiceName avoids colliding with a service the project already has. -func uniqueServiceName(services *yaml.Node) string { - taken := map[string]bool{} - for i := 0; i+1 < len(services.Content); i += 2 { - taken[services.Content[i].Value] = true - } - if !taken["evals"] { +// evalServiceName avoids colliding with a service the project already has. +// azd keys services by name, so the map key is the name to avoid. +func evalServiceName(proj *azdext.ProjectConfig) string { + taken := proj.GetServices() + if _, exists := taken["evals"]; !exists { return "evals" } for i := 2; ; i++ { candidate := fmt.Sprintf("evals%d", i) - if !taken[candidate] { + if _, exists := taken[candidate]; !exists { return candidate } } } -func mustAbs(p string) string { - abs, err := filepath.Abs(p) - if err != nil { - return p - } - return abs -} - -// hasInfra reports whether azd has a template to provision, which decides -// whether `azd up` can work here. -func hasInfra() bool { - _, err := os.Stat(filepath.Join("infra", "main.bicep")) - return err == nil -} - func buildGenerateScaffold(target, rubricName, evalModel string) *project.GenerateConfig { return &project.GenerateConfig{ Agent: project.AgentSpec{ diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_wiring_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_wiring_test.go index 86288bdf578..c5371c23db2 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_wiring_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_wiring_test.go @@ -4,123 +4,29 @@ package cmd import ( - "os" - "path/filepath" "testing" + "github.com/azure/azure-dev/cli/azd/pkg/azdext" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" ) -// init scaffolds a config that azd only acts on once the root project file -// references it. Printing the block and leaving the edit to the reader is what -// made the documented flow stop between `init` and `azd up`. -func TestEnsureRootEvalService_CreatesTheProjectFileWhenAbsent(t *testing.T) { - dir := t.TempDir() - t.Chdir(dir) - - action, err := ensureRootEvalService(rootConfigName, filepath.Join("evals", "azure.yaml")) - require.NoError(t, err) - assert.Equal(t, wiringCreated, action) - - body, err := os.ReadFile(rootConfigName) - require.NoError(t, err) - assert.Contains(t, string(body), "host: azure.ai.eval") - assert.Contains(t, string(body), "$ref: ./evals/azure.yaml") - assert.Contains(t, string(body), "name: "+filepath.Base(dir), - "azd needs a project name, taken from the directory") +func projectWith(names ...string) *azdext.ProjectConfig { + proj := &azdext.ProjectConfig{Services: map[string]*azdext.ServiceConfig{}} + for _, n := range names { + proj.Services[n] = &azdext.ServiceConfig{Name: n} + } + return proj } -// An existing project file belongs to the caller, so the service is added -// without disturbing what is already declared. -func TestEnsureRootEvalService_AddsToAnExistingProject(t *testing.T) { - dir := t.TempDir() - t.Chdir(dir) - existing := "name: my-app\n" + - "services:\n" + - " api:\n" + - " host: containerapp\n" + - " language: python\n" - require.NoError(t, os.WriteFile(rootConfigName, []byte(existing), 0o600)) - - action, err := ensureRootEvalService(rootConfigName, filepath.Join("evals", "azure.yaml")) - require.NoError(t, err) - assert.Equal(t, wiringAdded, action) - - body, err := os.ReadFile(rootConfigName) - require.NoError(t, err) - assert.Contains(t, string(body), "host: azure.ai.eval") - assert.Contains(t, string(body), "host: containerapp", "the existing service survives") - assert.Contains(t, string(body), "language: python") - assert.Contains(t, string(body), "name: my-app", "the project keeps its name") +// The eval service is added through azd's own AddService, so it has to pick a +// name azd will accept. Reusing one already in the project would overwrite +// somebody else's service. +func TestEvalServiceName_PrefersEvals(t *testing.T) { + assert.Equal(t, "evals", evalServiceName(projectWith())) + assert.Equal(t, "evals", evalServiceName(projectWith("api", "web"))) } -// Running init twice must not declare the evals twice, which would deploy them -// twice. The name is not what identifies it — the host is. -func TestEnsureRootEvalService_LeavesAnExistingEvalServiceAlone(t *testing.T) { - dir := t.TempDir() - t.Chdir(dir) - existing := "name: my-app\n" + - "services:\n" + - " quality:\n" + - " host: azure.ai.eval\n" + - " $ref: ./evals/azure.yaml\n" - require.NoError(t, os.WriteFile(rootConfigName, []byte(existing), 0o600)) - - action, err := ensureRootEvalService(rootConfigName, filepath.Join("evals", "azure.yaml")) - require.NoError(t, err) - assert.Equal(t, wiringPresent, action) - - body, err := os.ReadFile(rootConfigName) - require.NoError(t, err) - assert.Equal(t, existing, string(body), "an already-wired project is untouched") -} - -// A project declaring no services at all still needs the key adding. -func TestEnsureRootEvalService_AddsTheServicesKeyWhenMissing(t *testing.T) { - dir := t.TempDir() - t.Chdir(dir) - require.NoError(t, os.WriteFile(rootConfigName, []byte("name: my-app\n"), 0o600)) - - action, err := ensureRootEvalService(rootConfigName, filepath.Join("evals", "azure.yaml")) - require.NoError(t, err) - assert.Equal(t, wiringAdded, action) - - body, err := os.ReadFile(rootConfigName) - require.NoError(t, err) - assert.Contains(t, string(body), "services:") - assert.Contains(t, string(body), "host: azure.ai.eval") -} - -// A file that is not a YAML mapping is someone else's to fix; the block is -// printed instead of guessing at an edit. -func TestEnsureRootEvalService_FallsBackWhenTheProjectFileIsNotAMapping(t *testing.T) { - dir := t.TempDir() - t.Chdir(dir) - require.NoError(t, os.WriteFile(rootConfigName, []byte("- not\n- a mapping\n"), 0o600)) - - action, err := ensureRootEvalService(rootConfigName, filepath.Join("evals", "azure.yaml")) - require.NoError(t, err) - assert.Equal(t, wiringManual, action) -} - -// A service called "evals" already existing for something else must not be -// overwritten. -func TestEnsureRootEvalService_DoesNotClobberAnUnrelatedEvalsService(t *testing.T) { - dir := t.TempDir() - t.Chdir(dir) - existing := "name: my-app\n" + - "services:\n" + - " evals:\n" + - " host: containerapp\n" - require.NoError(t, os.WriteFile(rootConfigName, []byte(existing), 0o600)) - - action, err := ensureRootEvalService(rootConfigName, filepath.Join("evals", "azure.yaml")) - require.NoError(t, err) - assert.Equal(t, wiringAdded, action) - - body, err := os.ReadFile(rootConfigName) - require.NoError(t, err) - assert.Contains(t, string(body), "host: containerapp", "the unrelated service is intact") - assert.Contains(t, string(body), "evals2:", "the eval service takes a free name") +func TestEvalServiceName_StepsAsideForAnExistingName(t *testing.T) { + assert.Equal(t, "evals2", evalServiceName(projectWith("evals"))) + assert.Equal(t, "evals3", evalServiceName(projectWith("evals", "evals2"))) } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/jsonl_validation_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/jsonl_validation_test.go new file mode 100644 index 00000000000..e614af18378 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/jsonl_validation_test.go @@ -0,0 +1,75 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func writeJSONL(t *testing.T, body string) string { + t.Helper() + path := filepath.Join(t.TempDir(), "d.jsonl") + require.NoError(t, os.WriteFile(path, []byte(body), 0o600)) + return path +} + +// The service accepts whatever bytes it is given, so a malformed row becomes a +// published version with an eval group bound to it, and only fails much later +// on a row nobody has looked at. A live deploy published `{not json at all}` +// as version 1.0 before this existed. +func TestValidateJSONL_RejectsAMalformedRowByLine(t *testing.T) { + err := validateJSONL(writeJSONL(t, "{\"query\":\"fine\"}\n{not json at all}\n")) + require.Error(t, err) + assert.Contains(t, err.Error(), "line 2") + assert.Contains(t, err.Error(), "one JSON object") +} + +func TestValidateJSONL_AcceptsWellFormedRows(t *testing.T) { + assert.NoError(t, validateJSONL(writeJSONL(t, + "{\"query\":\"a\"}\n{\"query\":\"b\"}\n"))) +} + +// Trailing and interior blank lines are formatting, not rows. +func TestValidateJSONL_IgnoresBlankLines(t *testing.T) { + assert.NoError(t, validateJSONL(writeJSONL(t, + "{\"query\":\"a\"}\n\n{\"query\":\"b\"}\n\n"))) +} + +// A file with nothing in it publishes a version that can never score anything. +func TestValidateJSONL_RejectsAFileWithNoRows(t *testing.T) { + err := validateJSONL(writeJSONL(t, "\n\n")) + require.Error(t, err) + assert.Contains(t, err.Error(), "no rows") +} + +// A JSON array is the shape people reach for when they mean JSONL. +func TestValidateJSONL_RejectsAJSONArray(t *testing.T) { + err := validateJSONL(writeJSONL(t, "[{\"query\":\"a\"},{\"query\":\"b\"}]\n")) + require.Error(t, err) + assert.Contains(t, err.Error(), "line 1") +} + +// An empty object parses but evaluates to nothing. +func TestValidateJSONL_RejectsAnEmptyObject(t *testing.T) { + err := validateJSONL(writeJSONL(t, "{\"query\":\"a\"}\n{}\n")) + require.Error(t, err) + assert.Contains(t, err.Error(), "line 2") + assert.Contains(t, err.Error(), "empty object") +} + +// A conversation-level row holds a whole transcript and runs past bufio's +// default 64KB line limit, which would otherwise be reported as invalid JSON. +func TestValidateJSONL_AcceptsAVeryLongRow(t *testing.T) { + long := make([]byte, 200*1024) + for i := range long { + long[i] = 'x' + } + assert.NoError(t, validateJSONL(writeJSONL(t, + "{\"query\":\""+string(long)+"\"}\n"))) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go index 4b2c52401c8..a9d6737924e 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go @@ -4,12 +4,14 @@ package cmd import ( + "bufio" "context" "encoding/json" "fmt" "os" "path/filepath" "reflect" + "strings" "azureaieval/internal/pkg/dataset_api" "azureaieval/internal/project" @@ -66,6 +68,13 @@ func (r *evalReconciler) EnsureDataset( return "", false, fmt.Errorf("dataset source %q: %w", localPath, err) } + // A malformed row is only noticed once the service tries to evaluate it, + // by which point a version has been published and the eval group points at + // it. Reading the file here costs nothing and names the offending line. + if err := validateJSONL(localPath); err != nil { + return "", false, fmt.Errorf("dataset %q: %w", decl.Name, err) + } + digest, err := project.Fingerprint(localPath) if err != nil { return "", false, err @@ -140,6 +149,50 @@ func (r *evalReconciler) EnsureDataset( // pinning the eval group to the older version would quietly evaluate against // stale data. Publishing is not destructive — versions are immutable — so the // remedy is to sync, not to overwrite. +// validateJSONL checks that every row is a JSON object before the file is +// published. +// +// The service accepts the upload whatever the bytes are, so a typo becomes a +// registered version, an eval group bound to it, and a run that fails on a row +// nobody has looked at. Blank lines are skipped: they are not rows. +func validateJSONL(path string) error { + f, err := os.Open(path) + if err != nil { + return fmt.Errorf("reading %s: %w", path, err) + } + defer f.Close() + + scanner := bufio.NewScanner(f) + // A row carrying a whole conversation runs well past the 64KB default. + scanner.Buffer(make([]byte, 0, 64*1024), 8*1024*1024) + + rows := 0 + for line := 1; scanner.Scan(); line++ { + text := strings.TrimSpace(scanner.Text()) + if text == "" { + continue + } + var row map[string]any + if err := json.Unmarshal([]byte(text), &row); err != nil { + return fmt.Errorf( + "%s line %d is not valid JSON: %w. Every line must be one JSON object", + path, line, err) + } + if len(row) == 0 { + return fmt.Errorf( + "%s line %d is an empty object, which evaluates to nothing", path, line) + } + rows++ + } + if err := scanner.Err(); err != nil { + return fmt.Errorf("reading %s: %w", path, err) + } + if rows == 0 { + return fmt.Errorf("%s has no rows to evaluate", path) + } + return nil +} + func (r *evalReconciler) checkDatasetDrift( ctx context.Context, name, recorded string, diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/results.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/results.go index 1fc0ee4157b..0cc74767f1f 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/results.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/results.go @@ -222,6 +222,11 @@ func (ec *evalContext) latestOrNamedRun( list, err := ec.evalClient.ListOpenAIEvalRuns(ctx, evalID, 1) if err != nil { + if eval_api.IsNotFound(err) { + return nil, fmt.Errorf( + "no eval group %q in this project; "+ + "`azd up` creates the ones your config declares", evalID) + } return nil, fmt.Errorf("listing runs for eval group %s: %w", evalID, err) } if len(list.Data) == 0 { diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go index 1ff2ee7cebb..e815d4866bd 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go @@ -49,6 +49,11 @@ func newRunListCommand() *cobra.Command { list, err := ec.evalClient.ListOpenAIEvalRuns(ctx, evalID, 0) if err != nil { + if eval_api.IsNotFound(err) { + return fmt.Errorf( + "no eval group %q in this project; "+ + "`azd up` creates the ones your config declares", evalID) + } return fmt.Errorf("listing runs for %q: %w", evalID, err) } if isJSON(cmd) { From f3a8ace964e5e8966228157233a559aded355512 Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 28 Jul 2026 20:52:52 +0300 Subject: [PATCH 057/320] Order the eval service after the Foundry project when the repo declares one The service entry named no dependencies, so azd had nothing to order it against. The agents extension wires uses: to the ai-project service it created, and the eval service wants the same: it evaluates against that project, so the project should be provisioned first. It is conditional for the reason agents makes it conditional - naming a service the project does not declare is a broken reference, and an eval config can sit in a repo that reaches an existing Foundry project by endpoint rather than declaring one. --- .../azure.ai.evaluations/internal/cmd/init.go | 21 +++++++++++++++++++ .../internal/cmd/init_wiring_test.go | 16 ++++++++++++++ 2 files changed, 37 insertions(+) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go index be2d29941e2..b7804ae91a6 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go @@ -124,6 +124,10 @@ func newInitCommand() *cobra.Command { // rootConfigName is azd's project file, which the eval service is declared in. const rootConfigName = "azure.yaml" +// aiProjectHost is the Foundry project service other extensions declare. The +// eval service uses it for ordering when the repo has one. +const aiProjectHost = "azure.ai.project" + // How the root config ended up referencing the eval service. const ( wiringAdded = "added" // the service was added to the project @@ -176,6 +180,7 @@ func ensureRootEvalService(ctx context.Context, depPath string) (string, error) Service: &azdext.ServiceConfig{ Name: evalServiceName(resp.GetProject()), Host: project.EvalHost, + Uses: projectServiceUses(resp.GetProject()), AdditionalProperties: props, }, }) @@ -185,6 +190,22 @@ func ensureRootEvalService(ctx context.Context, depPath string) (string, error) return wiringAdded, nil } +// projectServiceUses points the eval service at the Foundry project service +// when the repo declares one, so azd provisions it first. +// +// It is conditional for the same reason the agents extension makes it +// conditional: naming a service the project does not declare is a broken +// reference, and an eval config can perfectly well sit in a repo that reaches +// an existing Foundry project by endpoint instead. +func projectServiceUses(proj *azdext.ProjectConfig) []string { + for name, svc := range proj.GetServices() { + if svc.GetHost() == aiProjectHost { + return []string{name} + } + } + return nil +} + // evalServiceName avoids colliding with a service the project already has. // azd keys services by name, so the map key is the name to avoid. func evalServiceName(proj *azdext.ProjectConfig) string { diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_wiring_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_wiring_test.go index c5371c23db2..18b37e8e969 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_wiring_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_wiring_test.go @@ -30,3 +30,19 @@ func TestEvalServiceName_StepsAsideForAnExistingName(t *testing.T) { assert.Equal(t, "evals2", evalServiceName(projectWith("evals"))) assert.Equal(t, "evals3", evalServiceName(projectWith("evals", "evals2"))) } + +// The agents extension wires uses: only to services the project actually +// declares. Naming one it does not have is a broken reference, and an eval +// config can sit in a repo that reaches an existing Foundry project by +// endpoint instead of declaring one. +func TestProjectServiceUses_OnlyWhenTheProjectDeclaresOne(t *testing.T) { + assert.Nil(t, projectServiceUses(projectWith("api", "web")), + "no Foundry project service means no uses entry") + + withProject := projectWith("api") + withProject.Services["ai-project"] = &azdext.ServiceConfig{ + Name: "ai-project", Host: aiProjectHost, + } + assert.Equal(t, []string{"ai-project"}, projectServiceUses(withProject), + "the eval service should be ordered after the project it evaluates against") +} From cfe114072c21bbf6b07df3a211a4f304b14e3ad8 Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 28 Jul 2026 22:14:15 +0300 Subject: [PATCH 058/320] Address review: drop "eval group", split the model flags, show output items Renames, all user-facing, from the spec review: - "eval group" is not a service concept. The object is an eval, so evalGroups: becomes evals:, --eval-group becomes --eval, EVAL_GROUP_ID becomes EVAL_ID, and the prose follows. Breaking, and cheapest now. - --eval-model meant the judge in run and the generator in generate, which is two different things wearing one name. Split into --judge-model and --generation-model. - --gen-instruction is the agent's instruction, so --agent-instruction. - evaluator upload becomes evaluator create, matching dataset create. update is gone: both published a new immutable version, so it was a synonym. - evaluator builtins folds into evaluator list --builtin. results show only ever fetched the run, which carries totals and a per-criterion breakdown - so it could say how many rows failed but never which or why, and --failed-only filtered criteria rather than rows. It now reads the run's output_items, which carry the evaluated row, each evaluator's score and verdict, and the judge's reason. --failed-only filters rows, and -o json carries the whole thing. run delete is added; the API supported it all along. --- .../internal/cmd/apiversions.go | 2 +- .../internal/cmd/build.go | 8 +- .../internal/cmd/build_live_test.go | 10 +- .../internal/cmd/build_test.go | 30 ++-- .../internal/cmd/compare.go | 12 +- .../internal/cmd/context.go | 4 +- .../internal/cmd/description_test.go | 4 +- .../internal/cmd/envkeys_test.go | 18 +- .../internal/cmd/evaluator.go | 62 +++---- .../internal/cmd/generate.go | 16 +- .../azure.ai.evaluations/internal/cmd/init.go | 6 +- .../internal/cmd/init_test.go | 6 +- .../internal/cmd/instruction_test.go | 2 +- .../internal/cmd/jsonl_validation_test.go | 2 +- .../internal/cmd/model_target_test.go | 6 +- .../internal/cmd/reconciler.go | 26 +-- .../internal/cmd/resolution_test.go | 10 +- .../internal/cmd/results.go | 156 +++++++++++++----- .../azure.ai.evaluations/internal/cmd/run.go | 76 ++++----- .../internal/cmd/run_ops.go | 68 +++++++- .../internal/cmd/run_ops_test.go | 8 +- .../internal/cmd/schedule.go | 18 +- .../internal/pkg/eval_api/evaluators.go | 34 ++++ .../internal/pkg/eval_api/models.go | 66 ++++++++ .../internal/pkg/evalcore/evaluator.go | 2 +- .../internal/project/eval_config.go | 54 +++--- .../internal/project/eval_config_test.go | 20 +-- .../internal/project/merge_test.go | 8 +- .../internal/project/service_target_eval.go | 28 ++-- .../project/service_target_eval_test.go | 22 +-- 30 files changed, 494 insertions(+), 290 deletions(-) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/apiversions.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/apiversions.go index 89b1d125f34..0c3ddb78734 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/apiversions.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/apiversions.go @@ -12,6 +12,6 @@ const ( // DataGenerationAPIVersion covers dataset generation jobs. DataGenerationAPIVersion = "v1" - // OpenAI-compatible eval group and run calls send no api-version, so there + // OpenAI-compatible eval and run calls send no api-version, so there // is deliberately no constant for them. ) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build.go index c92ff0bccf3..6ca6d2fcf61 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build.go @@ -246,7 +246,7 @@ func planCriterion( } if len(missingInit) > 0 { return nil, fmt.Errorf( - "evaluator %q requires %s; set the judge model on the eval group", + "evaluator %q requires %s; set the judge model on the eval", ref.Name, quoteList(missingInit), ) } @@ -255,7 +255,7 @@ func planCriterion( return plan, nil } -// buildEvalGroupRequest converts an eval group declaration into the create +// buildEvalRequest converts an eval declaration into the create // request. Each evaluator becomes a testing criterion bound to its own // contract, and the item schema declares every dataset column those bindings // reference. @@ -263,8 +263,8 @@ func planCriterion( // schemas may be nil or partial; an evaluator with no published contract falls // back to the agent-target shape. datasetColumns may be nil, meaning the // columns are unknown and every accepted field is assumed present. -func buildEvalGroupRequest( - group *project.EvalGroup, +func buildEvalRequest( + group *project.Eval, schemas map[string]*eval_api.EvaluatorSummary, datasetColumns map[string]bool, ) (*eval_api.CreateOpenAIEvalRequest, error) { diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build_live_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build_live_test.go index 124b56c0249..b7c9c32500c 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build_live_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build_live_test.go @@ -3,7 +3,7 @@ //go:build live -// This file proves the request buildEvalGroupRequest produces is accepted by +// This file proves the request buildEvalRequest produces is accepted by // the real service. It lives in the cmd package on purpose: the tests under // tests/live can only hand-roll a request, which validates the API but not the // code that ships. @@ -93,7 +93,7 @@ func TestLiveBuildAcceptedForEveryBuiltin(t *testing.T) { level = summary.SupportedEvaluationLevels[0] } - group := &project.EvalGroup{ + group := &project.Eval{ Name: fmt.Sprintf("azd-live-%d", time.Now().UTC().UnixNano()), Dataset: "inline", Target: &project.Target{Type: "agent", Name: "probe-agent"}, @@ -101,7 +101,7 @@ func TestLiveBuildAcceptedForEveryBuiltin(t *testing.T) { Options: &project.Options{EvalModel: judge, EvaluationLevel: level}, } - req, err := buildEvalGroupRequest(group, schemas, columns) + req, err := buildEvalRequest(group, schemas, columns) require.NoError(t, err, "the builder must satisfy every published contract") created, err := client.CreateOpenAIEval(ctx, req) @@ -135,7 +135,7 @@ func TestLiveBuildRejectsMissingColumnsLocally(t *testing.T) { require.NotEmpty(t, target.DataSchema().Required, "this test relies on ifeval declaring required inputs") - group := &project.EvalGroup{ + group := &project.Eval{ Name: "azd-live-negative", Dataset: "inline", Target: &project.Target{Type: "agent", Name: "probe-agent"}, @@ -144,7 +144,7 @@ func TestLiveBuildRejectsMissingColumnsLocally(t *testing.T) { } // A dataset with only `query` cannot satisfy ifeval. - _, err = buildEvalGroupRequest(group, schemas, map[string]bool{"query": true}) + _, err = buildEvalRequest(group, schemas, map[string]bool{"query": true}) require.Error(t, err) require.Contains(t, err.Error(), "instruction_id_list") t.Logf("pre-flight error: %v", err) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build_test.go index 89785096c8f..f6c80e602a8 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build_test.go @@ -35,8 +35,8 @@ func schema(name string, dataRequired, dataProps, initRequired, initProps []stri } } -func groupWith(evaluators []evalcore.EvaluatorRef, opts *project.Options) *project.EvalGroup { - return &project.EvalGroup{ +func groupWith(evaluators []evalcore.EvaluatorRef, opts *project.Options) *project.Eval { + return &project.Eval{ Name: "g", Dataset: "d", Target: &project.Target{Type: "agent", Name: "my-agent"}, @@ -59,7 +59,7 @@ func TestBuildBindsAgentFieldsFromSample(t *testing.T) { &project.Options{EvalModel: "gpt-4.1-nano"}, ) - req, err := buildEvalGroupRequest(group, schemas, map[string]bool{"query": true}) + req, err := buildEvalRequest(group, schemas, map[string]bool{"query": true}) require.NoError(t, err) require.Len(t, req.TestingCriteria, 1) @@ -82,7 +82,7 @@ func TestBuildRejectsUnsatisfiableEvaluator(t *testing.T) { } group := groupWith([]evalcore.EvaluatorRef{{Name: "builtin.ifeval"}}, nil) - _, err := buildEvalGroupRequest(group, schemas, map[string]bool{"query": true}) + _, err := buildEvalRequest(group, schemas, map[string]bool{"query": true}) require.Error(t, err) require.Contains(t, err.Error(), "instruction_id_list") require.Contains(t, err.Error(), "instruction_kwargs") @@ -98,7 +98,7 @@ func TestBuildAcceptsEvaluatorWhenDatasetSupplies(t *testing.T) { } group := groupWith([]evalcore.EvaluatorRef{{Name: "builtin.ifeval"}}, nil) - req, err := buildEvalGroupRequest(group, schemas, map[string]bool{ + req, err := buildEvalRequest(group, schemas, map[string]bool{ "instruction_id_list": true, "instruction_kwargs": true, }) @@ -131,7 +131,7 @@ func TestBuildOmitsUnacceptedInitParameters(t *testing.T) { {Name: "builtin.similarity", Threshold: &threshold}, }, &project.Options{EvalModel: "gpt-4.1-nano"}) - req, err := buildEvalGroupRequest(group, schemas, map[string]bool{ + req, err := buildEvalRequest(group, schemas, map[string]bool{ "query": true, "ground_truth": true, }) require.NoError(t, err) @@ -163,7 +163,7 @@ func TestBuildPassesEvaluationLevelAsInitParameter(t *testing.T) { {Name: "builtin.similarity"}, }, &project.Options{EvalModel: "m", EvaluationLevel: "turn"}) - req, err := buildEvalGroupRequest(group, schemas, map[string]bool{"query": true}) + req, err := buildEvalRequest(group, schemas, map[string]bool{"query": true}) require.NoError(t, err) require.Equal(t, "turn", req.TestingCriteria[0].InitializationParameters["evaluation_level"]) @@ -181,7 +181,7 @@ func TestBuildRejectsUnsupportedLevel(t *testing.T) { group := groupWith([]evalcore.EvaluatorRef{{Name: "builtin.similarity"}}, &project.Options{EvalModel: "m", EvaluationLevel: "conversation"}) - _, err := buildEvalGroupRequest(group, schemas, map[string]bool{"query": true}) + _, err := buildEvalRequest(group, schemas, map[string]bool{"query": true}) require.Error(t, err) require.Contains(t, err.Error(), "conversation") require.Contains(t, err.Error(), "turn") @@ -196,7 +196,7 @@ func TestBuildRequiresJudgeModelWhenEvaluatorDoes(t *testing.T) { } group := groupWith([]evalcore.EvaluatorRef{{Name: "builtin.similarity"}}, nil) - _, err := buildEvalGroupRequest(group, schemas, map[string]bool{"query": true}) + _, err := buildEvalRequest(group, schemas, map[string]bool{"query": true}) require.Error(t, err) require.Contains(t, err.Error(), "deployment_name") } @@ -207,7 +207,7 @@ func TestBuildFallsBackWithoutSchema(t *testing.T) { group := groupWith([]evalcore.EvaluatorRef{{Name: "my-custom-evaluator"}}, &project.Options{EvalModel: "m"}) - req, err := buildEvalGroupRequest(group, nil, nil) + req, err := buildEvalRequest(group, nil, nil) require.NoError(t, err) mapping := req.TestingCriteria[0].DataMapping @@ -232,7 +232,7 @@ func TestBuildResolvesConversationTurnExclusivity(t *testing.T) { // Turn level keeps query/response and drops messages. turn := groupWith([]evalcore.EvaluatorRef{{Name: "builtin.task_completion"}}, &project.Options{EvalModel: "m", EvaluationLevel: "turn"}) - req, err := buildEvalGroupRequest(turn, schemas, columns) + req, err := buildEvalRequest(turn, schemas, columns) require.NoError(t, err) mapping := req.TestingCriteria[0].DataMapping require.Contains(t, mapping, "query") @@ -241,7 +241,7 @@ func TestBuildResolvesConversationTurnExclusivity(t *testing.T) { // Conversation level keeps messages and drops query/response. conv := groupWith([]evalcore.EvaluatorRef{{Name: "builtin.task_completion"}}, &project.Options{EvalModel: "m", EvaluationLevel: "conversation"}) - req, err = buildEvalGroupRequest(conv, schemas, columns) + req, err = buildEvalRequest(conv, schemas, columns) require.NoError(t, err) mapping = req.TestingCriteria[0].DataMapping require.Contains(t, mapping, "messages") @@ -251,7 +251,7 @@ func TestBuildResolvesConversationTurnExclusivity(t *testing.T) { // An unset level behaves as turn, matching the service default. dflt := groupWith([]evalcore.EvaluatorRef{{Name: "builtin.task_completion"}}, &project.Options{EvalModel: "m"}) - req, err = buildEvalGroupRequest(dflt, schemas, columns) + req, err = buildEvalRequest(dflt, schemas, columns) require.NoError(t, err) require.NotContains(t, req.TestingCriteria[0].DataMapping, "messages") } @@ -273,7 +273,7 @@ func TestBuildBindsJudgeModelUnderTheDeclaredName(t *testing.T) { {Name: "my-rubric"}, }, &project.Options{EvalModel: "gpt-4.1-nano"}) - req, err := buildEvalGroupRequest(group, schemas, map[string]bool{"query": true}) + req, err := buildEvalRequest(group, schemas, map[string]bool{"query": true}) require.NoError(t, err) builtin := req.TestingCriteria[0].InitializationParameters @@ -297,7 +297,7 @@ func TestBuildWithoutTargetSourcesEverythingFromDataset(t *testing.T) { group := groupWith([]evalcore.EvaluatorRef{{Name: "builtin.similarity"}}, nil) group.Target = nil - req, err := buildEvalGroupRequest(group, schemas, map[string]bool{ + req, err := buildEvalRequest(group, schemas, map[string]bool{ "query": true, "response": true, "ground_truth": true, }) require.NoError(t, err) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/compare.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/compare.go index f90ecfb5ef8..3859dc3664f 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/compare.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/compare.go @@ -31,7 +31,7 @@ func newResultsCompareCommand() *cobra.Command { cmd := &cobra.Command{ Use: "compare [eval-id]", - Short: "Compare runs of an eval group against a baseline.", + Short: "Compare runs of an eval against a baseline.", Args: cobra.MaximumNArgs(1), RunE: func(cmd *cobra.Command, args []string) error { ctx := cmd.Context() @@ -90,7 +90,7 @@ func newResultsCompareCommand() *cobra.Command { cmd.Flags().StringArrayVar(&treatments, "treatment", nil, "Run to measure, repeatable. Defaults to the most recent completed run.") cmd.Flags().StringVar(&displayName, "name", "", "Name for this comparison.") - addEvalGroupFlags(cmd, &groupName) + addEvalFlags(cmd, &groupName) cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") return cmd } @@ -110,12 +110,12 @@ func (ec *evalContext) resolveComparisonRuns( list, err := ec.evalClient.ListOpenAIEvalRuns(ctx, evalID, 0) if err != nil { - return "", nil, fmt.Errorf("listing runs of eval group %s: %w", evalID, err) + return "", nil, fmt.Errorf("listing runs of eval %s: %w", evalID, err) } completed := make([]string, 0, 2) if list == nil { - return "", nil, fmt.Errorf("eval group %s has no runs", evalID) + return "", nil, fmt.Errorf("eval %s has no runs", evalID) } for _, run := range list.Data { if run.Status == "completed" { @@ -126,14 +126,14 @@ func (ec *evalContext) resolveComparisonRuns( if len(treatments) == 0 { if len(completed) == 0 { return "", nil, fmt.Errorf( - "eval group %s has no completed runs to compare", evalID) + "eval %s has no completed runs to compare", evalID) } treatments = []string{completed[0]} } if baseline == "" { if len(completed) < 2 { return "", nil, fmt.Errorf( - "eval group %s has only one completed run, so there is nothing to compare it "+ + "eval %s has only one completed run, so there is nothing to compare it "+ "against; run it again, or name a baseline with --baseline", evalID) } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/context.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/context.go index 7ec1da4237a..0b987db0315 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/context.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/context.go @@ -48,7 +48,7 @@ func newEvalContext(ctx context.Context, endpointFlag string) (*evalContext, err ec := &evalContext{azdClient: azdClient} // The environment name is resolved regardless of where the endpoint comes - // from: it is what the cached eval group and run ids are read from and + // from: it is what the cached eval and run ids are read from and // written to. Deriving it only when the endpoint came from azd meant // --project-endpoint silently disabled that cache. azdEndpoint, envName := lookupEndpointFromAzd(ctx, azdClient) @@ -153,7 +153,7 @@ func (ec *evalContext) Close() { // azd environment keys written by this extension. const ( - envKeyEvalGroupID = "EVAL_GROUP_ID" + envKeyEvalID = "EVAL_ID" envKeyEvalRunID = "EVAL_RUN_ID" envKeyDatasetVersion = "EVAL_DATASET_VERSION" envKeyFingerprintPrefix = "EVAL_FINGERPRINT_" diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/description_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/description_test.go index ea819af4ef9..6d14b672856 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/description_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/description_test.go @@ -25,7 +25,7 @@ func TestBuildCarriesGroupDescriptionInMetadata(t *testing.T) { &project.Options{EvalModel: "m"}) group.Description = "Quality gate for the support agent" - req, err := buildEvalGroupRequest(group, schemas, map[string]bool{"query": true}) + req, err := buildEvalRequest(group, schemas, map[string]bool{"query": true}) require.NoError(t, err) require.Equal(t, "Quality gate for the support agent", req.Metadata["azd_description"]) } @@ -40,7 +40,7 @@ func TestBuildOmitsEmptyDescription(t *testing.T) { group := groupWith([]evalcore.EvaluatorRef{{Name: "builtin.similarity"}}, &project.Options{EvalModel: "m"}) - req, err := buildEvalGroupRequest(group, schemas, map[string]bool{"query": true}) + req, err := buildEvalRequest(group, schemas, map[string]bool{"query": true}) require.NoError(t, err) require.NotContains(t, req.Metadata, "azd_description") } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/envkeys_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/envkeys_test.go index b78bd8debf9..e7c404211c7 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/envkeys_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/envkeys_test.go @@ -14,8 +14,8 @@ import ( // exists, and hands it back for the wrong group — so group A silently scores // group B's criteria. func TestIDKey_IsPerName(t *testing.T) { - a := idKey("evalgroup", "quality-a") - b := idKey("evalgroup", "quality-b") + a := idKey("eval", "quality-a") + b := idKey("eval", "quality-b") assert.NotEqual(t, a, b, "two groups must not share an id key") assert.Contains(t, a, "QUALITY_A") @@ -25,9 +25,9 @@ func TestIDKey_IsPerName(t *testing.T) { // Names that are not valid env identifiers still have to produce distinct, // stable keys. func TestIDKey_NormalizesNames(t *testing.T) { - assert.Equal(t, idKey("evalgroup", "my group"), idKey("evalgroup", "my-group"), + assert.Equal(t, idKey("eval", "my group"), idKey("eval", "my-group"), "characters that cannot appear in an env name normalize the same way") - assert.NotEqual(t, idKey("evalgroup", "a"), idKey("dataset", "a"), + assert.NotEqual(t, idKey("eval", "a"), idKey("dataset", "a"), "the kind keeps different resources apart") } @@ -36,18 +36,18 @@ func TestIDKey_DoesNotCollideWithVersionKey(t *testing.T) { assert.NotEqual(t, idKey("dataset", "golden"), versionKey("dataset", "golden")) } -// Setting EVAL_GROUP_ID by hand is the documented way to point a config at a +// Setting EVAL_ID by hand is the documented way to point a config at a // group that already exists. It is also the key the extension writes itself, // which is what let a second group adopt the first one's id — so it stays // readable only where it cannot be ambiguous. Fixing the aliasing dropped this // fallback entirely once, silently breaking the documented behaviour. func TestGroupIDKeys_SharedKeyReadOnlyWhenUnambiguous(t *testing.T) { - sole := groupIDKeys("quality", true) - assert.Equal(t, idKey("evalgroup", "quality"), sole[0], + sole := evalIDKeys("quality", true) + assert.Equal(t, idKey("eval", "quality"), sole[0], "a group's own entry is preferred over the shared one") - assert.Contains(t, sole, envKeyEvalGroupID, + assert.Contains(t, sole, envKeyEvalID, "a single-group config honours an id set by hand") - assert.Equal(t, []string{idKey("evalgroup", "quality")}, groupIDKeys("quality", false), + assert.Equal(t, []string{idKey("eval", "quality")}, evalIDKeys("quality", false), "with several groups the shared entry cannot say which group it means") } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator.go index 4ec85fca946..100e8e7b10a 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator.go @@ -19,32 +19,29 @@ func newEvaluatorCommand() *cobra.Command { Short: "Manage custom evaluators.", } cmd.AddCommand( - newEvaluatorUploadCommand(false), - newEvaluatorUploadCommand(true), + newEvaluatorCreateCommand(), newEvaluatorListCommand(), newEvaluatorShowCommand(), - newEvaluatorBuiltinsCommand(), newEvaluatorDeleteCommand(), ) return cmd } -// newEvaluatorUploadCommand builds `evaluator upload` and `evaluator update`. -// Both publish a new immutable version. +// newEvaluatorCreateCommand builds `evaluator create`, named to match +// `dataset create`: both register an artifact and both publish a new immutable +// version every time, so there is nothing for a separate `update` to do. // // M1 supports rubric evaluators only. Code evaluators need a folder walk, -// multi-blob upload, and the Azure AI User role assignment, so they land in M2. -func newEvaluatorUploadCommand(update bool) *cobra.Command { +// multi-blob upload, and the Azure AI User role assignment, so they land later. +func newEvaluatorCreateCommand() *cobra.Command { var ( name string rubric string endpointFlg string ) - use, short := "upload", "Register a rubric evaluator, creating its first version." - if update { - use, short = "update", "Publish a new version of an existing rubric evaluator." - } + use := "create" + short := "Register a rubric evaluator, publishing a new version." cmd := &cobra.Command{ Use: use, @@ -165,12 +162,13 @@ func normalizeRubricBody(name string, raw []byte) (json.RawMessage, error) { func newEvaluatorListCommand() *cobra.Command { var ( name string + builtin bool endpointFlg string ) cmd := &cobra.Command{ Use: "list", - Short: "List evaluators, or the versions of one evaluator.", + Short: "List evaluators, the versions of one evaluator, or the built-in evaluators.", RunE: func(cmd *cobra.Command, args []string) error { ctx := cmd.Context() ec, err := newEvalContext(ctx, endpointFlg) @@ -180,9 +178,15 @@ func newEvaluatorListCommand() *cobra.Command { defer ec.Close() var list *eval_api.EvaluatorListResponse - if name != "" { + switch { + case name != "": list, err = ec.evalClient.ListEvaluatorVersions(ctx, name, ProjectEndpointAPIVersion) - } else { + case builtin: + // The service filters by type, and asking for nothing returns + // only the project's own evaluators. + list, err = ec.evalClient.ListEvaluators( + ctx, eval_api.EvaluatorTypeBuiltin, ProjectEndpointAPIVersion) + default: list, err = ec.evalClient.ListEvaluators(ctx, "", ProjectEndpointAPIVersion) } if err != nil { @@ -193,34 +197,8 @@ func newEvaluatorListCommand() *cobra.Command { } cmd.Flags().StringVar(&name, "name", "", "Limit the listing to versions of this evaluator.") - cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") - return cmd -} - -func newEvaluatorBuiltinsCommand() *cobra.Command { - var endpointFlg string - - cmd := &cobra.Command{ - Use: "builtins", - Short: "List the platform's built-in evaluators.", - RunE: func(cmd *cobra.Command, args []string) error { - ctx := cmd.Context() - ec, err := newEvalContext(ctx, endpointFlg) - if err != nil { - return err - } - defer ec.Close() - - list, err := ec.evalClient.ListEvaluators( - ctx, eval_api.EvaluatorTypeBuiltin, ProjectEndpointAPIVersion, - ) - if err != nil { - return fmt.Errorf("listing built-in evaluators: %w", err) - } - return renderEvaluators(cmd, list) - }, - } - + cmd.Flags().BoolVar(&builtin, "builtin", false, "List the built-in evaluators instead of the project's own.") + cmd.MarkFlagsMutuallyExclusive("name", "builtin") cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") return cmd } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go index a3508ade263..e0203f07d5e 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go @@ -79,7 +79,7 @@ func newGenerateCommand() *cobra.Command { generatingDataset := datasetFlag == "" && cfg.Generate.Dataset != nil if (generatingRubric || generatingDataset) && generationModel(cfg) == "" { return fmt.Errorf( - "a model deployment is required to generate: pass --eval-model, " + + "a model deployment is required to generate: pass --generation-model, " + "or set generate.rubric.model in the generation spec") } @@ -154,11 +154,11 @@ func newGenerateCommand() *cobra.Command { cmd.Flags().StringVar(&deployPath, "deploy-config", project.DefaultDeployConfig, "Deployment spec to write source references into.") cmd.Flags().StringVar(&target, "target", "", "Agent whose context seeds generation.") - cmd.Flags().StringVar(&instruction, "gen-instruction", "", + cmd.Flags().StringVar(&instruction, "agent-instruction", "", "What the agent does and what to test.") - cmd.Flags().StringVar(&instructionFile, "gen-instruction-file", "", - "Read the generation instruction from this file. Mutually exclusive with --gen-instruction.") - cmd.MarkFlagsMutuallyExclusive("gen-instruction", "gen-instruction-file") + cmd.Flags().StringVar(&instructionFile, "agent-instruction-file", "", + "Read the agent instruction from this file. Mutually exclusive with --agent-instruction.") + cmd.MarkFlagsMutuallyExclusive("agent-instruction", "agent-instruction-file") cmd.Flags().StringVar(&datasetFlag, "dataset", "", "Use this dataset instead of generating one.") cmd.Flags().StringArrayVar(&evaluators, "evaluator", nil, @@ -167,7 +167,7 @@ func newGenerateCommand() *cobra.Command { fmt.Sprintf("Rows to synthesize (%d-%d).", project.MinSampleSize, project.MaxSampleSize)) cmd.Flags().IntVar(&traceDays, "trace-days", 0, "Days of traces to seed rubric generation. 0 disables.") - cmd.Flags().StringVar(&evalModel, "eval-model", "", "Model deployment used for generation.") + cmd.Flags().StringVar(&evalModel, "generation-model", "", "Model deployment that generates the dataset and rubric.") cmd.Flags().BoolVar(&noWait, "no-wait", false, "Submit the jobs and return without polling.") cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") return cmd @@ -221,11 +221,11 @@ func resolveInstruction(inline, path string) (string, error) { } raw, err := os.ReadFile(path) if err != nil { - return "", fmt.Errorf("reading --gen-instruction-file %q: %w", path, err) + return "", fmt.Errorf("reading --agent-instruction-file %q: %w", path, err) } text := strings.TrimSpace(string(raw)) if text == "" { - return "", fmt.Errorf("--gen-instruction-file %q is empty", path) + return "", fmt.Errorf("--agent-instruction-file %q is empty", path) } return text, nil } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go index b7804ae91a6..85359d184ae 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go @@ -114,7 +114,7 @@ func newInitCommand() *cobra.Command { cmd.Flags().StringVar(&dataset, "dataset", "", "Path to a local .jsonl, or the name of a registered dataset.") cmd.Flags().StringArrayVar(&evaluators, "evaluator", nil, "Evaluator reference, repeatable. Use builtin. for a built-in.") - cmd.Flags().StringVar(&evalModel, "eval-model", "", "Model deployment used as the LLM judge.") + cmd.Flags().StringVar(&evalModel, "judge-model", "", "Model deployment that scores the results.") cmd.Flags().StringVar(&outDir, "out-dir", project.DefaultEvalDir, "Directory to write the config into. Used verbatim, never re-rooted.") cmd.Flags().BoolVar(&force, "force", false, "Overwrite existing files.") @@ -334,7 +334,7 @@ func buildDeployScaffold( } } - group := project.EvalGroup{ + group := project.Eval{ Name: fmt.Sprintf("%s-quality", target), Description: fmt.Sprintf("Quality gate for %s", target), Dataset: datasetName, @@ -347,7 +347,7 @@ func buildDeployScaffold( if evalModel != "" { group.Options = &project.Options{EvalModel: evalModel} } - cfg.EvalGroups = append(cfg.EvalGroups, group) + cfg.Evals = append(cfg.Evals, group) return cfg } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_test.go index f0db8167284..a4e22c85f87 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_test.go @@ -60,9 +60,9 @@ func TestScaffold_BuiltinEvaluatorsAreNotDeclared(t *testing.T) { require.Len(t, cfg.Evaluators, 1, "only the custom evaluator should be declared") require.Equal(t, "my-custom", cfg.Evaluators[0].Name) - require.Len(t, cfg.EvalGroups[0].Evaluators, 2) - require.True(t, cfg.EvalGroups[0].Evaluators[0].IsBuiltin()) - require.False(t, cfg.EvalGroups[0].Evaluators[1].IsBuiltin()) + require.Len(t, cfg.Evals[0].Evaluators, 2) + require.True(t, cfg.Evals[0].Evaluators[0].IsBuiltin()) + require.False(t, cfg.Evals[0].Evaluators[1].IsBuiltin()) path := filepath.Join(t.TempDir(), "azure.yaml") require.NoError(t, writeYAML(path, cfg)) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/instruction_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/instruction_test.go index 4063a6ab1b4..52279b70b9b 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/instruction_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/instruction_test.go @@ -40,7 +40,7 @@ func TestResolveInstructionPrefersInlineWhenNoFile(t *testing.T) { func TestResolveInstructionRejectsUnusableFile(t *testing.T) { _, err := resolveInstruction("", filepath.Join(t.TempDir(), "absent.md")) require.Error(t, err) - require.Contains(t, err.Error(), "gen-instruction-file") + require.Contains(t, err.Error(), "agent-instruction-file") empty := filepath.Join(t.TempDir(), "empty.md") require.NoError(t, os.WriteFile(empty, []byte(" \n"), 0o600)) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/jsonl_validation_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/jsonl_validation_test.go index e614af18378..7cf4fc48788 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/jsonl_validation_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/jsonl_validation_test.go @@ -20,7 +20,7 @@ func writeJSONL(t *testing.T, body string) string { } // The service accepts whatever bytes it is given, so a malformed row becomes a -// published version with an eval group bound to it, and only fails much later +// published version with an eval bound to it, and only fails much later // on a row nobody has looked at. A live deploy published `{not json at all}` // as version 1.0 before this existed. func TestValidateJSONL_RejectsAMalformedRowByLine(t *testing.T) { diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/model_target_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/model_target_test.go index 2a458a5778c..c202a87a4c7 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/model_target_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/model_target_test.go @@ -33,7 +33,7 @@ func TestSampleBindingsFor(t *testing.T) { } // The criteria a group sends depend on what it targets. -func TestBuildEvalGroupRequest_BindsByTargetKind(t *testing.T) { +func TestBuildEvalRequest_BindsByTargetKind(t *testing.T) { schemas := map[string]*eval_api.EvaluatorSummary{ "builtin.coherence": { Name: "builtin.coherence", @@ -57,12 +57,12 @@ func TestBuildEvalGroupRequest_BindsByTargetKind(t *testing.T) { {project.TargetTypeModel, "{{sample.output_text}}"}, } { t.Run(tc.targetType, func(t *testing.T) { - group := &project.EvalGroup{ + group := &project.Eval{ Name: "quality", Evaluators: []evalcore.EvaluatorRef{{Name: "builtin.coherence"}}, Target: &project.Target{Type: tc.targetType, Name: "thing"}, } - req, err := buildEvalGroupRequest(group, schemas, map[string]bool{"query": true}) + req, err := buildEvalRequest(group, schemas, map[string]bool{"query": true}) require.NoError(t, err) require.Len(t, req.TestingCriteria, 1) assert.Equal(t, tc.want, req.TestingCriteria[0].DataMapping["response"]) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go index a9d6737924e..a06f13853d3 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go @@ -69,7 +69,7 @@ func (r *evalReconciler) EnsureDataset( } // A malformed row is only noticed once the service tries to evaluate it, - // by which point a version has been published and the eval group points at + // by which point a version has been published and the eval points at // it. Reading the file here costs nothing and names the offending line. if err := validateJSONL(localPath); err != nil { return "", false, fmt.Errorf("dataset %q: %w", decl.Name, err) @@ -146,14 +146,14 @@ func (r *evalReconciler) EnsureDataset( // // Local content being unchanged is not enough to reuse the recorded version: // someone may have published a newer one outside the repo, and silently -// pinning the eval group to the older version would quietly evaluate against +// pinning the eval to the older version would quietly evaluate against // stale data. Publishing is not destructive — versions are immutable — so the // remedy is to sync, not to overwrite. // validateJSONL checks that every row is a JSON object before the file is // published. // // The service accepts the upload whatever the bytes are, so a typo becomes a -// registered version, an eval group bound to it, and a run that fails on a row +// registered version, an eval bound to it, and a run that fails on a row // nobody has looked at. Blank lines are skipped: they are not rows. func validateJSONL(path string) error { f, err := os.Open(path) @@ -270,12 +270,12 @@ func (r *evalReconciler) EnsureEvaluator( return created.Version, true, nil } -// EnsureEvalGroup creates the group when it has never been deployed, or when an +// EnsureEval creates the group when it has never been deployed, or when an // upstream artifact changed. Groups are immutable, so a change means a new // group and a new id. -func (r *evalReconciler) EnsureEvalGroup( +func (r *evalReconciler) EnsureEval( ctx context.Context, - group project.EvalGroup, + group project.Eval, datasetPath string, recreate bool, ) (string, error) { @@ -290,24 +290,24 @@ func (r *evalReconciler) EnsureEvalGroup( if err != nil { return "", err } - key := project.FingerprintKey("evalgroup", group.Name) + key := project.FingerprintKey("eval", group.Name) if prior := r.ec.getEnvValue(ctx, key); prior != "" && prior != digest { recreate = true } - cached := r.ec.getEnvValue(ctx, idKey("evalgroup", group.Name)) + cached := r.ec.getEnvValue(ctx, idKey("eval", group.Name)) if cached != "" && !recreate { if _, err := r.ec.evalClient.GetOpenAIEval(ctx, cached); err == nil { // Record the digest on reuse as well, otherwise a group deployed // before fingerprinting existed never establishes a baseline and // later edits go undetected. _ = r.ec.setEnvValue(ctx, key, digest) - _ = r.ec.setEnvValue(ctx, envKeyEvalGroupID, cached) + _ = r.ec.setEnvValue(ctx, envKeyEvalID, cached) return cached, nil } } - req, err := buildEvalGroupRequest( + req, err := buildEvalRequest( &group, r.ec.evaluatorSchemas(ctx), datasetColumnsFromPath(datasetPath), @@ -320,10 +320,10 @@ func (r *evalReconciler) EnsureEvalGroup( return "", err } _ = r.ec.setEnvValue(ctx, key, digest) - _ = r.ec.setEnvValue(ctx, idKey("evalgroup", group.Name), created.ID) - // EVAL_GROUP_ID stays the last-deployed group, which is what the commands + _ = r.ec.setEnvValue(ctx, idKey("eval", group.Name), created.ID) + // EVAL_ID stays the last-deployed group, which is what the commands // fall back to when a config names only one. - _ = r.ec.setEnvValue(ctx, envKeyEvalGroupID, created.ID) + _ = r.ec.setEnvValue(ctx, envKeyEvalID, created.ID) return created.ID, nil } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/resolution_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/resolution_test.go index 9bd441b0f4b..1d8c809bb6b 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/resolution_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/resolution_test.go @@ -15,28 +15,28 @@ import ( // is silent. options.max_samples was parsed and dropped once already, which is // what these lock down. func TestResolveMaxSamples_Precedence(t *testing.T) { - withOptions := &project.EvalGroup{Options: &project.Options{MaxSamples: 25}} + withOptions := &project.Eval{Options: &project.Options{MaxSamples: 25}} assert.Equal(t, 5, resolveMaxSamples(5, withOptions), "the flag wins over the config") assert.Equal(t, 25, resolveMaxSamples(0, withOptions), "the config is used when no flag is given") - assert.Equal(t, 0, resolveMaxSamples(0, &project.EvalGroup{}), "neither means no cap") + assert.Equal(t, 0, resolveMaxSamples(0, &project.Eval{}), "neither means no cap") assert.Equal(t, 0, resolveMaxSamples(0, nil)) assert.Equal(t, 7, resolveMaxSamples(7, nil), "a flag stands on its own") // Zero in config is absent, not a cap of zero: a cap of zero would send // nothing at all. - assert.Equal(t, 0, resolveMaxSamples(0, &project.EvalGroup{Options: &project.Options{MaxSamples: 0}})) + assert.Equal(t, 0, resolveMaxSamples(0, &project.Eval{Options: &project.Options{MaxSamples: 0}})) } func TestResolveLevel_Precedence(t *testing.T) { - withOptions := &project.EvalGroup{ + withOptions := &project.Eval{ Options: &project.Options{EvaluationLevel: project.EvaluationLevelConversation}, } assert.Equal(t, project.EvaluationLevelTurn, resolveLevel(project.EvaluationLevelTurn, withOptions), "the flag wins over the config") assert.Equal(t, project.EvaluationLevelConversation, resolveLevel("", withOptions)) - assert.Empty(t, resolveLevel("", &project.EvalGroup{}), "unset defers to the service default") + assert.Empty(t, resolveLevel("", &project.Eval{}), "unset defers to the service default") assert.Empty(t, resolveLevel("", nil)) } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/results.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/results.go index 0cc74767f1f..405b538c01e 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/results.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/results.go @@ -56,25 +56,45 @@ func newResultsShowCommand() *cobra.Command { return err } + // The run carries totals and a per-criterion breakdown. The output + // items are the rows themselves, which is what "which one failed, + // and why" needs. A run that never produced any still renders its + // totals rather than failing. + items, err := ec.evalClient.ListOutputItems(ctx, evalID, run.ID, 0) + if err != nil { + return fmt.Errorf("reading the results of run %s: %w", run.ID, err) + } + rows := items.Data + if failedOnly { + kept := make([]eval_api.OutputItem, 0, len(rows)) + for _, it := range rows { + if it.Failed() { + kept = append(kept, it) + } + } + rows = kept + } + + payload := map[string]any{"run": run, "output_items": rows} if outFile != "" { f, err := os.Create(outFile) if err != nil { return fmt.Errorf("creating %q: %w", outFile, err) } defer f.Close() - return emitJSON(f, run) + return emitJSON(f, payload) } if isJSON(cmd) { - return emitJSON(cmd.OutOrStdout(), run) + return emitJSON(cmd.OutOrStdout(), payload) } - return renderResults(cmd.OutOrStdout(), run, failedOnly) + return renderResults(cmd.OutOrStdout(), run, rows, failedOnly) }, } cmd.Flags().StringVar(&runID, "run-id", "", "Run to show. Defaults to the most recent run.") - cmd.Flags().BoolVar(&failedOnly, "failed-only", false, "Show only criteria with failures.") + cmd.Flags().BoolVar(&failedOnly, "failed-only", false, "Show only the rows that failed.") cmd.Flags().StringVarP(&outFile, "out-file", "O", "", "Write JSON results to this path.") - addEvalGroupFlags(cmd, &groupName) + addEvalFlags(cmd, &groupName) cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") return cmd } @@ -135,16 +155,16 @@ func newResultsExportCommand() *cobra.Command { cmd.Flags().StringVar(&runID, "run-id", "", "Run to export. Defaults to the most recent run.") cmd.Flags().StringVar(&format, "format", "json", "Output format: json or csv.") cmd.Flags().StringVarP(&outFile, "out-file", "O", "", "Write to this path instead of stdout.") - addEvalGroupFlags(cmd, &groupName) + addEvalFlags(cmd, &groupName) cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") return cmd } -// resolveEvalID takes the eval group id from the argument, from a group named -// with --eval-group, or from the id cached in the azd environment. +// resolveEvalID takes the eval id from the argument, from a group named +// with --eval, or from the id cached in the azd environment. // // The cached id is the last group deployed, which is unambiguous only while a -// config declares one. --eval-group is how the others are reached without +// config declares one. --eval is how the others are reached without // having to know their service ids. func resolveEvalID( cmd *cobra.Command, @@ -161,38 +181,38 @@ func resolveEvalID( } if groupName != "" { - if id := ec.getEnvValue(cmd.Context(), idKey("evalgroup", groupName)); id != "" { + if id := ec.getEnvValue(cmd.Context(), idKey("eval", groupName)); id != "" { return id, nil } return "", fmt.Errorf( - "eval group %q has no id recorded in this environment; deploy it first, "+ + "eval %q has no id recorded in this environment; deploy it first, "+ "or pass its id directly", groupName) } - if cached := ec.getEnvValue(cmd.Context(), envKeyEvalGroupID); cached != "" { + if cached := ec.getEnvValue(cmd.Context(), envKeyEvalID); cached != "" { return cached, nil } return "", fmt.Errorf( - "no eval group id given; pass it as an argument, name one with --eval-group, "+ + "no eval id given; pass it as an argument, name one with --eval, "+ "or set %s in the azd environment", - envKeyEvalGroupID) + envKeyEvalID) } -// addEvalGroupFlag registers the flag that names a group from the config, so +// addEvalFlag registers the flag that names a group from the config, so // every command taking an eval-id can reach a group by the name its author // used. -// addEvalGroupFlags registers the two ways to say which group a command acts -// on: --eval-group names one from the config, --eval-id gives its service id. +// addEvalFlags registers the two ways to say which group a command acts +// on: --eval names one from the config, --eval-id gives its service id. // // The id is also accepted as a positional argument. The flag exists because // `run start --eval-id` already spells it that way, and a script that learned // it there should not have to find out that the sibling commands take only a // positional. -func addEvalGroupFlags(cmd *cobra.Command, target *string) { - cmd.Flags().StringVar(target, "eval-group", "", +func addEvalFlags(cmd *cobra.Command, target *string) { + cmd.Flags().StringVar(target, "eval", "", "Name a group from the config instead of passing its id.") cmd.Flags().String("eval-id", "", - "Id of the eval group. Same as passing the id as an argument.") + "Id of the eval. Same as passing the id as an argument.") } // latestOrNamedRun returns the named run, or the most recent one for the group. @@ -224,18 +244,23 @@ func (ec *evalContext) latestOrNamedRun( if err != nil { if eval_api.IsNotFound(err) { return nil, fmt.Errorf( - "no eval group %q in this project; "+ + "no eval %q in this project; "+ "`azd up` creates the ones your config declares", evalID) } - return nil, fmt.Errorf("listing runs for eval group %s: %w", evalID, err) + return nil, fmt.Errorf("listing runs for eval %s: %w", evalID, err) } if len(list.Data) == 0 { - return nil, fmt.Errorf("eval group %s has no runs yet", evalID) + return nil, fmt.Errorf("eval %s has no runs yet", evalID) } return &list.Data[0], nil } -func renderResults(w io.Writer, run *eval_api.OpenAIEvalRun, failedOnly bool) error { +func renderResults( + w io.Writer, + run *eval_api.OpenAIEvalRun, + items []eval_api.OutputItem, + failedOnly bool, +) error { fmt.Fprintf(w, "Run %s status: %s\n", run.ID, run.Status) if c := run.ResultCounts; c != nil { @@ -243,35 +268,80 @@ func renderResults(w io.Writer, run *eval_api.OpenAIEvalRun, failedOnly bool) er c.Passed, c.Failed, c.Errored) } - if len(run.PerTestingCriteria) == 0 { - fmt.Fprintln(w, "No per-criteria results are available yet.") - return nil + if len(run.PerTestingCriteria) > 0 { + rows := make([][]string, 0, len(run.PerTestingCriteria)) + for _, cr := range run.PerTestingCriteria { + if failedOnly && cr.Failed == 0 { + continue + } + rows = append(rows, []string{ + cr.TestingCriteria, + strconv.Itoa(cr.Passed), + strconv.Itoa(cr.Failed), + }) + } + if len(rows) > 0 { + if err := emitTable(w, []string{"CRITERION", "PASSED", "FAILED"}, rows); err != nil { + return err + } + } } - rows := make([][]string, 0, len(run.PerTestingCriteria)) - for _, cr := range run.PerTestingCriteria { - if failedOnly && cr.Failed == 0 { - continue + // The rows are the point of `results show`: totals say how many failed, + // these say which and why. + if len(items) == 0 { + if failedOnly { + fmt.Fprintln(w, "\nNo failing rows.") + } else { + fmt.Fprintln(w, "\nNo rows have been scored yet.") + } + } else { + fmt.Fprintln(w) + rows := make([][]string, 0, len(items)) + for _, it := range items { + for _, r := range it.Results { + if failedOnly && r.Passed { + continue + } + verdict := "pass" + if !r.Passed { + verdict = "FAIL" + } + rows = append(rows, []string{ + it.ID, + r.Name, + verdict, + formatStat("%.3f", r.Score), + truncate(it.Input(), 48), + truncate(r.Reason, 60), + }) + } + } + if err := emitTable(w, + []string{"ITEM", "EVALUATOR", "RESULT", "SCORE", "INPUT", "REASON"}, rows); err != nil { + return err } - rows = append(rows, []string{ - cr.TestingCriteria, - strconv.Itoa(cr.Passed), - strconv.Itoa(cr.Failed), - }) - } - if len(rows) == 0 { - fmt.Fprintln(w, "No failing criteria.") - return nil - } - if err := emitTable(w, []string{"CRITERION", "PASSED", "FAILED"}, rows); err != nil { - return err } + if run.ReportURL != "" { fmt.Fprintf(w, "\nReport: %s\n", run.ReportURL) } return nil } +// truncate keeps a table readable when a reason runs to a paragraph. The full +// text is always in `-o json`. +func truncate(s string, n int) string { + s = strings.ReplaceAll(strings.ReplaceAll(s, "\n", " "), "\r", "") + if len(s) <= n { + return s + } + if n <= 1 { + return s[:n] + } + return s[:n-1] + "…" +} + func writeResultsCSV(w io.Writer, run *eval_api.OpenAIEvalRun) error { cw := csv.NewWriter(w) defer cw.Flush() diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go index 3f5d81ebb0f..a9a370113ee 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go @@ -37,10 +37,10 @@ var terminalRunStates = map[string]bool{ // of this same command. func newRunCommand() *cobra.Command { cmd := buildRunCommand( - "run", "Run an evaluation, creating the eval group if it does not exist yet.") + "run", "Run an evaluation, creating the eval if it does not exist yet.") addRunSubcommands(cmd) cmd.AddCommand(buildRunCommand( - "start", "Start a run, creating the eval group if it does not exist yet.")) + "start", "Start a run, creating the eval if it does not exist yet.")) return cmd } @@ -77,7 +77,7 @@ func buildRunCommand(use, short string) *cobra.Command { defer ec.Close() // --eval-id bypasses the config entirely. - var group *project.EvalGroup + var group *project.Eval if evalID == "" { cfg, err := project.LoadEvalConfig(configPath) if err != nil { @@ -95,9 +95,9 @@ func buildRunCommand(use, short string) *cobra.Command { return err } - evalID, err = ec.resolveEvalGroupID( + evalID, err = ec.resolveEvalIDFromConfig( ctx, group, configPath, resolveLevel(level, group), - len(cfg.EvalGroups) == 1, out, isJSON(cmd)) + len(cfg.Evals) == 1, out, isJSON(cmd)) if err != nil { return err } @@ -179,10 +179,10 @@ func buildRunCommand(use, short string) *cobra.Command { cmd.Flags().StringVar(&configPath, "config", project.DefaultDeployConfig, "Path to the eval deployment config.") - cmd.Flags().StringVar(&groupName, "eval-group", "", - "Which evalGroups entry to run. Defaults to the only one.") + cmd.Flags().StringVar(&groupName, "eval", "", + "Which evals entry to run. Defaults to the only one.") cmd.Flags().StringVar(&evalID, "eval-id", "", - "Run against an existing eval group by id, ignoring the config.") + "Run against an existing eval by id, ignoring the config.") cmd.Flags().StringVar(&runName, "name", "", "Name for this run. Defaults to the group name plus a timestamp.") cmd.Flags().StringVar(&level, "level", "", "Scoring granularity: turn or conversation. Defaults to the service default (turn).") @@ -213,12 +213,12 @@ func buildRunCommand(use, short string) *cobra.Command { return cmd } -// resolveEvalGroupID finds the eval group to run against, creating it when it +// resolveEvalIDFromConfig finds the eval to run against, creating it when it // has never been deployed. Resolution order: an id pinned on the group, then // the azd environment, then create. -func (ec *evalContext) resolveEvalGroupID( +func (ec *evalContext) resolveEvalIDFromConfig( ctx context.Context, - group *project.EvalGroup, + group *project.Eval, configPath string, level string, soleGroup bool, @@ -229,7 +229,7 @@ func (ec *evalContext) resolveEvalGroupID( return group.ID, nil } - for _, key := range groupIDKeys(group.Name, soleGroup) { + for _, key := range evalIDKeys(group.Name, soleGroup) { cached := ec.getEnvValue(ctx, key) if cached == "" { continue @@ -241,7 +241,7 @@ func (ec *evalContext) resolveEvalGroupID( } if !jsonMode { - fmt.Fprintf(out, "Creating eval group %q...\n", group.Name) + fmt.Fprintf(out, "Creating eval %q...\n", group.Name) } // The level from the flag wins over the group's own options, so it has to @@ -256,7 +256,7 @@ func (ec *evalContext) resolveEvalGroupID( effective.Options = &opts } - req, err := buildEvalGroupRequest( + req, err := buildEvalRequest( &effective, ec.evaluatorSchemas(ctx), datasetColumns(configPath, group), @@ -266,28 +266,28 @@ func (ec *evalContext) resolveEvalGroupID( } created, err := ec.evalClient.CreateOpenAIEval(ctx, req) if err != nil { - return "", fmt.Errorf("creating eval group %q: %w", group.Name, err) + return "", fmt.Errorf("creating eval %q: %w", group.Name, err) } - if err := ec.setEnvValue(ctx, idKey("evalgroup", group.Name), created.ID); err != nil { + if err := ec.setEnvValue(ctx, idKey("eval", group.Name), created.ID); err != nil { fmt.Fprintf(out, "warning: %v\n", err) } - _ = ec.setEnvValue(ctx, envKeyEvalGroupID, created.ID) + _ = ec.setEnvValue(ctx, envKeyEvalID, created.ID) return created.ID, nil } -// groupIDKeys lists the env entries that may hold this group's id, most +// evalIDKeys lists the env entries that may hold this group's id, most // specific first. // -// The per-name entry is what the extension writes. EVAL_GROUP_ID is also the +// The per-name entry is what the extension writes. EVAL_ID is also the // documented way to point a config at a group that already exists, created in // the portal or by another tool, so it stays readable — but only when the // config declares a single group. With more than one there is no way to tell // which group a shared entry refers to, and reading it anyway is what let a // second group adopt the first one's id. -func groupIDKeys(name string, soleGroup bool) []string { - keys := []string{idKey("evalgroup", name)} +func evalIDKeys(name string, soleGroup bool) []string { + keys := []string{idKey("eval", name)} if soleGroup { - keys = append(keys, envKeyEvalGroupID) + keys = append(keys, envKeyEvalID) } return keys } @@ -297,7 +297,7 @@ func groupIDKeys(name string, soleGroup bool) []string { // // A run sends a local dataset inline, so without this the run would evaluate // content that no registered version corresponds to: the results are attributed -// to the eval group but cannot be traced back to a dataset version, which +// to the eval but cannot be traced back to a dataset version, which // makes them impossible to reproduce or compare. // // The check only applies once a deploy has recorded a fingerprint. Before that @@ -306,7 +306,7 @@ func groupIDKeys(name string, soleGroup bool) []string { func (ec *evalContext) checkDatasetRegistered( ctx context.Context, cfg *project.EvalConfig, - group *project.EvalGroup, + group *project.Eval, configPath string, ) error { localPath := localDatasetPath(configPath, group) @@ -336,7 +336,7 @@ func (ec *evalContext) checkDatasetRegistered( return fmt.Errorf( "dataset %q has local edits that are not registered.\n"+ " Run `azd up` to register them, or `--eval-id ` to run against "+ - "an existing eval group", + "an existing eval", decl.Name) } @@ -344,7 +344,7 @@ func (ec *evalContext) checkDatasetRegistered( // recent run. // // `--eval-id` deliberately ignores the config, but a run still needs a target -// and a dataset, and an eval group carries neither: the group holds only its +// and a dataset, and an eval carries neither: the group holds only its // testing criteria, and the dataset travels on the run. The previous run is the // only place that pairing survives, so re-running a group means repeating what // it last ran. @@ -354,11 +354,11 @@ func (ec *evalContext) reuseDataSourceFromLastRun( ) (*eval_api.EvalRunDataSource, error) { list, err := ec.evalClient.ListOpenAIEvalRuns(ctx, evalID, 1) if err != nil { - return nil, fmt.Errorf("reading previous runs of eval group %s: %w", evalID, err) + return nil, fmt.Errorf("reading previous runs of eval %s: %w", evalID, err) } if list == nil || len(list.Data) == 0 || list.Data[0].DataSource == nil { return nil, fmt.Errorf( - "eval group %s has no previous run to repeat, so there is no target or dataset "+ + "eval %s has no previous run to repeat, so there is no target or dataset "+ "to reuse.\n"+ " Run it from the config once with `azd ai eval run`, or pass a config that "+ "declares the group", @@ -376,7 +376,7 @@ func (ec *evalContext) reuseDataSourceFromLastRun( func buildTracesDataSource( ctx context.Context, ec *evalContext, - group *project.EvalGroup, + group *project.Eval, evalID, window string, maxTraces int, ) (*eval_api.EvalRunDataSource, error) { @@ -397,7 +397,7 @@ func buildTracesDataSource( } if agent == "" { return nil, fmt.Errorf( - "--from-traces needs to know whose traces to read, and the eval group does not " + + "--from-traces needs to know whose traces to read, and the eval does not " + "name an agent. Declare target.type: agent on the group") } @@ -408,16 +408,16 @@ func buildTracesDataSource( return eval_api.NewTracesDataSource(agent, lookbackHours, time.Time{}, maxTraces), nil } -// buildRunDataSource binds the dataset to the run. The eval group carries no +// buildRunDataSource binds the dataset to the run. The eval carries no // dataset today, so it is supplied here. func (ec *evalContext) buildRunDataSource( ctx context.Context, - group *project.EvalGroup, + group *project.Eval, configPath string, maxSamples int, ) (*eval_api.EvalRunDataSource, error) { if group == nil { - return nil, fmt.Errorf("no eval group to run") + return nil, fmt.Errorf("no eval to run") } // A group with no target scores a dataset that already holds the exchange, @@ -434,7 +434,7 @@ func (ec *evalContext) buildRunDataSource( } if group.Dataset == "" { - return nil, fmt.Errorf("eval group %q does not reference a dataset", group.Name) + return nil, fmt.Errorf("eval %q does not reference a dataset", group.Name) } // A local source is read from disk; anything else is already registered and @@ -512,7 +512,7 @@ func (ec *evalContext) readRegisteredDataset( // A nil result means the columns are unknown, which is the case for a dataset // already registered in the project. The builder then assumes every field an // evaluator accepts is present. -func datasetColumns(configPath string, group *project.EvalGroup) map[string]bool { +func datasetColumns(configPath string, group *project.Eval) map[string]bool { return datasetColumnsFromPath(localDatasetPath(configPath, group)) } @@ -536,7 +536,7 @@ func datasetColumnsFromPath(localPath string) map[string]bool { // localDatasetPath resolves the dataset's local source relative to the config // file, returning empty when the dataset is registered rather than local. -func localDatasetPath(configPath string, group *project.EvalGroup) string { +func localDatasetPath(configPath string, group *project.Eval) string { cfg, err := project.LoadEvalConfig(configPath) if err != nil { return "" @@ -601,7 +601,7 @@ func scanJSONL(r io.Reader, limit int) ([]map[string]any, error) { } // resolveLevel prefers the flag, then the group's options. -func resolveLevel(flag string, group *project.EvalGroup) string { +func resolveLevel(flag string, group *project.Eval) string { if flag != "" { return flag } @@ -617,7 +617,7 @@ func resolveLevel(flag string, group *project.EvalGroup) string { // Without this, options.max_samples parsed and did nothing: a group that caps // its sample count in config would send the whole dataset, and only a flag on // every invocation would honour the cap. -func resolveMaxSamples(flag int, group *project.EvalGroup) int { +func resolveMaxSamples(flag int, group *project.Eval) int { if flag > 0 { return flag } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go index e815d4866bd..8fab2f3bca6 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go @@ -21,6 +21,7 @@ func addRunSubcommands(cmd *cobra.Command) { newRunListCommand(), newRunShowCommand(), newRunCancelCommand(), + newRunDeleteCommand(), ) } @@ -32,7 +33,7 @@ func newRunListCommand() *cobra.Command { cmd := &cobra.Command{ Use: "list [eval-id]", - Short: "List runs for an eval group.", + Short: "List runs for an eval.", Args: cobra.MaximumNArgs(1), RunE: func(cmd *cobra.Command, args []string) error { ctx := cmd.Context() @@ -51,7 +52,7 @@ func newRunListCommand() *cobra.Command { if err != nil { if eval_api.IsNotFound(err) { return fmt.Errorf( - "no eval group %q in this project; "+ + "no eval %q in this project; "+ "`azd up` creates the ones your config declares", evalID) } return fmt.Errorf("listing runs for %q: %w", evalID, err) @@ -64,7 +65,7 @@ func newRunListCommand() *cobra.Command { return emitJSONList(cmd.OutOrStdout(), runs) } if list == nil || len(list.Data) == 0 { - fmt.Fprintf(cmd.OutOrStdout(), "Eval group %s has no runs yet.\n", evalID) + fmt.Fprintf(cmd.OutOrStdout(), "Eval %s has no runs yet.\n", evalID) return nil } @@ -76,7 +77,7 @@ func newRunListCommand() *cobra.Command { []string{"RUN ID", "NAME", "STATUS", "RESULTS"}, rows) }, } - addEvalGroupFlags(cmd, &groupName) + addEvalFlags(cmd, &groupName) cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") return cmd } @@ -127,7 +128,7 @@ func newRunShowCommand() *cobra.Command { }, } cmd.Flags().StringVar(&runID, "run-id", "", "Run to show. Defaults to the most recent run.") - addEvalGroupFlags(cmd, &groupName) + addEvalFlags(cmd, &groupName) cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") return cmd } @@ -183,7 +184,62 @@ func newRunCancelCommand() *cobra.Command { }, } cmd.Flags().StringVar(&runID, "run-id", "", "Run to cancel. Defaults to the most recent run.") - addEvalGroupFlags(cmd, &groupName) + addEvalFlags(cmd, &groupName) + cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") + return cmd +} + +// newRunDeleteCommand removes a run. +// +// Runs accumulate — every `azd ai eval run` adds one — and a run that evaluated +// the wrong dataset or target is noise in every later listing and comparison. +// The id is required rather than defaulted to the most recent run, because +// deleting is not undoable and "the latest one" is a poor thing to guess at. +func newRunDeleteCommand() *cobra.Command { + var ( + runID string + endpointFlg string + groupName string + ) + + cmd := &cobra.Command{ + Use: "delete [eval-id]", + Short: "Delete a run.", + Args: cobra.MaximumNArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + ctx := cmd.Context() + if runID == "" { + return requireFlag("run-id") + } + ec, err := newEvalContext(ctx, endpointFlg) + if err != nil { + return err + } + defer ec.Close() + + evalID, err := resolveEvalID(cmd, ec, args, groupName) + if err != nil { + return err + } + + if err := ec.evalClient.DeleteOpenAIEvalRun(ctx, evalID, runID); err != nil { + if eval_api.IsNotFound(err) { + return fmt.Errorf("no run %q on eval %q", runID, evalID) + } + return fmt.Errorf("deleting run %s: %w", runID, err) + } + + if isJSON(cmd) { + return emitJSON(cmd.OutOrStdout(), map[string]string{ + "id": runID, "eval_id": evalID, "status": "deleted", + }) + } + fmt.Fprintf(cmd.OutOrStdout(), "Deleted run %s\n", runID) + return nil + }, + } + cmd.Flags().StringVar(&runID, "run-id", "", "Run to delete.") + addEvalFlags(cmd, &groupName) cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") return cmd } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops_test.go index edc4d99ccef..8063f09253b 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops_test.go @@ -57,16 +57,16 @@ func TestRunStartMirrorsCompositeFlags(t *testing.T) { } require.NotNil(t, start) - for _, flag := range []string{"eval-id", "eval-group", "name", "level", "max-samples", "wait", "no-wait"} { + for _, flag := range []string{"eval-id", "eval", "name", "level", "max-samples", "wait", "no-wait"} { require.NotNil(t, start.Flags().Lookup(flag), "run start should accept --%s", flag) } } -// Every command that acts on an eval group takes the id the same two ways. +// Every command that acts on an eval takes the id the same two ways. // `run start --eval-id` is the form the CI example uses, and `run list` used to // reject that flag and accept only a positional, so a script that worked for // one sibling failed on the next. -func TestEvalGroupCommandsAcceptIDAsAFlag(t *testing.T) { +func TestEvalCommandsAcceptIDAsAFlag(t *testing.T) { subs := map[string]*cobra.Command{} for _, sub := range newRunCommand().Commands() { subs["run "+sub.Name()] = sub @@ -82,7 +82,7 @@ func TestEvalGroupCommandsAcceptIDAsAFlag(t *testing.T) { cmd := subs[name] require.NotNil(t, cmd, "%s should exist", name) require.NotNil(t, cmd.Flags().Lookup("eval-id"), "%s should accept --eval-id", name) - require.NotNil(t, cmd.Flags().Lookup("eval-group"), "%s should accept --eval-group", name) + require.NotNil(t, cmd.Flags().Lookup("eval"), "%s should accept --eval", name) } } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/schedule.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/schedule.go index b2972b29ac9..b2f223ebf0c 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/schedule.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/schedule.go @@ -20,7 +20,7 @@ import ( func newScheduleCommand() *cobra.Command { cmd := &cobra.Command{ Use: "schedule", - Short: "Run an eval group on a schedule.", + Short: "Run an eval on a schedule.", } cmd.AddCommand( newScheduleSetCommand(), @@ -31,7 +31,7 @@ func newScheduleCommand() *cobra.Command { return cmd } -// newScheduleSetCommand creates the schedule that runs an eval group. +// newScheduleSetCommand creates the schedule that runs an eval. // // It does not update. The service accepts a PUT over an existing schedule, // echoes the new body and keeps the old trigger, so an in-place edit would @@ -64,7 +64,7 @@ func newScheduleSetCommand() *cobra.Command { cmd := &cobra.Command{ Use: "set [eval-id]", - Short: "Create the schedule that runs an eval group.", + Short: "Create the schedule that runs an eval.", Args: cobra.MaximumNArgs(1), RunE: func(cmd *cobra.Command, args []string) error { ctx := cmd.Context() @@ -99,7 +99,7 @@ func newScheduleSetCommand() *cobra.Command { // Same resolution as `run`: the config names the group unless // --eval-id bypasses it, and the run payload carries the target // and dataset because the group holds neither. - var group *project.EvalGroup + var group *project.Eval var dataSource *eval_api.EvalRunDataSource if evalID == "" { cfg, err := project.LoadEvalConfig(configPath) @@ -116,9 +116,9 @@ func newScheduleSetCommand() *cobra.Command { if err := ec.checkDatasetRegistered(ctx, cfg, group, configPath); err != nil { return err } - evalID, err = ec.resolveEvalGroupID( + evalID, err = ec.resolveEvalIDFromConfig( ctx, group, configPath, resolveLevel(level, group), - len(cfg.EvalGroups) == 1, out, isJSON(cmd)) + len(cfg.Evals) == 1, out, isJSON(cmd)) if err != nil { return err } @@ -195,8 +195,8 @@ func newScheduleSetCommand() *cobra.Command { cmd.Flags().StringVar(&configPath, "config", project.DefaultDeployConfig, "Path to the eval deployment config.") - cmd.Flags().StringVar(&groupName, "eval-group", "", "Which evalGroups entry to schedule.") - cmd.Flags().StringVar(&evalID, "eval-id", "", "Schedule an existing eval group by id, ignoring config.") + cmd.Flags().StringVar(&groupName, "eval", "", "Which evals entry to schedule.") + cmd.Flags().StringVar(&evalID, "eval-id", "", "Schedule an existing eval by id, ignoring config.") cmd.Flags().StringVar(&name, "name", "", "Schedule name. Defaults to the group name.") cmd.Flags().StringVar(&description, "description", "", "Schedule description.") cmd.Flags().StringVar(&cron, "cron", "", `Cron expression, for example "0 9 * * *".`) @@ -535,7 +535,7 @@ func normalizeDaysOfWeek(days []string) ([]string, error) { } // defaultScheduleName derives a schedule name from the group being scheduled. -func defaultScheduleName(group *project.EvalGroup) string { +func defaultScheduleName(group *project.Eval) string { if group != nil && group.Name != "" { return group.Name } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/evaluators.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/evaluators.go index e03a7059efd..2d9a85ffa65 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/evaluators.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/evaluators.go @@ -9,6 +9,7 @@ import ( "net/http" "net/url" "sort" + "strconv" "strings" ) @@ -184,3 +185,36 @@ func (c *EvalClient) CancelOpenAIEvalRun( ) return doRequestTyped[OpenAIEvalRun](c, ctx, http.MethodPost, path, nil, nil, "") } + +// DeleteOpenAIEvalRun removes a single run. +func (c *EvalClient) DeleteOpenAIEvalRun(ctx context.Context, evalID, runID string) error { + path := fmt.Sprintf( + "%s/%s/runs/%s", + pathOpenAIEvals, url.PathEscape(evalID), url.PathEscape(runID), + ) + _, err := c.doRequest(ctx, http.MethodDelete, path, nil, nil, "") + return err +} + +// ListOutputItems returns a run's per-sample results. +// +// The run itself carries only totals and a per-criterion breakdown. The output +// items are the rows: each one holds the dataset item that was evaluated, what +// the target answered, and every evaluator's score, verdict and reason. Showing +// results without them can say how many failed but never which, or why. +func (c *EvalClient) ListOutputItems( + ctx context.Context, + evalID, runID string, + limit int, +) (*OutputItemList, error) { + query := map[string]string{} + if limit > 0 { + query["limit"] = strconv.Itoa(limit) + } + + path := fmt.Sprintf( + "%s/%s/runs/%s/output_items", + pathOpenAIEvals, url.PathEscape(evalID), url.PathEscape(runID), + ) + return doRequestTyped[OutputItemList](c, ctx, http.MethodGet, path, query, nil, "") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/models.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/models.go index 506fefd9578..2b9fa2ca9d1 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/models.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/models.go @@ -5,6 +5,8 @@ package eval_api import ( "encoding/json" + "fmt" + "sort" "strings" "time" ) @@ -566,3 +568,67 @@ type EvalRunCriteriaResult struct { type OpenAIEvalRunList struct { Data []OpenAIEvalRun `json:"data"` } + +// OutputItemList is a page of a run's per-sample results. +type OutputItemList struct { + Data []OutputItem `json:"data"` +} + +// OutputItem is one evaluated row: the dataset item, and every evaluator's +// verdict on it. +type OutputItem struct { + ID string `json:"id"` + RunID string `json:"run_id"` + Status string `json:"status"` + DataSourceItem map[string]any `json:"datasource_item,omitempty"` + Results []OutputResult `json:"results,omitempty"` +} + +// OutputResult is one evaluator's verdict on one row. +type OutputResult struct { + Name string `json:"name"` + Metric string `json:"metric,omitempty"` + Score LenientFloat `json:"score"` + Label string `json:"label,omitempty"` + Passed bool `json:"passed"` + // Reason is the judge's explanation, which is the part a failing row is + // actually looked at for. + Reason string `json:"reason,omitempty"` +} + +// Failed reports whether any evaluator failed this row. +func (o OutputItem) Failed() bool { + for _, r := range o.Results { + if !r.Passed { + return true + } + } + return false +} + +// Input renders the row's own columns for display, leaving out the +// service-injected `sample.*` bindings and the plumbing ids, which are not what +// the dataset author wrote. +func (o OutputItem) Input() string { + if len(o.DataSourceItem) == 0 { + return "" + } + skip := map[string]bool{ + "response_id": true, "agent_id": true, "agent_name": true, + "agent_version": true, "conversation_id": true, + "previous_response_id": true, "trace_id": true, "span_id": true, + } + keys := make([]string, 0, len(o.DataSourceItem)) + for k := range o.DataSourceItem { + if skip[k] || strings.HasPrefix(k, "sample.") { + continue + } + keys = append(keys, k) + } + sort.Strings(keys) + parts := make([]string, 0, len(keys)) + for _, k := range keys { + parts = append(parts, fmt.Sprintf("%s=%v", k, o.DataSourceItem[k])) + } + return strings.Join(parts, " ") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/evaluator.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/evaluator.go index 4832d52c42d..f31584fd48a 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/evaluator.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/evaluator.go @@ -16,7 +16,7 @@ import ( // stripped before the name is sent as testing_criteria[].evaluator_name. const BuiltinPrefix = "builtin." -// EvaluatorRef references an evaluator from an eval group. It accepts either a +// EvaluatorRef references an evaluator from an eval. It accepts either a // bare string or a mapping carrying a pass threshold: // // evaluators: diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go index 2a12cb6bfab..bfa923167c3 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go @@ -20,7 +20,7 @@ import ( type EvalConfig struct { Evaluators []EvaluatorDecl `yaml:"evaluators,omitempty" json:"evaluators,omitempty"` Datasets []DatasetDecl `yaml:"datasets,omitempty" json:"datasets,omitempty"` - EvalGroups []EvalGroup `yaml:"evalGroups,omitempty" json:"evalGroups,omitempty"` + Evals []Eval `yaml:"evals,omitempty" json:"evals,omitempty"` } // DatasetDecl declares a dataset. A local Source is uploaded on deploy; without @@ -32,15 +32,15 @@ type DatasetDecl struct { } // EvaluatorDecl declares a custom evaluator. Built-ins are referenced directly -// from an eval group and never declared here. +// from an eval and never declared here. type EvaluatorDecl struct { Name string `yaml:"name" json:"name"` Source string `yaml:"source,omitempty" json:"source,omitempty"` Version string `yaml:"version,omitempty" json:"version,omitempty"` } -// EvalGroup is a run definition: evaluators plus options, bound to a dataset. -type EvalGroup struct { +// Eval is a run definition: evaluators plus options, bound to a dataset. +type Eval struct { Name string `yaml:"name" json:"name"` ID string `yaml:"id,omitempty" json:"id,omitempty"` Description string `yaml:"description,omitempty" json:"description,omitempty"` @@ -113,7 +113,7 @@ func (c *EvalConfig) Validate() error { if strings.HasPrefix(e.Name, evalcore.BuiltinPrefix) { return fmt.Errorf( "evaluators[%d]: built-in evaluator %q must not be declared; "+ - "reference it directly from an eval group", i, e.Name) + "reference it directly from an eval", i, e.Name) } if evaluators[e.Name] { return fmt.Errorf("evaluators[%d]: duplicate evaluator name %q", i, e.Name) @@ -133,22 +133,22 @@ func (c *EvalConfig) Validate() error { } groups := map[string]bool{} - for i, g := range c.EvalGroups { + for i, g := range c.Evals { if g.Name == "" { - return fmt.Errorf("evalGroups[%d]: 'name' is required", i) + return fmt.Errorf("evals[%d]: 'name' is required", i) } if groups[g.Name] { - return fmt.Errorf("evalGroups[%d]: duplicate eval group name %q", i, g.Name) + return fmt.Errorf("evals[%d]: duplicate eval name %q", i, g.Name) } groups[g.Name] = true if g.Dataset != "" && !datasets[g.Dataset] { return fmt.Errorf( - "evalGroups[%d] (%s): dataset %q is not declared in datasets", + "evals[%d] (%s): dataset %q is not declared in datasets", i, g.Name, g.Dataset) } if len(g.Evaluators) == 0 { - return fmt.Errorf("evalGroups[%d] (%s): at least one evaluator is required", i, g.Name) + return fmt.Errorf("evals[%d] (%s): at least one evaluator is required", i, g.Name) } for _, ref := range g.Evaluators { if ref.IsBuiltin() { @@ -156,7 +156,7 @@ func (c *EvalConfig) Validate() error { } if !evaluators[ref.Name] { return fmt.Errorf( - "evalGroups[%d] (%s): evaluator %q is not declared in evaluators "+ + "evals[%d] (%s): evaluator %q is not declared in evaluators "+ "(built-ins need the %q prefix)", i, g.Name, ref.Name, evalcore.BuiltinPrefix) } @@ -164,7 +164,7 @@ func (c *EvalConfig) Validate() error { if g.Target != nil && g.Target.Type != "" && g.Target.Type != TargetTypeAgent && g.Target.Type != TargetTypeModel { return fmt.Errorf( - "evalGroups[%d] (%s): target.type %q is not supported; use %q or %q", + "evals[%d] (%s): target.type %q is not supported; use %q or %q", i, g.Name, g.Target.Type, TargetTypeAgent, TargetTypeModel) } if g.Options != nil { @@ -172,7 +172,7 @@ func (c *EvalConfig) Validate() error { case "", EvaluationLevelTurn, EvaluationLevelConversation: default: return fmt.Errorf( - "evalGroups[%d] (%s): evaluation_level %q is invalid; expected %q or %q", + "evals[%d] (%s): evaluation_level %q is invalid; expected %q or %q", i, g.Name, g.Options.EvaluationLevel, EvaluationLevelTurn, EvaluationLevelConversation) } @@ -202,11 +202,11 @@ func (c *EvalConfig) Evaluator(name string) (*EvaluatorDecl, bool) { return nil, false } -// Group returns the eval group with the given name. -func (c *EvalConfig) Group(name string) (*EvalGroup, bool) { - for i := range c.EvalGroups { - if c.EvalGroups[i].Name == name { - return &c.EvalGroups[i], true +// Group returns the eval with the given name. +func (c *EvalConfig) Group(name string) (*Eval, bool) { + for i := range c.Evals { + if c.Evals[i].Name == name { + return &c.Evals[i], true } } return nil, false @@ -214,26 +214,26 @@ func (c *EvalConfig) Group(name string) (*EvalGroup, bool) { // ResolveGroup picks the group to act on: the named one, or the only one when // the config declares exactly one. -func (c *EvalConfig) ResolveGroup(name string) (*EvalGroup, error) { +func (c *EvalConfig) ResolveGroup(name string) (*Eval, error) { if name != "" { g, ok := c.Group(name) if !ok { - return nil, fmt.Errorf("eval group %q is not declared in the config", name) + return nil, fmt.Errorf("eval %q is not declared in the config", name) } return g, nil } - switch len(c.EvalGroups) { + switch len(c.Evals) { case 0: - return nil, fmt.Errorf("no eval groups are declared in the config") + return nil, fmt.Errorf("no evals are declared in the config") case 1: - return &c.EvalGroups[0], nil + return &c.Evals[0], nil default: - names := make([]string, 0, len(c.EvalGroups)) - for _, g := range c.EvalGroups { + names := make([]string, 0, len(c.Evals)) + for _, g := range c.Evals { names = append(names, g.Name) } return nil, fmt.Errorf( - "the config declares %d eval groups (%s); choose one with --eval-group", - len(c.EvalGroups), strings.Join(names, ", ")) + "the config declares %d evals (%s); choose one with --eval", + len(c.Evals), strings.Join(names, ", ")) } } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_test.go index 177638d5507..76a5a0cc96b 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_test.go @@ -26,7 +26,7 @@ datasets: source: ./datasets/support-golden.jsonl version: "1" -evalGroups: +evals: - name: pr-gate description: Quality gate for the support agent dataset: support-golden @@ -57,7 +57,7 @@ func TestLoadEvalConfig_ParsesAllSections(t *testing.T) { require.Len(t, cfg.Evaluators, 2) require.Len(t, cfg.Datasets, 1) - require.Len(t, cfg.EvalGroups, 1) + require.Len(t, cfg.Evals, 1) ds, ok := cfg.Dataset("support-golden") require.True(t, ok) @@ -127,13 +127,13 @@ func TestValidate_Rejects(t *testing.T) { }{ { name: "dataset referenced but not declared", - body: "evalGroups:\n - name: g\n dataset: missing\n evaluators: [builtin.relevance]\n", + body: "evals:\n - name: g\n dataset: missing\n evaluators: [builtin.relevance]\n", wantErr: "is not declared in datasets", }, { name: "custom evaluator referenced but not declared", body: "datasets:\n - name: d\n" + - "evalGroups:\n - name: g\n dataset: d\n evaluators: [not-declared]\n", + "evals:\n - name: g\n dataset: d\n evaluators: [not-declared]\n", wantErr: "is not declared in evaluators", }, { @@ -143,17 +143,17 @@ func TestValidate_Rejects(t *testing.T) { }, { name: "group without evaluators", - body: "evalGroups:\n - name: g\n evaluators: []\n", + body: "evals:\n - name: g\n evaluators: []\n", wantErr: "at least one evaluator is required", }, { name: "unsupported target type", - body: "evalGroups:\n - name: g\n evaluators: [builtin.relevance]\n target:\n type: prompt\n", + body: "evals:\n - name: g\n evaluators: [builtin.relevance]\n target:\n type: prompt\n", wantErr: "is not supported", }, { name: "invalid evaluation level", - body: "evalGroups:\n - name: g\n evaluators: [builtin.relevance]\n" + + body: "evals:\n - name: g\n evaluators: [builtin.relevance]\n" + " options:\n evaluation_level: sentence\n", wantErr: "evaluation_level", }, @@ -195,16 +195,16 @@ func TestResolveGroup(t *testing.T) { t.Run("ambiguous without a name", func(t *testing.T) { multi := loadFromString(t, - "evalGroups:\n - name: pr-gate\n evaluators: [builtin.relevance]\n"+ + "evals:\n - name: pr-gate\n evaluators: [builtin.relevance]\n"+ " - name: nightly\n evaluators: [builtin.relevance]\n") _, err := multi.ResolveGroup("") - require.ErrorContains(t, err, "--eval-group") + require.ErrorContains(t, err, "--eval") require.ErrorContains(t, err, "nightly") }) t.Run("empty config", func(t *testing.T) { _, err := (&EvalConfig{}).ResolveGroup("") - require.ErrorContains(t, err, "no eval groups") + require.ErrorContains(t, err, "no evals") }) } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/merge_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/merge_test.go index b253c8ad1f2..56f159102dd 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/merge_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/merge_test.go @@ -22,7 +22,7 @@ datasets: source: ./datasets/old.jsonl version: "3" -evalGroups: +evals: - name: pr-gate dataset: support-golden evaluators: @@ -68,8 +68,8 @@ func TestMergeArtifactRefs_PreservesCommentsAndSiblings(t *testing.T) { require.Len(t, cfg.Evaluators, 2) } -// The eval group's evaluator list must be left exactly as written. -func TestMergeArtifactRefs_DoesNotTouchEvalGroups(t *testing.T) { +// The eval's evaluator list must be left exactly as written. +func TestMergeArtifactRefs_DoesNotTouchEvals(t *testing.T) { path := writeTemp(t, handAuthored) require.NoError(t, MergeArtifactRefs(path, nil, []ArtifactRef{{Name: "support-quality", Source: "./evaluators/q.json"}})) @@ -85,7 +85,7 @@ func TestMergeArtifactRefs_DoesNotTouchEvalGroups(t *testing.T) { // Sections absent from the file are created rather than erroring. func TestMergeArtifactRefs_CreatesMissingSections(t *testing.T) { - path := writeTemp(t, "evalGroups:\n - name: pr-gate\n evaluators: [builtin.relevance]\n") + path := writeTemp(t, "evals:\n - name: pr-gate\n evaluators: [builtin.relevance]\n") require.NoError(t, MergeArtifactRefs(path, []ArtifactRef{{Name: "d1", Source: "./datasets/d1.jsonl"}}, diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go index 39e6f8ed9fa..4fb967c30a8 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go @@ -23,7 +23,7 @@ const EvalHost = "azure.ai.eval" // azd environment keys owned by this extension. const ( - EnvKeyEvalGroupID = "EVAL_GROUP_ID" + EnvKeyEvalID = "EVAL_ID" EnvKeyDatasetVersion = "EVAL_DATASET_VERSION" EnvKeyFingerprintPrefix = "EVAL_FINGERPRINT_" ) @@ -37,11 +37,11 @@ type Reconciler interface { // EnsureEvaluator registers a new evaluator version when the definition // differs from what the service already holds. EnsureEvaluator(ctx context.Context, decl EvaluatorDecl, localPath string) (version string, changed bool, err error) - // EnsureEvalGroup creates the group when it is absent or its resolved + // EnsureEval creates the group when it is absent or its resolved // evaluators or options changed, returning its id. datasetPath is the local // dataset backing the group, or empty when it is already registered; it lets // the reconciler bind criteria to the columns that actually exist. - EnsureEvalGroup(ctx context.Context, group EvalGroup, datasetPath string, recreate bool) (id string, err error) + EnsureEval(ctx context.Context, group Eval, datasetPath string, recreate bool) (id string, err error) } // EvalServiceTargetProvider deploys eval resources during `azd up`. azd owns @@ -120,7 +120,7 @@ func (p *EvalServiceTargetProvider) Publish( } // Deploy reconciles the eval configuration in a fixed order — datasets, then -// evaluators, then eval groups — because a group references the versions the +// evaluators, then evals — because a group references the versions the // first two resolve to. It fails fast; the next `azd up` resumes from wherever // it stopped. func (p *EvalServiceTargetProvider) Deploy( @@ -170,19 +170,19 @@ func (p *EvalServiceTargetProvider) Deploy( report(progress, describeResult("evaluator", decl.Name, version, changed)) } - // 3. Eval groups. Groups are immutable, so a change upstream means a new + // 3. Evals. Groups are immutable, so a change upstream means a new // group must be created and the stored id replaced. - for _, group := range cfg.EvalGroups { - report(progress, fmt.Sprintf("Reconciling eval group %s", group.Name)) + for _, group := range cfg.Evals { + report(progress, fmt.Sprintf("Reconciling eval %s", group.Name)) datasetPath := "" if decl, ok := cfg.Dataset(group.Dataset); ok { datasetPath = resolveSource(baseDir, decl.Source) } - id, err := reconciler.EnsureEvalGroup(ctx, group, datasetPath, anyChanged) + id, err := reconciler.EnsureEval(ctx, group, datasetPath, anyChanged) if err != nil { - return nil, fmt.Errorf("eval group %q: %w", group.Name, err) + return nil, fmt.Errorf("eval %q: %w", group.Name, err) } - report(progress, fmt.Sprintf("Eval group %s is %s", group.Name, id)) + report(progress, fmt.Sprintf("Eval %s is %s", group.Name, id)) } return &azdext.ServiceDeployResult{}, nil @@ -229,7 +229,7 @@ func EvalConfigFromService(svc *azdext.ServiceConfig, projectRoot string) (*Eval props := serviceProps(svc) if props == nil || len(props.GetFields()) == 0 { return nil, fmt.Errorf( - "service %q carries no eval configuration; expected evaluators, datasets, or evalGroups", + "service %q carries no eval configuration; expected evaluators, datasets, or evals", svc.GetName()) } @@ -313,14 +313,14 @@ func Fingerprint(path string) (string, error) { return hex.EncodeToString(sum[:]), nil } -// FingerprintGroup hashes an eval group's own declaration. +// FingerprintGroup hashes an eval's own declaration. // // Change detection on upstream artifacts is not sufficient: editing a group's // evaluators, target, or options changes what the group means, and groups are // immutable, so the group has to be recreated even when the dataset and // evaluators are untouched. Without this a retargeted group keeps running // against the old definition. -func FingerprintGroup(group EvalGroup) (string, error) { +func FingerprintGroup(group Eval) (string, error) { // The id is server-assigned. The description is carried in the group's // metadata, so editing it does change the request, but recreating an // immutable group over a reworded description would cost the group id and @@ -332,7 +332,7 @@ func FingerprintGroup(group EvalGroup) (string, error) { data, err := json.Marshal(group) if err != nil { - return "", fmt.Errorf("hashing eval group %q: %w", group.Name, err) + return "", fmt.Errorf("hashing eval %q: %w", group.Name, err) } sum := sha256.Sum256(data) return hex.EncodeToString(sum[:]), nil diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval_test.go index 6052cb99c55..bd24e6c0030 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval_test.go @@ -73,7 +73,7 @@ func TestEvalConfigFromServiceReadsInlineConfig(t *testing.T) { "datasets": []any{ map[string]any{"name": "golden", "source": "./datasets/golden.jsonl"}, }, - "evalGroups": []any{ + "evals": []any{ map[string]any{ "name": "quality", "dataset": "golden", @@ -88,9 +88,9 @@ func TestEvalConfigFromServiceReadsInlineConfig(t *testing.T) { require.NoError(t, err) require.Len(t, cfg.Datasets, 1) require.Equal(t, "golden", cfg.Datasets[0].Name) - require.Len(t, cfg.EvalGroups, 1) - require.Len(t, cfg.EvalGroups[0].Evaluators, 1) - require.Equal(t, "builtin.task_adherence", cfg.EvalGroups[0].Evaluators[0].Name) + require.Len(t, cfg.Evals, 1) + require.Len(t, cfg.Evals[0].Evaluators, 1) + require.Equal(t, "builtin.task_adherence", cfg.Evals[0].Evaluators[0].Name) } func TestEvalConfigFromServiceRejectsEmptyService(t *testing.T) { @@ -103,7 +103,7 @@ func TestEvalConfigFromServiceRejectsEmptyService(t *testing.T) { // detectable. Upstream artifact fingerprints do not cover it: retargeting a // group at a different agent leaves the dataset and evaluators untouched. func TestFingerprintGroupTracksMeaningfulChanges(t *testing.T) { - base := EvalGroup{ + base := Eval{ Name: "quality", Dataset: "golden", Evaluators: evalcore.EvaluatorList{{Name: "builtin.task_adherence"}}, @@ -118,13 +118,13 @@ func TestFingerprintGroupTracksMeaningfulChanges(t *testing.T) { require.NoError(t, err) require.Equal(t, original, same, "an unchanged group must keep its fingerprint") - cases := map[string]func(g *EvalGroup){ - "target": func(g *EvalGroup) { g.Target = &Target{Type: "agent", Name: "agent-b"} }, - "evaluators": func(g *EvalGroup) { + cases := map[string]func(g *Eval){ + "target": func(g *Eval) { g.Target = &Target{Type: "agent", Name: "agent-b"} }, + "evaluators": func(g *Eval) { g.Evaluators = append(g.Evaluators, evalcore.EvaluatorRef{Name: "builtin.similarity"}) }, - "options": func(g *EvalGroup) { g.Options = &Options{EvalModel: "gpt-4o-mini"} }, - "dataset": func(g *EvalGroup) { g.Dataset = "other" }, + "options": func(g *Eval) { g.Options = &Options{EvalModel: "gpt-4o-mini"} }, + "dataset": func(g *Eval) { g.Dataset = "other" }, } for name, mutate := range cases { t.Run(name, func(t *testing.T) { @@ -141,7 +141,7 @@ func TestFingerprintGroupTracksMeaningfulChanges(t *testing.T) { // Server-assigned and cosmetic fields must not force a recreate. func TestFingerprintGroupIgnoresIdAndDescription(t *testing.T) { - base := EvalGroup{ + base := Eval{ Name: "quality", Dataset: "golden", Evaluators: evalcore.EvaluatorList{{Name: "builtin.task_adherence"}}, From 69335b54c598f9982e24628f4d8c9935f45a96f8 Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 28 Jul 2026 22:23:55 +0300 Subject: [PATCH 059/320] Drop agent.context.tools, and the dataset update synonym Nothing read agent.context.tools. Tool definitions reach an evaluator from the live agent as {{sample.tool_definitions}}, which is what the tool evaluators score, and the agents extension's own eval and eval-generate paths do not read a local tools file either - only optimize does, and optimize stays there. The generation API has no field for tools, so the only way to pass them would have been prose folded into the prompt. A key with no consumer is worse than no key, so it is gone rather than warned about. dataset update goes the same way evaluator update did: both published a new immutable version, so update was a synonym for create. --- .../azd-eval-extension-implementation.md | 534 ++++++++++++++++++ .../internal/cmd/agent_context_test.go | 24 +- .../internal/cmd/dataset.go | 18 +- .../internal/cmd/generate.go | 6 +- .../internal/project/generate_config.go | 1 - .../tests/live/live_test.go | 12 +- 6 files changed, 561 insertions(+), 34 deletions(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/azd-eval-extension-implementation.md diff --git a/cli/azd/extensions/azure.ai.evaluations/azd-eval-extension-implementation.md b/cli/azd/extensions/azure.ai.evaluations/azd-eval-extension-implementation.md new file mode 100644 index 00000000000..44fb4ae170b --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/azd-eval-extension-implementation.md @@ -0,0 +1,534 @@ +# `azd ai eval` — Implementation Handoff + +> Self-contained brief for building the **`azure.ai.evaluations`** azd extension (`azd ai eval`). Everything below is verified against the shipping `azure.ai.agents` extension, azd core, and RAISvc source. Design source of truth is `spec.md` in `foundrysdk_specs/specs/evaluations/azd_eval_extension/` — if the two disagree, the spec wins. +> +> **Uncommitted working document.** Not part of any PR. + +--- + +## 0. TL;DR + +A new azd extension — id `azure.ai.evaluations`, namespace `azd ai eval` — that is a thin Go client over the **existing** Foundry evaluations data plane, plus **one azd service-target provider** for `host: azure.ai.evals`. + +**Non-negotiables** + +1. **Two-tier commands.** Atomic (`dataset` / `evaluator` / `run` / `results`) map ~1:1 to the API. Composite (`init` / `generate` / `run`) are wrappers, never the only path. +2. **No `deploy` command.** Deployment is `azd up` / `azd deploy` invoking our service-target provider. We ship no deploy verb. +3. **`generate` is separate from deploy.** Generate once, deploy the artifacts to many environments. +4. **`init` touches no network.** +5. **Everything is built on APIs that exist today.** No service changes. +6. **Deterministic.** `-o json` + `--no-prompt` everywhere; a supplied flag fully suppresses its prompt. + +**Out of scope for M1:** scheduled/continuous eval, baseline comparison (both exist server-side — M2), non-agent targets, traces as a run data source, eval by response/run id, `optimize` (stays in the agents extension). + +--- + +## 1. Where the code lives & how to build it + +### 1.1 Paths +- **Repo:** `Azure/azure-dev`. +- **New extension:** `cli/azd/extensions/azure.ai.evaluations/` +- **Reference to copy:** `cli/azd/extensions/azure.ai.agents/` — closest analog, currently hosts `azd ai agent eval …`. **Read it first.** + +### 1.2 File layout (mirror the agents extension) +``` +cli/azd/extensions/azure.ai.evaluations/ +├── internal/ +│ ├── cmd/ # cobra commands, one file per group; listen.go wires the provider +│ ├── pkg/eval_api/ # data-plane client (lifted, see §1.4) +│ ├── pkg/dataset_api/ # dataset client (lifted) +│ └── project/ # azure.yaml service-entry model + YAML round-trip +├── schemas/ # JSON schemas +├── tests/ +├── extension.yaml # manifest +├── go.mod / go.sum +├── main.go +└── version.txt +``` + +**`extension.yaml`** — note the `service-target-provider` capability and `providers` block; both are required for `azd up` to route to us: +```yaml +# yaml-language-server: $schema=../extension.schema.json +id: azure.ai.evaluations +namespace: ai.eval # dotted → CLI surface `azd ai eval` +displayName: Foundry evaluations (Beta) +description: Define and run Foundry evaluations from your terminal. (Beta) +usage: azd ai eval [options] +version: 1.0.0-beta.1 # keep version.txt in sync +requiredAzdVersion: ">=1.27.1" +language: go +capabilities: + - custom-commands + - lifecycle-events + - service-target-provider + - metadata +providers: + - name: azure.ai.evals + type: service-target + description: Deploys evaluation datasets, evaluators, and eval groups to Foundry +``` + +### 1.3 Stack facts (verified) +- **Go 1.26.x**, **cobra**. Entry point: + ```go + package main + import ( + "azureaieval/internal/cmd" + "github.com/azure/azure-dev/cli/azd/pkg/azdext" + ) + func main() { azdext.Run(cmd.NewRootCommand()) } + ``` +- SDK module `github.com/azure/azure-dev/cli/azd` (agents pins `v1.28.0`); surface is `pkg/azdext`. +- Data plane over REST using the **azcore pipeline**, not raw `net/http`: bearer-token policy scoped to **`https://ai.azure.com/.default`**, plus `azsdk.NewMsCorrelationPolicy()` and `azsdk.NewUserAgentPolicy(...)`. +- Dev loop: + ```bash + azd ext install microsoft.azd.extensions # one-time + cd cli/azd/extensions/azure.ai.evaluations + azd x build # build + install locally + azd x watch # ongoing + ``` + +### 1.4 What to lift from `azure.ai.agents` + +Measured, non-test: + +| Source | Files | LOC | Gives you | +|---|---|---|---| +| `internal/pkg/agents/eval_api/` | 7 | 1,350 | `EvalClient` over `/data_generation_jobs`, `/evaluator_generation_jobs`, `/evaluators`, `/datasets`, `/openai/v1/evals`; LRO poller; artifact download; portal URLs; api-version constants | +| `internal/pkg/agents/dataset_api/` | 2 | 550 | Full pending-upload → blob → finalize → download | +| `internal/pkg/agents/opt_eval/` | 2 | 555 | eval.yaml config model (adapt, don't copy wholesale) | +| `internal/cmd/eval_*.go` | 10 | 2,890 | generate / run / show / list / update / progress UX | + +**≈1,900 LOC of API client is effectively done.** Genuinely net-new: + +1. **The service-target provider** (§4) — no prior art. +2. **Change detection** (§5) — no prior art. +3. **YAML round-trip merge** for `generate` writing `source:` back (§6). +4. The atomic command layer and the offline `init`. + +> Do **not** port the `/evaluation_suites` client. That endpoint is abandoned — the eval group is the unit. + +--- + +## 2. Data-plane API contract + +### 2.1 Base, auth, api-versions +- **Base:** azd env `FOUNDRY_PROJECT_ENDPOINT`, shape `https://{resource}.services.ai.azure.com/api/projects/{project}/…`. Global `--project-endpoint` overrides. +- **Scope:** `https://ai.azure.com/.default`. +- **api-versions:** project-endpoint calls (datasets, evaluators) use **`2025-11-15-preview`**; data generation uses **`v1`**; **`/openai/v1/evals*` sends no api-version**. + +### 2.2 Datasets +| Command | Calls | +|---|---| +| `dataset create` / `update` | `POST /datasets` *(first version only)* → `POST /datasets/{name}/versions/{v}/startPendingUpload` → `PUT ` → `PUT /datasets/{name}/versions/{v}` | +| `dataset list` | `GET /datasets` | +| `dataset show` | `GET /datasets/{name}/versions/{v}` | +| `dataset delete` | `DELETE /datasets/{name}/versions/{v}` | + +Model — **note there is no content hash or etag**, which drives §5: +```go +type Dataset struct { Name, Version, BlobURI, Format, DataURI, ContentURI string } +``` +A dataset is a **single `.jsonl`**. A directory today just picks the first `.jsonl`; no folder walk. + +### 2.3 Evaluators +| Command | Calls | +|---|---| +| `evaluator upload` / `update` | *(code only)* pending-upload → blob upload; then `POST /evaluators/{name}/versions` | +| `evaluator show` | `GET /evaluators/{name}` — returns the definition inline | +| `evaluator builtins` | `GET /evaluators?type=Builtin` | + +Built-ins are referenced as `builtin.`; the prefix is stripped before the value goes into `testing_criteria[].evaluator_name`. Custom-evaluator upload needs the project MI to hold **Azure AI User**. + +### 2.4 Eval groups and runs (OpenAI-compatible, no api-version) +```go +type CreateOpenAIEvalRequest struct { + Name string + Metadata map[string]string + DataSourceConfig *DataSourceConfig // {Type, ItemSchema, IncludeSampleSchema} + TestingCriteria []TestingCriterion // the evaluators +} +type TestingCriterion struct { + Type, Name, EvaluatorName string + InitializationParameters map[string]any // threshold lives here + DataMapping map[string]string +} +type OpenAIEval struct { ID, Name string } // ID is canonical; Name is NOT unique +``` + +| Command | Calls | +|---|---| +| create group | `POST /openai/v1/evals` | +| get / list | `GET /openai/v1/evals/{id}` · `GET /openai/v1/evals?limit=` | +| start run | `POST /openai/v1/evals/{evalId}/runs` | +| poll / list runs | `GET /openai/v1/evals/{evalId}/runs/{runId}` · `GET …/runs` | +| cancel | `POST /openai/v1/evals/{evalId}/runs/{runId}/cancel` | +| results | `GET …/runs/{runId}` → `result_counts` + `per_testing_criteria_results` | + +**The group carries evaluators, not the dataset.** The dataset goes on the **run**. `evaluation_level` is `turn` | `conversation`, service default **`turn`**. + +**`data_source_config` and `data_mapping` are derived from each evaluator's published contract.** The original plan was to copy the agents extension's hardcoded mapping. Live testing showed that is wrong: it only suits agent-target quality evaluators and the service rejects the rest. + +`GET /evaluators` returns a contract per evaluator: +```jsonc +"supported_evaluation_levels": ["turn"], +"definition": { + "data_schema": { "required": ["response", "instruction_id_list"], "properties": { … } }, + "init_parameters": { "required": ["deployment_name"], "properties": { … } } +} +``` + +`internal/cmd/build.go` reads it and, per criterion: +- binds each accepted input to the agent sample (`response`, `tool_calls`, `tool_definitions`) or to a dataset column `{{item.}}`; +- declares the referenced columns in the item schema; +- filters `initialization_parameters` to the declared properties — no evaluator accepts `model`, and `builtin.ifeval` accepts nothing; +- validates `--level` against `supported_evaluation_levels`; +- reports a missing required column locally, naming it. + +Two service rules are encoded: `messages` and `query`/`response` are mutually exclusive (the level selects), and `evaluation_level` is an **initialization parameter**, not run metadata. An evaluator with no published contract falls back to the agent-target shape. + +Covered by `build_test.go`, and by `build_live_test.go` which posts a group for every built-in the project exposes. + +### 2.5 Generation (LRO) +`POST /data_generation_jobs` and `POST /evaluator_generation_jobs`, each polled by `GET …/{id}`. The ~11-minute "timeout" is a **client poll budget (2 s × 300)**, not a service limit — raise it and default to `--no-wait` in CI. + +### 2.6 M2 only — schedules and comparison +Both are **project-endpoint reachable** and **feature-gated per project**: +- `/schedules` — `PUT {id}` · `GET {id}` · `GET` · `DELETE {id}`; requires `FoundryFeature.Schedules_V1Preview`. Trigger is `Cron{Expression, StartTime, EndTime, Timezone}` or `Recurrence{Frequency, Interval, Schedule}`. +- Insights compare — `POST /insights` (async) or `POST /insights/sync`, body `{evalId, baselineRunId, treatmentRunIds}`; requires `FoundryFeature.Insights_V1Preview`. + +--- + +## 3. Configuration model + +Two files. Neither is loaded by azd core — **we parse both**. + +**`evals/eval_generate.yaml`** — input to `generate`, never deployed. `agent.context.{instructions,tools}` are file paths; `local_dir` accepts a directory or an explicit file path. + +**`evals/azure.yaml`** — the deployment spec, `$ref`'d from the root `azure.yaml`: +```yaml +# /azure.yaml +services: + evals: + host: azure.ai.evals + uses: [ai-project] + $ref: ./evals/azure.yaml +``` +It carries three arrays: `evaluators[]`, `datasets[]`, `evalGroups[]` (see `spec.md` for the full shape). + +**Why arrays on a service work.** azd core's `ServiceConfig` captures unknown keys: +```go +AdditionalProperties map[string]any `yaml:",inline"` +``` +and hands them to the extension, which unmarshals them itself — the pattern `LoadServiceTargetAgentConfig` → `ServiceConfigProps` uses. `azure.ai.project` already carries `deployments[]` this way. **`$ref` resolution is ours too**: `pkg/foundry.ResolveFileRefs(cfg, projectRoot)`, called by the extension, not by azd. + +--- + +## 4. The service-target provider (net-new, highest risk) + +Wire it in `listen.go`, mirroring `azure.ai.agents`: +```go +func configureExtensionHost(host *azdext.ExtensionHost) { + azdClient := host.Client() + host. + WithServiceTarget("azure.ai.evals", func() azdext.ServiceTargetProvider { + return project.NewEvalServiceTargetProvider(azdClient) + }). + WithServiceEventHandler("postdeploy", func(ctx context.Context, args *azdext.ServiceEventArgs) error { + return postdeployHandler(ctx, azdClient, args) + }, &azdext.ServiceEventOptions{Host: "azure.ai.evals"}) +} +``` + +`ServiceTargetProvider` requires `Initialize`, `Endpoints`, `GetTargetResource`, `Package`, `Publish`, `Deploy`. For eval, **`Package` and `Publish` are near no-ops**; `Deploy` does the work, in this fixed order: + +1. **Datasets** — change-detect (§5); if changed, run the `dataset create` sequence. +2. **Evaluators** — `GET /evaluators/{name}`, compare the definition, upload only if different. +3. **Drift check** — if the server's latest version is ahead of the recorded one, fail with "sync first". +4. **Eval groups** — `POST /openai/v1/evals` with `testing_criteria` from the resolved evaluator versions. Groups are immutable, so only recreate when the resolved versions or options actually changed. +5. Persist resolved ids, versions, and fingerprints to the azd env. + +**How azd reaches us:** `azd up` runs one DAG; per service it calls `GetServiceTarget()`, which does `serviceLocator.ResolveNamed(host, &target)`. If our extension is not installed, azd fails that service with *"install an extension that provides this host."* We implement **no sequencing or rollback across services** — `uses:` and the DAG handle that. + +--- + +## 5. Change detection (net-new) + +Without this, every `azd up` publishes a redundant version. + +- **Datasets** — the API returns no hash or etag, so comparing against the server would mean downloading the blob every deploy. Instead: **SHA-256 the local file**, store it with the resolved version in the azd env, re-hash locally next deploy, skip when unchanged. +- **Evaluators** — definitions come back inline from `GET /evaluators/{name}`; compare directly, no cache needed. +- **Drift** — a *version* comparison, not content: server latest vs. the version recorded at last deploy. + +**Open:** how to fingerprint a **code** evaluator (a folder). Suggest hashing sorted relative paths + contents, excluding `__pycache__` and `.pyc`. + +--- + +## 6. `generate` writes back into `evals/azure.yaml` + +After downloading artifacts, `generate` adds/updates `source:` references. Requirements: + +- Match entries **by `name`**; update `source` in place; append when absent. +- **Preserve comments and key order** — use the `yaml.v3` Node API, not plain marshal/unmarshal. +- Do not clobber a field the user hand-edited other than `source`. +- If the array is itself a `$ref`, write into the referenced file. + +There is no `emitDeploymentConfig` block — this is default behavior, not configurable. + +--- + +## 6a. What generation is seeded from + +The generation API takes an `agent` source that is meant to pull the agent's own instructions, and it fails for every agent (§11d). The client resolves that context itself, most specific first: + +1. `--gen-instruction` / `--gen-instruction-file` +2. the file named by `agent.context.instructions`, resolved **relative to the spec that declared it**, not the working directory +3. the agent's published instructions — `GET /agents/{name}` → `versions.latest.definition.instructions` + +Step 2 tolerates a missing file on purpose: `init` writes the path before the file exists, so treating the gap as an error would break the flow init scaffolds. Step 3 is what makes `init` → `generate` work with nothing authored. + +The agent source is still sent. When the service starts honouring it, it contributes on top of the prompt; nothing has to be removed. + +`agent.context.tools` is still read by nothing, so it is warned about rather than dropped silently, and `init` no longer scaffolds it — a warning for a field the user never chose is just noise. + +--- + +## 7. Behavioral bugs to fix (measured in `azd ai agent eval`) + +Treat each as an acceptance criterion. + +1. **Path handling (highest priority).** `--out-file` is re-rooted under the agent directory; `--config` re-roots again. **Fix:** treat paths as relative to CWD (or `-C/--cwd`), used verbatim, single-rooted. Test `./x.yaml`, `../x/x.yaml`, absolute. +2. **Wizard overrides flags.** Prompts still fire when flags are supplied, and pre-filled prompts *append* typed input. **Fix:** a supplied flag fully suppresses its prompt; `--no-prompt` errors on a missing required value. +3. **`--evaluator` ignored during generation.** Passing `--evaluator` does not stop rubric generation. **Fix:** honor it, skip that generation. +4. **Client-side generation timeout.** Resolved: it is a client poll budget, not a service limit. Raise it; default `--no-wait` under `--no-prompt`. +5. **Shallow results.** `eval show` returns counts only. **Fix:** per-sample scores via `per_testing_criteria_results`. +6. **Auth friction.** Native azd token failed with "Reauthentication required"; workaround `azd config set auth.useAzCliAuth true`. Detect and surface clearly. + +--- + +## 8. azd environment + +Extensions read and write env values themselves via `azdClient.Environment().GetValue / SetValue` — azd sets none of these. The agents extension does this from lifecycle handlers. + +| Key | Written by | +|---|---| +| `FOUNDRY_PROJECT_ENDPOINT` | consumed, not written | +| `EVAL_GROUP_ID` | provider during `azd up`; `run` when it creates the group | +| `EVAL_DATASET_VERSION`, artifact fingerprints | provider during `azd up` | +| `EVAL_RUN_ID` | `run` | + +Setting `EVAL_GROUP_ID` manually targets a pre-existing group; `--eval-id` does the same per-invocation. + +--- + +## 9. Build order + +M1 is everything in the spec. Within it, build in dependency order: + +| Step | Work | Done when | +|---|---|---| +| **1. Scaffold** | Extension skeleton, `extension.yaml`, `main.go`, root cobra command, local install via `azd x build` | `azd ai eval --help` works | +| **2. Lift the clients** | Copy `eval_api` + `dataset_api`, de-agent-scope, keep api-version constants | Unit tests pass against an `httptest` fake | +| **3. Atomic commands** | `dataset`, `evaluator`, `run`, `results` with `-o json` / `--no-prompt` | **E2E-1** below | +| **4. Config model** | `evals/azure.yaml` load, `$ref` resolve via `pkg/foundry.ResolveFileRefs`, validation | Round-trip test preserves comments | +| **5. Service-target provider** | `listen.go` wiring + `Deploy` reconciliation + change detection + drift | `azd up` creates all three resource kinds; second `azd up` is a no-op | +| **6. `init`** | Offline scaffold of both YAMLs | Runs with no network/auth | +| **7. `generate`** | Generation LROs, artifact download, write-back into `evals/azure.yaml` | **E2E-3** | +| **8. `run`** | Group resolve-or-create, run, poll, render | **E2E-2** | + +Steps 1–3 are mostly mechanical. **Step 5 is the risk** — budget accordingly. + +--- + +## 10. Testing + +| Tier | Coverage | Auth | Where | +|---|---|---|---| +| **0 — offline** | flag parsing; YAML round-trip; path resolution (§7.1); flag→prompt suppression (§7.2); request bodies against an `httptest` fake (copy `eval_api_version_test.go`); schema validation | No | PR gate | +| **1 — `init` record/playback** | interactive prompt flows | No | PR gate | +| **2 — live golden path** | full flows against a real Foundry project | Yes | On-demand/scheduled, **not** the PR gate | + +Tier 2: env-gate on `AZURE_AI_EVAL_E2E_LIVE=1`, build tag `//go:build linux` (needs a PTY), drive `init` prompts via `go-expect`+`vt10x`+`creack/pty`, everything else through `--no-prompt -o json`. `t.Cleanup` must delete every version it created. + +**Golden paths** + +- **E2E-1 — atomic:** `dataset create` → `evaluator upload` → `run start` → `results export`. Assert valid JSON, resolved versions, terminal run status, **per-sample** scores, and that re-running `create` yields the *next* version rather than an error. +- **E2E-2 — init → azd up → run:** `init` makes **zero network calls** (run it unauthenticated), writes both YAMLs, does not double the path; `azd up` creates the resources and pins versions back; a second `azd up` creates **no new versions**; `run` completes with no prompt. +- **E2E-3 — generate:** completes without a client timeout; a supplied `--evaluator` is honored; artifacts land locally and `evals/azure.yaml` gains correct `source:` entries with comments preserved. +- **E2E-4 — CI invariants:** every command with `--no-prompt -o json` is non-interactive, emits parseable JSON, and exits non-zero on a missing required value. + +--- + +## 11. Decisions still open + +| # | Question | Blocks | Suggested default | +|---|---|---|---| +| 1 | Host name `azure.ai.evals` — agreed? Not registered anywhere yet | Step 1 | Use it; renaming is cheap before publish | +| 2 | Which storage connection for evaluator pending-upload (`connectionName`) | Step 3 | Project default; expose a flag | +| 3 | Are **code** evaluators in M1, or rubric-only? | Steps 3, 5 | Rubric-only for M1 — removes the folder-hashing problem entirely | +| 4 | Where do fingerprints live — azd env or a lock file? | Step 5 | azd env, so they are environment-scoped | +| 5 | Do we depend on `azure.ai.projects` for the project service? | Step 1 | Yes, mirror the agents manifest | +| 6 | Bundling into `microsoft.foundry` — who owns it | Ship | Extensions team | + +--- + +## 11b. Assumptions made while implementing + +Recorded for review. Anything marked **corrected** was an assumption that live testing disproved; the code already reflects the correction. + +| # | Assumption | Status | +|---|---|---| +| 1 | Dataset versions are decimal (`1.0`, `2.0`) | **Verified live** — `UploadNewVersion` advanced 1.0 → 2.0 | +| 2 | `--wait` defaults true for `run` | Held; matches the spec's blocking-by-default UX | +| 3 | `evaluation_level` travels as run **metadata** | **Corrected** — it is an `initialization_parameters` property on evaluators that declare it. Metadata had no effect | +| 4 | A cached eval group id that 404s means recreate | Held; not yet exercised live | +| 5 | `dataset create` accepts a file or a directory | Held; the upload helper scans a directory for the first `.jsonl` | +| 6 | Evaluator sameness compares only the `definition` block | Held; avoids server-assigned version/timestamp churn | +| 7 | `GetTargetResource` returns a subscription-only resource | Held; eval resources have no ARM resource | +| 8 | Rubric evaluators only in M1; code evaluators in M2 | Open decision 3 | +| 9 | One fixed data mapping suits all evaluators | **Corrected** — contracts differ per evaluator; the mapping is now derived from the published contract | +| 10 | Dataset URIs come back snake_case | **Corrected** — the project endpoint returns camelCase (`dataUri`). Both spellings are now bound | +| 11 | A dataset blob URI can be downloaded directly | **Corrected** — true only for uploads. A *generated* dataset's URI names the container, not the blob, with `isSingleFile` true either way, and downloading a container returns 409. The URI also carries no SAS, so a credential is always needed. Downloads now fetch a credential and list the container when the URI does not name a file | +| 22 | Agent-seeded data generation would be fixed service-side before ship | **Corrected** — traced to the AOAI generator, outside this repo. `generate` now reads the agent's instructions itself and passes them as the prompt source; the agent source is still sent so it contributes once fixed | +| 23 | `agent.context.instructions` and `.tools` were wired up | **Corrected** — both were written by `init`, declared on the config, and read by nothing. `instructions` is now honoured; `tools` is warned about and no longer scaffolded | +| 24 | The service would reject a missing generation model clearly | **Corrected** — it fails partway through the command with a message naming nothing the caller controls. Checked up front instead | +| 25 | `max_samples` was free-form | **Corrected** — the service requires 15–1000. The config already validated this; the floor is now documented in the spec | +| 26 | Schedule creation would be a POST to a collection | **Corrected** — `POST` 404s on every route. It is `PUT /schedules/{name}`, a named resource | +| 27 | The bodiless 400s meant `displayName`/`description`/`enabled` were required | **Corrected, and this one was my error** — in that probe only the *first* create succeeded and I read the rest as field validation. The real cause is one schedule per project. Re-tested from a drained state, a minimal body creates fine | +| 28 | A named PUT would update in place | **Corrected** — accepted, echoes the new body, changes nothing. `set` refuses an existing name instead of reporting a change that did not happen | +| 29 | Deleting and recreating under the same name would work as a replace | **Corrected** — the replacement never leaves `Creating` and cannot then be deleted. The `--replace` flag was removed before shipping | +| 30 | M4's "traces as a run data source" was awaiting service support | **Corrected** — `azure_ai_traces` is in the run data-source discriminator and the service executes it. The note was never re-tested. Shipped as `run --from-traces` | +| 31 | Traces were a generation input only, never a run's data source | **Corrected** — that comment described the *generation* API. The run API takes them directly | +| 32 | The traces window could be sent as `start_time`/`end_time` | **Corrected, and this one I shipped** — the data source has no start bound. `start_time` is accepted and discarded, leaving the default 7 days. It looked right only because the first value I tested, 7d, *is* the default; 30d silently queried a week. Now sends `lookback_hours` | +| 33 | M4's "evaluation by response id" was awaiting service support | **Corrected** — works today. The ids are not a list on the data source: they are JSONL rows plus a `data_mapping` to `response_id` | +| 34 | `target.type: model` was unsupported | **Corrected** — supported. The config rejected it by name *and* the test used it as the example of an unsupported type, so the gap read as deliberate in two places. Sample bindings now follow the target kind, since a model returns `output_text` where an agent returns `output_items` | +| 35 | A run could reference a registered dataset by name as a `file_id` | **Corrected** — `file_id` means an uploaded file; a dataset name is rejected with `invalid data source file ids`. Registered datasets are fetched and sent inline. Every earlier test used a local `source:`, so this path had never run | +| 36 | M4's "subsetting a registered dataset" needed service support | **Corrected** — the service cannot narrow a file reference, but fetching the rows client-side makes `--max-samples` mean the same thing for any dataset | +| 37 | One env key per resolved id was enough | **Corrected** — only true for a single-group config. With two, the second deploy handed the first group the second's id and both declarations pointed at one group. Ids are now keyed by name, as fingerprints already were | +| 38 | The remembered run id could be shared | **Corrected** — same shape as 37. Asking group A for its latest fetched group B's run inside A and 404'd | +| 39 | A dataset's `version:` was the version published | **Corrected** — it was passed to the helper that *counts from* its argument, so `1.0` published 2.0. It also meant two things: unchanged content resolved to it, changed content published above it | +| 40 | An evaluator's `version:` behaved like a dataset's | **Corrected** — the service assigns an evaluator's version on publish, so a pin alongside `source:` was never honoured. A config asking for 7 deployed 1 silently. Now refused | +| 41 | Criteria were being shaped from each built-in's published schema | **Corrected, and this one invalidated an earlier §11c row** — the schemas were fetched with an unfiltered list, which returns only the project's own evaluators. Every built-in fell back to `legacyInputs`. It matched query/response so nothing looked wrong; `task_completion` at conversation level published an empty `data_mapping` | +| 42 | A run needs a target | **Corrected** — a dataset holding both sides of the exchange has nothing to invoke, and the service runs it. The requirement was ours | +| 12 | `$ref` is resolved by azd core before the extension sees the config | **Corrected** — core leaves `$ref` for the owning extension. The provider now calls `foundry.ResolveFileRefs`, and relative `source:` paths are based on the included file's directory | +| 13 | Upstream artifact fingerprints are enough to know when to recreate a group | **Corrected** — editing the group's own target/evaluators/options changed nothing. The group declaration is fingerprinted too | +| 14 | The host is `azure.ai.evals` | **Corrected** — it is `azure.ai.eval`; the spec has been aligned | +| 15 | `run` only needed the composite form | **Corrected** — the spec lists `start`/`list`/`show`/`cancel`, and M1 requires every operation to be reachable atomically. All four now exist | +| 16 | `--project-endpoint` only selects the endpoint | **Corrected** — it also suppressed the azd environment name, silently disabling the cached eval-group and run ids. The name is now resolved independently | +| 17 | No evaluator accepts `model` | **Corrected** — true for built-ins, false for custom rubrics, which *require* `model`. The judge model is bound under whichever name the evaluator declares | +| 18 | An evaluator definition can be compared whole to detect changes | **Corrected** — the service enriches it on create, so only the authored keys can be compared | +| 19 | `GET /evaluators/{name}` returns the latest version | **Corrected** — it 404s; the version has to be resolved first, numerically | +| 20 | Rubric weights are free-form | **Corrected** — integers 1–10; the spec now says so | +| 21 | `--dataset` suppresses data generation | **Corrected** — only did so for a local path, not for a registered dataset name, which the flag also accepts | + +--- + +## 11c. Verified end to end against a live project + +| Flow | Result | +|---|---| +| `azd ai eval init` | Scaffolds `evals/azure.yaml` + `evals/eval_generate.yaml` matching the spec | +| `azd provision` → `azd deploy evals` → `azd up` | Provider runs; datasets and groups reconcile | +| Dataset first deploy | Published at version 1.0 | +| Dataset unchanged | Reported unchanged, nothing uploaded | +| Dataset edited | Published 2.0 and the group recreated | +| Group retargeted | New group id; two further no-op deploys reused it | +| `$ref` service entry | Deploys, and the fingerprint matches the equivalent inline config | +| `azd ai eval run` | Real run against a live agent, completed | +| `azd ai eval results show` | 3 passed / 1 failed, per-criterion breakdown, portal link | +| `evaluator builtins` | 10 built-ins with versions and type | +| Eval group create | Accepted for **all 10** built-ins, each with its own contract || `generate` rubric | Succeeds, writes the evaluator JSON | +| Build → pack → publish → install | Installs from the local registry; `azd ai eval --help` lists every command | +| Atomic surface | Every command group and subcommand the spec lists is present | +| `run start` / `list` / `show` / `cancel` | Exercised live, including the guard that refuses to cancel a finished run | +| `results export` | JSON and CSV both written | +| `-o json` | Valid JSON from every read command | +| `dataset` create/show/update/list/delete | Full lifecycle, 1.0 → 2.0, nothing left behind | +| `evaluator` upload/show/update/list/delete | Full lifecycle, version 1 → 2, nothing left behind | +| Deploy with a **custom** evaluator | Publishes once, redeploys are no-ops, an edit publishes the next version | +| Run with built-in **and** custom evaluators | 4 passed / 0 failed, both criteria reported | +| `--no-prompt` | Every required value fails fast naming the flag; nothing blocks | +| **Spec Example 1, verbatim**: `init` → `generate --max-samples 50` → `azd up` → `run` → `results show --failed-only -O` | All five steps from an empty directory. Dataset generated and downloaded, group `eval_78de667a…` deployed in 40s, `evalrun_6b2044cf…` completed, `results.json` written | +| Spec Examples 2, 3, 4 | Verified verbatim | +| `results compare` (M2) | Baseline vs treatment, `PairedTTest`, signed deltas and p-values; `-o json` valid | +| `generate` write-back | Adds the artifact reference and preserves comments, ordering and siblings | +| Generated dataset download | Container-URI case exercised: credential fetched, container listed, JSONL read | +| **Agent-seeded generation, nothing authored** | `init` → `generate` with no instruction file: seeded from the agent's published instructions, 14 rows generated, 13 of 14 on the agent's actual catalog/policies; `azd deploy` published them; the run scored 14 passed / 0 failed / 0 errored | +| Missing generation model | Fails before any network call, naming `--eval-model` and the spec field | +| **`schedule` (M2)** | Create, list, show and delete against the live project; trigger read back from the service as stored, not echoed. One-per-project and existing-name refusals both verified, each naming the schedule and the command to clear it. Delete waits out `Creating` and leaves the project empty | +| **`run --from-traces` (M4)** | Accepted and executed by the service, which stored the payload and normalised `7d` into `lookback_hours: 168` while honouring `max_traces`. The run fails only because this project's agent emits no GenAI traces, and now says exactly that | +| Failed runs | The reason reaches the caller instead of just the word "failed" | +| **`run --response-id` (M4)** | Three stored responses evaluated, 3 passed / 0 errored; the stored payload matched what was sent field for field | +| Sent-vs-stored audit | Every payload compared against what the service kept. Only the trace window was actually being dropped; inline content becoming a `file_id`, and `item_schema` being normalised to `schema.item`, are both benign | +| **`target.type: model` (M4)** | Group deployed with `response` bound to `{{sample.output_text}}`, ran, and scored 2 passed / 1 failed / 0 errored across coherence and fluency | +| **Registered dataset on a run (M4)** | A group with no local `source:` now runs: whole set scores 2 passed / 1 failed, `--max-samples 2` scores 2 rows. Previously a 400 | +| **Two groups in one config** | Distinct ids across repeated deploys, each running its own criteria. Previously the second deploy aliased them onto one group | +| Pinned dataset `version:` | `1.0` publishes 1.0; editing the file while pinned stops with an instruction. Previously published 2.0, then 3.0 | +| Evaluator declaration forms | `source:` alone publishes then reports unchanged; `version:` alone references; both together refused | +| Conversation-level evaluation | `task_completion` publishes `messages` bound to `{{item.messages}}` with `evaluation_level: conversation`, and runs 1 passed / 1 failed / 0 errored with no target | +| `--eval-group` on the id-taking commands | Each group's own runs and results reachable by name; an undeployed name refused by name | +| **All four spec examples, verbatim** | 1: `init` → `generate --max-samples 50` → `azd up` (dataset 10.0, evaluator 22, group created) → `run` completed → `results show --failed-only -O ./results.json` (1503 b). 2: BYO dataset + `builtin.task_adherence`, `run --max-samples 25` completed. 3: `dataset create`, `run start --eval-id --no-prompt -o json` parsed, `results export --format csv -O gate.csv` (108 b). 4: the unregistered-edit error, wording matching the spec | +| Repeated `azd up` | 2nd and 3rd deploys both report `Dataset golden is unchanged at version 3.0`; no new versions | +| Hand-set `EVAL_GROUP_ID` | Honoured on a single-group config — the group is reused, not recreated. The per-group fix had silently removed this documented path | +| `-o json` on list commands | `dataset`, `evaluator`, `schedule`, `run list` all emit a bare array. They previously leaked two different service envelopes, `value` and `data` | +| `--eval-id` on the sibling commands | Accepted by `run list\|show\|cancel` and `results show\|export\|compare`, matching `run start`; positional still wins | +| `results compare` on one-sample runs | Service sends `"standardDeviation": "NaN"` — a quoted string, since JSON has no NaN literal. Decoding into `float64` failed the whole comparison, discarding the `TooFewSamples` verdict that explains it. Now decodes, renders the undefined statistic as `-`, and emits `null` in JSON. The earlier pass only held because those runs had enough samples | +| **Scenario suite, 18 assertions on substance** | Every scenario checked on result counts and file contents rather than exit status: agent run (1 passed / 0 errored), `results show -O` parseable, CSV header + rows, JSON export valid, model target (2 scored), `--max-samples 1` scoring exactly 1 of 2 rows, conversation level (1 scored), `--response-id`, `--from-traces`, `results compare` table + JSON, schedule set/show/delete. 18/18 | +| `TestLiveRun` | Was **skipping** unless `AZURE_AI_EVAL_AGENT` is set, so the run phase had never executed in any "full suite green" claim. Now run against a real agent, and it asserts no errored samples and at least one scored — reaching a terminal state alone would stay green with a broken target | +| Schedule inherits the group's last run | A schedule repeats the most recent run, so `--from-traces` turns the next schedule into a trace evaluation, which the service restricts to hourly. Proved by experiment: daily accepted after an agent run, refused after a traces run on the same group, hourly accepted for that traces run. The bare service message named neither the cause nor the remedy | + +## 11d. Blocked — needs the service team + +**Agent-seeded data generation fails for every agent.** `POST /data_generation_jobs` with an `agent` source in `inputs.sources` is accepted (201) and then fails within seconds: + +``` +"error": { "code": "DataGenerationJobSystemError", + "message": "Something went wrong during data generation. Please try again." } +``` + +**Ruled out, by probe.** The payload matches the published contract (`AgentDataGenerationJobSource` in `RAISvc/Contracts/DataGenerationJobs/Models/DataGenerationJobSource.cs`: `agent_name` + optional `agent_version`, which is exactly what is sent). Every identifier form fails the same way — name, `agent_version` pinned to `1`/`2`/`latest`, an assistant id, an assistant name, agent with and without a prompt source, and all three api-versions. **A nonexistent agent name fails identically**, so the agent is never resolved and the error carries no signal. + +**Where it goes.** `{project}/data_generation_jobs` → RAISvc S2S client (`DependencyExtensions.cs`, targeting `FineTuningHostUri`) → FineTuning `foundryProxy/data_generation_jobs` → `FoundryProxyTransform.cs` rewrites the path to `{aoaiEndpointTarget}/openai/v1/data_generation_jobs`. Neither RAISvc nor FineTuning resolves the agent — FineTuning has no reference to `agent_name` anywhere. The failure is in the AOAI generator, outside this repo. + +**What the CLI does instead.** The contract says the agent source exists to "fetch instructions / metadata from" the agent, which is a read the client can do itself. `generate` resolves the agent's instructions locally (§6a) and passes them as the prompt source. The agent source is still sent, so it starts contributing when the service is fixed, and the retry covers the failure until then. + +**Related, worth reporting:** an invalid enum value anywhere in the request returns `"The dataGenerationJob field is required."` — a whole-body deserialization failure reported as a missing field. Same misleading shape as the `definition.type` case on evaluator upload. + +**Also reported by the service, worth filing:** `results compare` returns `"standardDeviation": "NaN"` as a **quoted string** whenever a run has a single sample. JSON has no NaN literal, so this is the service's workaround, but it means a typed client must special-case the field or lose the whole comparison. The extension now decodes it (§11c); the service would be better emitting `null`. + +--- + +## 11e. Probed and genuinely unavailable + +Recorded because four M4 items were filed as "awaiting service support" and every one of them turned out to be already shipped. These three were checked rather than assumed, and they hold. + +| Claim | How it was checked | Result | +|---|---|---| +| M3: the eval group is versioned | `Evaluation.cs` in `RAISvc/Contracts/UnifiedEvaluationV2` | No `version` property. Blocked | +| M3: the eval group binds a dataset | `DataSourceConfig.cs` derived types | `custom`, `logs`, `stored_completions`, and the `azure_ai_source` scenarios (`red_team`, `synthetic_data_gen`, `responses`, `traces`, `benchmark_preview`, `conversation_simulation_preview`). None binds a registered dataset. Blocked | +| M4: a prompt target exists | `Target.cs` `TargetType` enum | Values are `azure_ai_model`, `azure_ai_agent`, `azure_ai_assistant`, plus `azure_ai_traces` marked `[NotARequestDiscriminator]`. No prompt target. Blocked | +| An assistant target could be exposed | Live POST of a known-good run body with only `target` swapped to `{"type":"azure_ai_assistant","id":"asst_…"}` | **400** `Unsupported target type in TargetCompletionsEvalRunDataSource: AzureAIAssistant is invalid`. On the enum, refused by the run data source. Building the CLI surface would have shipped a dead path | + +Unexposed capability seen while checking, out of the spec's scope and not implemented: `red_team`, `synthetic_data_gen`, `benchmark_preview` and `conversation_simulation_preview` data source configs, and an `EvalCsvRunDataSource`. + +--- + +## 12. Source-of-truth index + +| Doc / path | Gives you | +|---|---| +| `../azure.ai.agents/internal/pkg/agents/{eval_api,dataset_api}/` | The clients to lift. **Start here.** | +| `../azure.ai.agents/internal/cmd/eval_*.go` | Current command implementations, progress UX, api-version wiring | +| `../azure.ai.agents/internal/cmd/listen.go` | Service-target + lifecycle wiring to copy | +| `../azure.ai.agents/extension.yaml`, `main.go`, `go.mod` | Manifest, entrypoint, dependency versions | +| `cli/azd/pkg/azdext/` | Extension SDK: `ServiceTargetProvider`, `EventManager`, `Environment()` | +| `cli/azd/pkg/project/service_config.go` | `AdditionalProperties` inline capture | +| `cli/azd/pkg/foundry/includes.go` | `ResolveFileRefs` | +| `cli/azd/internal/cmd/up_graph.go` | What `azd up` actually runs | +| `foundrysdk_specs/.../azd_eval_extension/spec.md` | Design spec (authoritative) | +| `foundrysdk_specs/.../azd-agent-eval-public-preview-findings.md` | Measured bugs and timings in §7 | +| `foundrysdk_specs/.../custom_evaluator_upload/spec.md` | Evaluator upload flow, packaging, RBAC | + +--- + +*Keep in sync with `spec.md`. Uncommitted working document.* diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/agent_context_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/agent_context_test.go index 5fc740610e5..7ba9e5ce23a 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/agent_context_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/agent_context_test.go @@ -85,9 +85,8 @@ func TestGenerationModel(t *testing.T) { assert.Equal(t, "gpt-4.1-nano", generationModel(cfg)) } -// `tools` is accepted and ignored, so it has to be called out — the same -// reasoning as the trace fields it now shares a warning with. -func TestWarnIgnoredFields_CoversTools(t *testing.T) { +// Trace selection is accepted and ignored, so it has to be called out. +func TestWarnIgnoredFields_CoversTraceSelection(t *testing.T) { cases := []struct { name string build func(*project.GenerateConfig) @@ -100,17 +99,18 @@ func TestWarnIgnoredFields_CoversTools(t *testing.T) { quiet: true, }, { - name: "tools alone", - build: func(c *project.GenerateConfig) { c.Agent.Context.Tools = "./agent/tools.json" }, - want: []string{"agent.context.tools", "has no effect"}, + name: "a source alone", + build: func(c *project.GenerateConfig) { + c.Agent.Context.Traces = &project.TraceSpec{Source: "app-insights"} + }, + want: []string{"agent.context.traces.source", "has no effect"}, }, { - name: "tools and a trace field agree in number", + name: "source and sample agree in number", build: func(c *project.GenerateConfig) { - c.Agent.Context.Tools = "./agent/tools.json" - c.Agent.Context.Traces = &project.TraceSpec{Source: "app-insights"} + c.Agent.Context.Traces = &project.TraceSpec{Source: "app-insights", Sample: 100} }, - want: []string{"agent.context.traces.source", "agent.context.tools", "have no effect"}, + want: []string{"agent.context.traces.source", "agent.context.traces.sample", "have no effect"}, }, { name: "a window alone is honored, so no warning", @@ -141,9 +141,7 @@ func TestWarnIgnoredFields_CoversTools(t *testing.T) { } // init scaffolds only the context fields that are read. -func TestInitScaffold_OmitsToolsButKeepsInstructions(t *testing.T) { +func TestInitScaffold_KeepsInstructions(t *testing.T) { cfg := buildGenerateScaffold("support-agent", "support-agent-quality", "gpt-4.1-nano") assert.Equal(t, "./agent/instructions.md", cfg.Agent.Context.Instructions) - assert.Empty(t, cfg.Agent.Context.Tools, - "scaffolding a field nothing reads would warn on every default init") } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset.go index 3e4513b07c3..fb5b5865e79 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset.go @@ -21,8 +21,7 @@ func newDatasetCommand() *cobra.Command { Short: "Manage evaluation datasets.", } cmd.AddCommand( - newDatasetCreateCommand(false), - newDatasetCreateCommand(true), + newDatasetCreateCommand(), newDatasetListCommand(), newDatasetShowCommand(), newDatasetDeleteCommand(), @@ -30,9 +29,12 @@ func newDatasetCommand() *cobra.Command { return cmd } -// newDatasetCreateCommand builds `dataset create` and `dataset update`. Both -// publish a new immutable version; the server auto-increments. -func newDatasetCreateCommand(update bool) *cobra.Command { +// newDatasetCreateCommand builds `dataset create`. +// +// There is no separate `update`: every registration publishes a new immutable +// version and the server auto-increments, so `create` covers both the first +// version and every later one. +func newDatasetCreateCommand() *cobra.Command { var ( name string file string @@ -40,10 +42,8 @@ func newDatasetCreateCommand(update bool) *cobra.Command { endpointFlg string ) - use, short := "create", "Register a dataset, creating its first version." - if update { - use, short = "update", "Publish a new version of an existing dataset." - } + use := "create" + short := "Register a dataset, publishing a new version." cmd := &cobra.Command{ Use: use, diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go index e0203f07d5e..1b33daeba1d 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go @@ -179,8 +179,7 @@ func newGenerateCommand() *cobra.Command { // The generation API takes a day window and nothing else, so `source` and // `sample` are parsed and dropped. Silently discarding them is worse than not // accepting them: the author believes they narrowed the trace selection when -// nothing changed. `agent.context.tools` is in the same position — nothing -// reads it, and only the instructions half of the agent's context is used. +// nothing changed. func warnIgnoredTraceFields(cfg *project.GenerateConfig, out io.Writer) { var fields []string @@ -192,9 +191,6 @@ func warnIgnoredTraceFields(cfg *project.GenerateConfig, out io.Writer) { fields = append(fields, "agent.context.traces.sample") } } - if cfg.Agent.Context.Tools != "" { - fields = append(fields, "agent.context.tools") - } if len(fields) == 0 { return } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/generate_config.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/generate_config.go index 22ec21ab3e0..1c41c92f9cf 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/generate_config.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/generate_config.go @@ -38,7 +38,6 @@ type AgentSpec struct { // AgentContext points at the material used to synthesize a rubric and dataset. type AgentContext struct { Instructions string `yaml:"instructions,omitempty" json:"instructions,omitempty"` - Tools string `yaml:"tools,omitempty" json:"tools,omitempty"` Traces *TraceSpec `yaml:"traces,omitempty" json:"traces,omitempty"` } diff --git a/cli/azd/extensions/azure.ai.evaluations/tests/live/live_test.go b/cli/azd/extensions/azure.ai.evaluations/tests/live/live_test.go index 2ecceff6f7e..4b3c8765728 100644 --- a/cli/azd/extensions/azure.ai.evaluations/tests/live/live_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/tests/live/live_test.go @@ -191,9 +191,9 @@ func TestLiveDatasetLifecycle(t *testing.T) { }) } -// TestLiveEvalGroupLifecycle proves the create request this extension builds is +// TestLiveEvalLifecycle proves the create request this extension builds is // accepted, which is the single most important contract to get right. -func TestLiveEvalGroupLifecycle(t *testing.T) { +func TestLiveEvalLifecycle(t *testing.T) { env := setup(t) ctx := context.Background() @@ -234,12 +234,12 @@ func TestLiveEvalGroupLifecycle(t *testing.T) { } group, err := env.evalClient.CreateOpenAIEval(ctx, req) - require.NoError(t, err, "creating the eval group") + require.NoError(t, err, "creating the eval") require.NotEmpty(t, group.ID, "the service assigns the id; name is not unique") - t.Logf("created eval group %s (name %q)", group.ID, group.Name) + t.Logf("created eval %s (name %q)", group.ID, group.Name) fetched, err := env.evalClient.GetOpenAIEval(ctx, group.ID) - require.NoError(t, err, "reading the eval group back") + require.NoError(t, err, "reading the eval back") require.Equal(t, group.ID, fetched.ID) } @@ -283,7 +283,7 @@ func TestLiveRun(t *testing.T) { }, }}, }) - require.NoError(t, err, "creating the eval group for the run") + require.NoError(t, err, "creating the eval for the run") ds := eval_api.NewAgentTargetDataSource(env.agentName, nil) ds.SetFileContent([]map[string]any{ From da39cb87ddf821fa18f83e40d79fd1711e5986ce Mon Sep 17 00:00:00 2001 From: mohessie Date: Thu, 30 Jul 2026 00:40:56 +0300 Subject: [PATCH 060/320] Finish the eval-group rename in user-facing strings, and add run list --limit The evalGroups -> evals rename covered the config key and the flags but stopped there. Ten user-facing strings still said "group", so the CLI contradicted the spec and read as though the rename had been abandoned half-way: --eval help "Name a group from the config" run --name help "Defaults to the group name plus a timestamp" schedule --name help "Defaults to the group name" schedule list header "EVAL GROUP" schedule show field "group:" run no-previous-run "pass a config that declares the group" run from-traces "Declare target.type: agent on the group" schedule traces-only "this group's most recent run", "run the group once" Also rename the metadata written onto the eval, azd_eval_group -> azd_eval. Nothing reads it back, so this is safe. Separately, run list called ListOpenAIEvalRuns with a hardcoded limit of 0 and exposed no way to change it, even though the client already took the parameter. Add --limit. --- .../azure.ai.evaluations/internal/cmd/build.go | 2 +- .../azure.ai.evaluations/internal/cmd/results.go | 4 ++-- .../azure.ai.evaluations/internal/cmd/run.go | 6 +++--- .../azure.ai.evaluations/internal/cmd/run_ops.go | 5 ++++- .../internal/cmd/schedule.go | 16 ++++++++-------- 5 files changed, 18 insertions(+), 15 deletions(-) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build.go index 6ca6d2fcf61..6c938f32a70 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build.go @@ -279,7 +279,7 @@ func buildEvalRequest( } } targetBindings := sampleBindingsFor(targetType) - metadata["azd_eval_group"] = group.Name + metadata["azd_eval"] = group.Name // The create request has no description field, so the group's own // description rides in metadata rather than being dropped. if group.Description != "" { diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/results.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/results.go index 405b538c01e..8b7728bb7bc 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/results.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/results.go @@ -210,12 +210,12 @@ func resolveEvalID( // positional. func addEvalFlags(cmd *cobra.Command, target *string) { cmd.Flags().StringVar(target, "eval", "", - "Name a group from the config instead of passing its id.") + "Name an evals entry from the config instead of passing its id.") cmd.Flags().String("eval-id", "", "Id of the eval. Same as passing the id as an argument.") } -// latestOrNamedRun returns the named run, or the most recent one for the group. +// latestOrNamedRun returns the named run, or the most recent one for the eval. func (ec *evalContext) latestOrNamedRun( cmd *cobra.Command, evalID, runID string, diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go index a9a370113ee..e6087ae0e38 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go @@ -183,7 +183,7 @@ func buildRunCommand(use, short string) *cobra.Command { "Which evals entry to run. Defaults to the only one.") cmd.Flags().StringVar(&evalID, "eval-id", "", "Run against an existing eval by id, ignoring the config.") - cmd.Flags().StringVar(&runName, "name", "", "Name for this run. Defaults to the group name plus a timestamp.") + cmd.Flags().StringVar(&runName, "name", "", "Name for this run. Defaults to the eval name plus a timestamp.") cmd.Flags().StringVar(&level, "level", "", "Scoring granularity: turn or conversation. Defaults to the service default (turn).") cmd.Flags().IntVar(&maxSamples, "max-samples", 0, @@ -361,7 +361,7 @@ func (ec *evalContext) reuseDataSourceFromLastRun( "eval %s has no previous run to repeat, so there is no target or dataset "+ "to reuse.\n"+ " Run it from the config once with `azd ai eval run`, or pass a config that "+ - "declares the group", + "declares the eval", evalID) } return list.Data[0].DataSource, nil @@ -398,7 +398,7 @@ func buildTracesDataSource( if agent == "" { return nil, fmt.Errorf( "--from-traces needs to know whose traces to read, and the eval does not " + - "name an agent. Declare target.type: agent on the group") + "name an agent. Declare target.type: agent on the eval") } var lookbackHours int diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go index 8fab2f3bca6..fbfdb2c8271 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go @@ -29,6 +29,7 @@ func newRunListCommand() *cobra.Command { var ( endpointFlg string groupName string + limit int ) cmd := &cobra.Command{ @@ -48,7 +49,7 @@ func newRunListCommand() *cobra.Command { return err } - list, err := ec.evalClient.ListOpenAIEvalRuns(ctx, evalID, 0) + list, err := ec.evalClient.ListOpenAIEvalRuns(ctx, evalID, limit) if err != nil { if eval_api.IsNotFound(err) { return fmt.Errorf( @@ -78,6 +79,8 @@ func newRunListCommand() *cobra.Command { }, } addEvalFlags(cmd, &groupName) + cmd.Flags().IntVar(&limit, "limit", 0, + "Return at most this many runs. Omit for the service default.") cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") return cmd } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/schedule.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/schedule.go index b2f223ebf0c..5a8affe44bd 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/schedule.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/schedule.go @@ -197,7 +197,7 @@ func newScheduleSetCommand() *cobra.Command { "Path to the eval deployment config.") cmd.Flags().StringVar(&groupName, "eval", "", "Which evals entry to schedule.") cmd.Flags().StringVar(&evalID, "eval-id", "", "Schedule an existing eval by id, ignoring config.") - cmd.Flags().StringVar(&name, "name", "", "Schedule name. Defaults to the group name.") + cmd.Flags().StringVar(&name, "name", "", "Schedule name. Defaults to the eval name.") cmd.Flags().StringVar(&description, "description", "", "Schedule description.") cmd.Flags().StringVar(&cron, "cron", "", `Cron expression, for example "0 9 * * *".`) cmd.Flags().StringVar(&every, "every", "", @@ -254,20 +254,20 @@ func newScheduleListCommand() *cobra.Command { rows := make([][]string, 0, len(list.Value)) for i := range list.Value { s := &list.Value[i] - evalGroup := "" + evalID := "" if s.Task != nil { - evalGroup = s.Task.EvalID + evalID = s.Task.EvalID } rows = append(rows, []string{ s.ID, strconv.FormatBool(s.Enabled), s.ProvisioningStatus, s.Summary(), - evalGroup, + evalID, }) } return emitTable(out, - []string{"NAME", "ENABLED", "STATUS", "TRIGGER", "EVAL GROUP"}, rows) + []string{"NAME", "ENABLED", "STATUS", "TRIGGER", "EVAL"}, rows) }, } cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") @@ -317,7 +317,7 @@ func newScheduleShowCommand() *cobra.Command { fmt.Fprintf(out, " timezone: %s\n", s.Trigger.Timezone) } if s.Task != nil { - fmt.Fprintf(out, " group: %s\n", s.Task.EvalID) + fmt.Fprintf(out, " eval: %s\n", s.Task.EvalID) } if s.Description != "" { fmt.Fprintf(out, " about: %s\n", s.Description) @@ -599,9 +599,9 @@ func explainScheduleFailure( // is bewildering when the trigger was the only thing asked for. if isTracesHourlyOnly(cause) { return fmt.Errorf( - "saving schedule %q: this group's most recent run read from traces, and a schedule "+ + "saving schedule %q: this eval's most recent run read from traces, and a schedule "+ "repeats that run, so the service treats it as a scheduled trace evaluation "+ - "and allows only `--every hourly`. Use `--every hourly`, or run the group "+ + "and allows only `--every hourly`. Use `--every hourly`, or run the eval "+ "once against its dataset first so the schedule repeats that instead", name) } From c919ed687dfa5731d29755a5a5ca82a8862aba11 Mon Sep 17 00:00:00 2001 From: mohessie Date: Thu, 30 Jul 2026 01:12:14 +0300 Subject: [PATCH 061/320] Add custom code evaluators `evaluator create --folder ` publishes a folder of Python as an evaluator version, and an `evaluators:` entry whose `source:` is a directory is reconciled by `azd up` the same way a dataset is. The wire shape was ambiguous. Two documents disagreed, and both were partly wrong. The published OpenAPI document that RAISvc vendors and contract-tests against settles it: {"type":"code", "code_text", "entry_point", "image_tag", "blob_uri"} plus inherited init_parameters, data_schema and metrics, all snake_case. The sibling custom_evaluator_upload spec is stale on four counts - the route is POST /evaluators/{name}/versions not PUT .../versions/{version}, credentials is POST not GET, the body is snake_case not camelCase, and the discriminator is "code" not "CodeBased". There is no version-based shape negotiation; one contract serves every api-version. blob_uri is a container prefix rather than an archive: the consumer enumerates the container, so files upload at their relative paths. Two deliberate choices where the evidence did not reach: Only blob_uri is sent, never code_text. The contract allows either, and inline would save a round trip for a single-file evaluator, but nothing observable confirms the executor runs it - the hand-off converter drops both fields and refetches from the catalog. blob_uri is the alternative with a demonstrated consumer. Sending the unproven one would trade a saved upload for an evaluator that registers cleanly and then fails when it is run, which is much harder to diagnose. Dot-prefixed files are excluded from the upload, not just dot-prefixed directories. An evaluator folder kept in a repo collects .env, .netrc and .pypirc, and publishing would copy those secrets into blob storage. Nothing a Python evaluator needs at runtime starts with a dot. entry_point is sent although RAISvc has no such property yet; it is spec-declared, and the create path ignores unknown members, so it starts persisting when the service catches up. The live round-trip test is written but unrun - it needs a project endpoint. It is the only thing that will confirm the service accepts this body; everything here is read from source and spec, not observed. --- .../internal/cmd/evaluator.go | 187 ++++++++- .../internal/cmd/evaluator_code_live_test.go | 227 +++++++++++ .../internal/cmd/evaluator_code_test.go | 194 ++++++++++ .../internal/cmd/reconciler.go | 100 ++++- .../internal/pkg/eval_api/code_evaluator.go | 364 ++++++++++++++++++ .../internal/pkg/eval_api/operations.go | 17 + .../internal/pkg/evalcore/codefolder.go | 356 +++++++++++++++++ .../internal/pkg/evalcore/codefolder_test.go | 329 ++++++++++++++++ .../internal/project/eval_config.go | 5 + .../internal/project/service_target_eval.go | 18 + 10 files changed, 1789 insertions(+), 8 deletions(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator_code_live_test.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator_code_test.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/code_evaluator.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/codefolder.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/codefolder_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator.go index 100e8e7b10a..6e15b0a6442 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator.go @@ -9,6 +9,7 @@ import ( "os" "azureaieval/internal/pkg/eval_api" + "azureaieval/internal/pkg/evalcore" "github.com/spf13/cobra" ) @@ -31,17 +32,22 @@ func newEvaluatorCommand() *cobra.Command { // `dataset create`: both register an artifact and both publish a new immutable // version every time, so there is nothing for a separate `update` to do. // -// M1 supports rubric evaluators only. Code evaluators need a folder walk, -// multi-blob upload, and the Azure AI User role assignment, so they land later. +// An evaluator is either a rubric — a JSON file of weighted dimensions — or +// code — a folder of Python. They are different definition types on the wire, +// so exactly one of the two sources has to be named. func newEvaluatorCreateCommand() *cobra.Command { var ( name string rubric string + folder string + initParams string + dataSchema string + metrics string endpointFlg string ) use := "create" - short := "Register a rubric evaluator, publishing a new version." + short := "Register a rubric or code evaluator, publishing a new version." cmd := &cobra.Command{ Use: use, @@ -50,8 +56,20 @@ func newEvaluatorCreateCommand() *cobra.Command { if name == "" { return requireFlag("name") } - if rubric == "" { - return requireFlag("rubric") + flags := codeEvaluatorFlags{ + initParams: initParams, + dataSchema: dataSchema, + metrics: metrics, + endpoint: endpointFlg, + } + if err := validateEvaluatorSource(rubric, folder, flags); err != nil { + return err + } + + ctx := cmd.Context() + + if folder != "" { + return runEvaluatorCreateFromFolder(cmd, name, folder, flags) } raw, err := os.ReadFile(rubric) @@ -64,7 +82,6 @@ func newEvaluatorCreateCommand() *cobra.Command { return fmt.Errorf("rubric %q: %w", rubric, err) } - ctx := cmd.Context() ec, err := newEvalContext(ctx, endpointFlg) if err != nil { return err @@ -89,10 +106,168 @@ func newEvaluatorCreateCommand() *cobra.Command { cmd.Flags().StringVar(&name, "name", "", "Name of the evaluator.") cmd.Flags().StringVar(&rubric, "rubric", "", "Path to the rubric JSON file.") + cmd.Flags().StringVar(&folder, "folder", "", + "Path to a folder of Python holding the evaluator code.") + cmd.Flags().StringVar(&initParams, "init-params", "", + "Path to a JSON Schema for the evaluator's initialization parameters. "+ + "Overrides the folder's "+evalcore.CodeEvaluatorMetadataFile+".") + cmd.Flags().StringVar(&dataSchema, "data-schema", "", + "Path to a JSON Schema for the evaluator's input data. "+ + "Overrides the folder's "+evalcore.CodeEvaluatorMetadataFile+".") + cmd.Flags().StringVar(&metrics, "metrics", "", + "Path to a JSON object describing the metrics the evaluator produces. "+ + "Overrides the folder's "+evalcore.CodeEvaluatorMetadataFile+".") cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") return cmd } +// codeEvaluatorFlags are the optional overrides for a code evaluator. +type codeEvaluatorFlags struct { + initParams string + dataSchema string + metrics string + endpoint string +} + +// validateEvaluatorSource enforces that exactly one source is named, and that +// the schema overrides are only used with the source they apply to. +// +// Deliberately checked here rather than with MarkFlagsMutuallyExclusive: that +// only rejects the "both" case, and its message names a flag group rather than +// saying what the two flags mean. Both mistakes deserve advice, and this is +// testable without driving cobra. +func validateEvaluatorSource(rubric, folder string, flags codeEvaluatorFlags) error { + switch { + case rubric == "" && folder == "": + return fmt.Errorf( + "one of --rubric or --folder is required: --rubric takes a JSON file of " + + "weighted dimensions, --folder takes a directory of Python") + case rubric != "" && folder != "": + return fmt.Errorf( + "--rubric and --folder cannot be used together: an evaluator is either a " + + "rubric or code, not both") + } + + // A rubric's schemas are fixed by the service, so these would be accepted + // and then quietly dropped — the worst kind of no-op, because the author + // believes the evaluator was published carrying them. + if folder == "" { + for _, named := range []struct { + flag string + value string + }{ + {"init-params", flags.initParams}, + {"data-schema", flags.dataSchema}, + {"metrics", flags.metrics}, + } { + if named.value != "" { + return fmt.Errorf( + "--%s applies to a code evaluator and needs --folder; "+ + "a rubric's schemas are set by the service", named.flag) + } + } + } + return nil +} + +// runEvaluatorCreateFromFolder validates the folder, then publishes it. +func runEvaluatorCreateFromFolder( + cmd *cobra.Command, + name string, + folder string, + flags codeEvaluatorFlags, +) error { + pkg, err := evalcore.LoadCodeEvaluator(name, folder) + if err != nil { + return err + } + + opts, err := codeEvaluatorOptions(pkg, flags) + if err != nil { + return err + } + + ctx := cmd.Context() + ec, err := newEvalContext(ctx, flags.endpoint) + if err != nil { + return err + } + defer ec.Close() + + created, err := ec.evalClient.UploadCodeEvaluatorVersion( + ctx, pkg, opts, ProjectEndpointAPIVersion, + ) + if err != nil { + return fmt.Errorf("publishing evaluator %q: %w", name, err) + } + + if isJSON(cmd) { + return emitJSON(cmd.OutOrStdout(), created) + } + fmt.Fprintf(cmd.OutOrStdout(), + "Published evaluator %s version %s from %d file(s) in %s\n", + created.Name, created.Version, len(pkg.Files), folder) + return nil +} + +// codeEvaluatorOptions resolves the evaluator's schemas, preferring an +// explicit flag over whatever the folder declares. +// +// The folder is the better place for them — they describe the code and belong +// beside it — but a folder that has none must still be publishable without +// editing it, which is what the flags are for. +func codeEvaluatorOptions( + pkg *evalcore.CodeEvaluatorPackage, + flags codeEvaluatorFlags, +) (eval_api.CodeEvaluatorOptions, error) { + var opts eval_api.CodeEvaluatorOptions + if md := pkg.Metadata; md != nil { + opts.DisplayName = md.DisplayName + opts.Description = md.Description + opts.Categories = md.Categories + opts.InitParameters = md.InitParameters + opts.DataSchema = md.DataSchema + opts.Metrics = md.Metrics + } + + for _, override := range []struct { + path string + flag string + field *json.RawMessage + }{ + {flags.initParams, "init-params", &opts.InitParameters}, + {flags.dataSchema, "data-schema", &opts.DataSchema}, + {flags.metrics, "metrics", &opts.Metrics}, + } { + if override.path == "" { + continue + } + raw, err := readJSONObject(override.path) + if err != nil { + return opts, fmt.Errorf("--%s %q: %w", override.flag, override.path, err) + } + *override.field = raw + } + + return opts, nil +} + +// readJSONObject reads a file that must hold a JSON object. +// +// Parsing here rather than letting the service reject it keeps a typo from +// costing an upload and a published version, and names the file that is wrong. +func readJSONObject(path string) (json.RawMessage, error) { + raw, err := os.ReadFile(path) + if err != nil { + return nil, err + } + var probe map[string]json.RawMessage + if err := json.Unmarshal(raw, &probe); err != nil { + return nil, fmt.Errorf("not a JSON object: %w", err) + } + return json.RawMessage(raw), nil +} + // normalizeRubricBody accepts either a bare definition ({type, dimensions}) or // a full evaluator document ({name, definition}) and returns the request body. // rubricDefinitionType is the discriminator the service uses to deserialize a diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator_code_live_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator_code_live_test.go new file mode 100644 index 00000000000..f9c20c6d07d --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator_code_live_test.go @@ -0,0 +1,227 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +//go:build live + +// This file proves the code-evaluator body the extension builds is accepted by +// the real service, and that what comes back is the shape the extension +// expects. +// +// It exists because the wire contract was settled from source rather than from +// a live call: two published documents disagreed on the definition body, and +// only the service can say which one it honours. It asserts the round trip +// field by field so a drift shows up as a named mismatch, not a vague failure. +// +// go test -tags live -v ./internal/cmd/ -run TestLiveCodeEvaluator +// +// Required: AZURE_AI_EVAL_E2E_LIVE=1 and FOUNDRY_PROJECT_ENDPOINT. + +package cmd + +import ( + "context" + "encoding/json" + "fmt" + "os" + "path/filepath" + "testing" + "time" + + "azureaieval/internal/pkg/eval_api" + "azureaieval/internal/pkg/evalcore" + + "github.com/stretchr/testify/require" +) + +// liveCodeEvaluatorName is unique per run so concurrent runs, and reruns after +// a failure that skipped cleanup, do not collide. +func liveCodeEvaluatorName(t *testing.T, suffix string) string { + t.Helper() + return fmt.Sprintf("azdcode_%s_%d", suffix, time.Now().UnixNano()) +} + +// writeLiveEvaluator lays out a folder to the packaging convention and returns +// it. The source is written for the derived class name so the production +// validation is exercised rather than bypassed. +func writeLiveEvaluator(t *testing.T, name string, extraFiles map[string]string) string { + t.Helper() + dir := t.TempDir() + + className := evalcore.EvaluatorClassName(name) + entry := fmt.Sprintf(`class %s: + def __call__(self, **kwargs): + return {"result": float(len(kwargs.get("response", "")))} +`, className) + + require.NoError(t, os.WriteFile( + filepath.Join(dir, name+".py"), []byte(entry), 0o600)) + + for rel, content := range extraFiles { + path := filepath.Join(dir, filepath.FromSlash(rel)) + require.NoError(t, os.MkdirAll(filepath.Dir(path), 0o755)) + require.NoError(t, os.WriteFile(path, []byte(content), 0o600)) + } + return dir +} + +// codeDefinitionOnService reads the registered version back and returns its +// definition, so the assertions run against what the service persisted rather +// than against what was sent. +func codeDefinitionOnService( + t *testing.T, + client *eval_api.EvalClient, + name, version string, +) map[string]json.RawMessage { + t.Helper() + + raw, err := client.GetEvaluatorRaw( + context.Background(), name, version, ProjectEndpointAPIVersion) + require.NoError(t, err, "reading back evaluator %s version %s", name, version) + + var doc map[string]json.RawMessage + require.NoError(t, json.Unmarshal(raw, &doc)) + require.Contains(t, doc, "definition", + "the registered evaluator carries no definition: %s", string(raw)) + + var definition map[string]json.RawMessage + require.NoError(t, json.Unmarshal(doc["definition"], &definition)) + return definition +} + +func stringField(t *testing.T, definition map[string]json.RawMessage, key string) string { + t.Helper() + raw, ok := definition[key] + if !ok { + return "" + } + var value string + if err := json.Unmarshal(raw, &value); err != nil { + return "" + } + return value +} + +// TestLiveCodeEvaluatorSingleFileRoundTrip publishes a one-file evaluator and +// asserts it comes back as a code definition pointing at storage. +// +// A single file takes the same path as a folder. The contract also accepts +// inline source through code_text, which would save the upload, but nothing +// observable confirms the executor runs it, so the CLI does not send it. If +// that is ever settled, this is the test that should change first. +func TestLiveCodeEvaluatorSingleFileRoundTrip(t *testing.T) { + client, _ := liveEvalClient(t) + ctx := context.Background() + + name := liveCodeEvaluatorName(t, "single") + dir := writeLiveEvaluator(t, name, nil) + + // The shipping loader, not a hand-built package: this test has to fail if + // the production path stops producing a valid package. + pkg, err := evalcore.LoadCodeEvaluator(name, dir) + require.NoError(t, err) + require.Len(t, pkg.Files, 1) + + opts, err := codeEvaluatorOptions(pkg, codeEvaluatorFlags{}) + require.NoError(t, err) + + created, err := client.UploadCodeEvaluatorVersion(ctx, pkg, opts, ProjectEndpointAPIVersion) + require.NoError(t, err, "the service rejected the code evaluator body") + require.NotEmpty(t, created.Version) + t.Cleanup(func() { + _ = client.DeleteEvaluatorVersion( + context.Background(), name, created.Version, ProjectEndpointAPIVersion) + }) + + definition := codeDefinitionOnService(t, client, name, created.Version) + + require.Equal(t, eval_api.CodeDefinitionType, stringField(t, definition, "type"), + "the discriminator must round-trip as the lowercase snake_case value") + require.NotEmpty(t, stringField(t, definition, "blob_uri"), + "a published evaluator must record the storage location it was uploaded to") + require.Contains(t, definition, "metrics", + "a code definition must carry metrics; the service rejects one without") +} + +// TestLiveCodeEvaluatorFolderRoundTrip publishes a multi-file evaluator and +// asserts the service records the storage location it handed out. +// +// This is the path that exercises startPendingUpload, the SAS write, and the +// blob_uri property, none of which the single-file case touches. +func TestLiveCodeEvaluatorFolderRoundTrip(t *testing.T) { + client, _ := liveEvalClient(t) + ctx := context.Background() + + name := liveCodeEvaluatorName(t, "folder") + dir := writeLiveEvaluator(t, name, map[string]string{ + "helpers/text.py": "def clean(value):\n return value.strip()\n", + // Must be excluded from both the upload and the fingerprint. + "__pycache__/stale.pyc": "cache", + }) + + pkg, err := evalcore.LoadCodeEvaluator(name, dir) + require.NoError(t, err) + require.Len(t, pkg.Files, 2, "the compiled artifact must not be part of the package") + + opts, err := codeEvaluatorOptions(pkg, codeEvaluatorFlags{}) + require.NoError(t, err) + + created, err := client.UploadCodeEvaluatorVersion(ctx, pkg, opts, ProjectEndpointAPIVersion) + require.NoError(t, err, "the service rejected the uploaded code evaluator") + require.NotEmpty(t, created.Version) + t.Cleanup(func() { + _ = client.DeleteEvaluatorVersion( + context.Background(), name, created.Version, ProjectEndpointAPIVersion) + }) + + definition := codeDefinitionOnService(t, client, name, created.Version) + + require.Equal(t, eval_api.CodeDefinitionType, stringField(t, definition, "type")) + require.NotEmpty(t, stringField(t, definition, "blob_uri"), + "a multi-file evaluator must round-trip carrying the storage location; "+ + "an empty blob_uri means the service dropped the preview property") + require.Contains(t, definition, "metrics") +} + +// TestLiveCodeEvaluatorPublishesNextVersion proves the version the upload +// reserves storage under is the one the create then assigns. +// +// Storage is provisioned per version before the version exists, so the client +// has to predict it. A drift between the two would leave the code in one +// version's container and the definition on another. +func TestLiveCodeEvaluatorPublishesNextVersion(t *testing.T) { + client, _ := liveEvalClient(t) + ctx := context.Background() + + name := liveCodeEvaluatorName(t, "versions") + dir := writeLiveEvaluator(t, name, map[string]string{ + "helpers.py": "VALUE = 1\n", + }) + + pkg, err := evalcore.LoadCodeEvaluator(name, dir) + require.NoError(t, err) + opts, err := codeEvaluatorOptions(pkg, codeEvaluatorFlags{}) + require.NoError(t, err) + + predicted := client.NextEvaluatorVersion(ctx, name, ProjectEndpointAPIVersion) + require.Equal(t, "1", predicted, "an unpublished evaluator starts at version 1") + + first, err := client.UploadCodeEvaluatorVersion(ctx, pkg, opts, ProjectEndpointAPIVersion) + require.NoError(t, err) + t.Cleanup(func() { + _ = client.DeleteEvaluatorVersion( + context.Background(), name, first.Version, ProjectEndpointAPIVersion) + }) + require.Equal(t, predicted, first.Version, + "the service assigned a version other than the one storage was reserved under") + + // Publishing again must land on the next version, not overwrite the first. + predictedSecond := client.NextEvaluatorVersion(ctx, name, ProjectEndpointAPIVersion) + second, err := client.UploadCodeEvaluatorVersion(ctx, pkg, opts, ProjectEndpointAPIVersion) + require.NoError(t, err) + t.Cleanup(func() { + _ = client.DeleteEvaluatorVersion( + context.Background(), name, second.Version, ProjectEndpointAPIVersion) + }) + require.Equal(t, predictedSecond, second.Version) + require.NotEqual(t, first.Version, second.Version) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator_code_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator_code_test.go new file mode 100644 index 00000000000..a919bd56a90 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator_code_test.go @@ -0,0 +1,194 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "encoding/json" + "io" + "os" + "path/filepath" + "testing" + + "azureaieval/internal/pkg/eval_api" + "azureaieval/internal/pkg/evalcore" + "azureaieval/internal/project" + + "github.com/stretchr/testify/require" +) + +func writeTestFile(t *testing.T, root, rel, content string) string { + t.Helper() + path := filepath.Join(root, filepath.FromSlash(rel)) + require.NoError(t, os.MkdirAll(filepath.Dir(path), 0o755)) + require.NoError(t, os.WriteFile(path, []byte(content), 0o600)) + return path +} + +const toneEvaluatorSource = "class ToneEvaluator:\n" + + " def __call__(self, **kwargs):\n" + + " return {\"result\": 1}\n" + +// An evaluator is either a rubric or code. Naming both, or neither, is a +// mistake the command has to name precisely — the two flags take different +// kinds of path and produce different definition types. +func TestValidateEvaluatorSource(t *testing.T) { + err := validateEvaluatorSource("", "", codeEvaluatorFlags{}) + require.Error(t, err) + require.Contains(t, err.Error(), "--rubric") + require.Contains(t, err.Error(), "--folder") + require.Contains(t, err.Error(), "required") + + err = validateEvaluatorSource("rubric.json", "./evaluator", codeEvaluatorFlags{}) + require.Error(t, err) + require.Contains(t, err.Error(), "cannot be used together") + + require.NoError(t, validateEvaluatorSource("rubric.json", "", codeEvaluatorFlags{})) + require.NoError(t, validateEvaluatorSource("", "./evaluator", codeEvaluatorFlags{})) +} + +// The schema overrides describe a code evaluator. Accepting them beside a +// rubric and dropping them would leave the author believing the evaluator was +// published carrying schemas it never had. +func TestValidateEvaluatorSource_RejectsCodeFlagsOnARubric(t *testing.T) { + for flag, flags := range map[string]codeEvaluatorFlags{ + "init-params": {initParams: "init.json"}, + "data-schema": {dataSchema: "schema.json"}, + "metrics": {metrics: "metrics.json"}, + } { + err := validateEvaluatorSource("rubric.json", "", flags) + require.Error(t, err, "for --%s", flag) + require.Contains(t, err.Error(), "--"+flag) + require.Contains(t, err.Error(), "--folder") + + require.NoError(t, validateEvaluatorSource("", "./evaluator", flags), + "--%s is valid with --folder", flag) + } +} + +// The same check the command runs must be reachable from the command, so a +// future refactor cannot leave the flags declared but unvalidated. +func TestEvaluatorCreateRejectsBothSources(t *testing.T) { + cmd := newEvaluatorCreateCommand() + cmd.SetArgs([]string{"--name", "tone", "--rubric", "r.json", "--folder", "./x"}) + cmd.SetOut(io.Discard) + cmd.SetErr(io.Discard) + cmd.SilenceUsage = true + + err := cmd.Execute() + require.Error(t, err) + require.Contains(t, err.Error(), "cannot be used together") +} + +func TestEvaluatorCreateRejectsNeitherSource(t *testing.T) { + cmd := newEvaluatorCreateCommand() + cmd.SetArgs([]string{"--name", "tone"}) + cmd.SetOut(io.Discard) + cmd.SetErr(io.Discard) + cmd.SilenceUsage = true + + err := cmd.Execute() + require.Error(t, err) + require.Contains(t, err.Error(), "one of --rubric or --folder") +} + +// The folder is the natural home for the schemas, but a folder that carries +// none still has to be publishable without editing it. +func TestCodeEvaluatorOptions_FlagsOverrideFolderMetadata(t *testing.T) { + dir := t.TempDir() + writeTestFile(t, dir, "tone.py", toneEvaluatorSource) + writeTestFile(t, dir, evalcore.CodeEvaluatorMetadataFile, `{ + "display_name": "Tone", + "metrics": {"result": {"type": "ordinal"}}, + "data_schema": {"type": "object", "properties": {"a": {"type": "string"}}} + }`) + + pkg, err := evalcore.LoadCodeEvaluator("tone", dir) + require.NoError(t, err) + + // Nothing overridden: the folder wins. + opts, err := codeEvaluatorOptions(pkg, codeEvaluatorFlags{}) + require.NoError(t, err) + require.Equal(t, "Tone", opts.DisplayName) + require.Contains(t, string(opts.Metrics), "ordinal") + require.Contains(t, string(opts.DataSchema), `"a"`) + require.Empty(t, opts.InitParameters) + + overrides := t.TempDir() + metricsPath := writeTestFile(t, overrides, "metrics.json", + `{"result":{"type":"continuous"}}`) + initPath := writeTestFile(t, overrides, "init.json", + `{"type":"object","properties":{"deployment_name":{"type":"string"}}}`) + + opts, err = codeEvaluatorOptions(pkg, codeEvaluatorFlags{ + metrics: metricsPath, + initParams: initPath, + }) + require.NoError(t, err) + require.Contains(t, string(opts.Metrics), "continuous") + require.NotContains(t, string(opts.Metrics), "ordinal") + require.Contains(t, string(opts.InitParameters), "deployment_name") + // Untouched by the overrides. + require.Contains(t, string(opts.DataSchema), `"a"`) +} + +// A typo in a schema file must be reported against the flag that named it, +// not discovered by the service after a version has been published. +func TestCodeEvaluatorOptions_RejectsMalformedOverride(t *testing.T) { + dir := t.TempDir() + writeTestFile(t, dir, "tone.py", toneEvaluatorSource) + pkg, err := evalcore.LoadCodeEvaluator("tone", dir) + require.NoError(t, err) + + bad := writeTestFile(t, dir, "metrics.json", "[1,2,3]") + _, err = codeEvaluatorOptions(pkg, codeEvaluatorFlags{metrics: bad}) + require.Error(t, err) + require.Contains(t, err.Error(), "--metrics") + require.Contains(t, err.Error(), "JSON object") + + _, err = codeEvaluatorOptions(pkg, codeEvaluatorFlags{ + dataSchema: filepath.Join(dir, "absent.json"), + }) + require.Error(t, err) + require.Contains(t, err.Error(), "--data-schema") +} + +// The service rejects a code definition carrying no metrics, so a folder that +// declares none still has to publish with one. +func TestDefaultCodeMetricsIsAJSONObject(t *testing.T) { + var metrics map[string]map[string]any + require.NoError(t, json.Unmarshal(eval_api.DefaultCodeMetrics, &metrics)) + require.Len(t, metrics, 1) + require.Contains(t, metrics, "result") + require.Equal(t, "continuous", metrics["result"]["type"]) +} + +// Change detection has to work for both kinds of evaluator source, and the +// reconciler decides which by stat-ing the path. +func TestFingerprintPath_HandlesFilesAndFolders(t *testing.T) { + root := t.TempDir() + + file := writeTestFile(t, root, "rubric.json", `{"dimensions":[]}`) + fileDigest, err := project.FingerprintPath(file) + require.NoError(t, err) + plainDigest, err := project.Fingerprint(file) + require.NoError(t, err) + require.Equal(t, plainDigest, fileDigest, + "a file must hash the same through either entry point") + + dir := t.TempDir() + writeTestFile(t, dir, "tone.py", toneEvaluatorSource) + folderDigest, err := project.FingerprintPath(dir) + require.NoError(t, err) + require.NotEmpty(t, folderDigest) + require.NotEqual(t, fileDigest, folderDigest) + + writeTestFile(t, dir, "helpers.py", "X = 1\n") + changed, err := project.FingerprintPath(dir) + require.NoError(t, err) + require.NotEqual(t, folderDigest, changed, + "adding a file to the folder must change the digest") + + _, err = project.FingerprintPath(filepath.Join(root, "absent")) + require.Error(t, err) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go index a06f13853d3..3c5c93234f9 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go @@ -11,9 +11,11 @@ import ( "os" "path/filepath" "reflect" + "strconv" "strings" "azureaieval/internal/pkg/dataset_api" + "azureaieval/internal/pkg/evalcore" "azureaieval/internal/project" ) @@ -223,8 +225,14 @@ func (r *evalReconciler) latestDatasetVersion(ctx context.Context, name string) } // EnsureEvaluator publishes a new version when the local definition differs -// from what the service holds. Evaluator definitions come back inline, so this -// compares content directly rather than relying on a cached digest. +// from what the service holds. +// +// The two kinds of evaluator are told apart by what the source names, not by +// its spelling: a folder is code, a file is a rubric. They also detect change +// differently. A rubric definition comes back inline, so it is compared +// directly; code does not — the service returns a storage URI, never the +// source — so a fingerprint of the folder is kept in the azd environment, the +// same way datasets work. func (r *evalReconciler) EnsureEvaluator( ctx context.Context, decl project.EvaluatorDecl, @@ -242,6 +250,14 @@ func (r *evalReconciler) EnsureEvaluator( return versionFromRaw(raw, decl.Version), false, nil } + info, err := os.Stat(localPath) + if err != nil { + return "", false, fmt.Errorf("evaluator source %q: %w", localPath, err) + } + if info.IsDir() { + return r.ensureCodeEvaluator(ctx, decl, localPath) + } + raw, err := os.ReadFile(localPath) if err != nil { return "", false, fmt.Errorf("evaluator source %q: %w", localPath, err) @@ -270,6 +286,86 @@ func (r *evalReconciler) EnsureEvaluator( return created.Version, true, nil } +// ensureCodeEvaluator publishes a folder of Python only when its content +// changed since the last deploy. +// +// Every publish is a new immutable version, so without this a repeated +// `azd up` would leave a trail of identical versions and force every eval +// bound to the evaluator to be recreated along with them. +func (r *evalReconciler) ensureCodeEvaluator( + ctx context.Context, + decl project.EvaluatorDecl, + dir string, +) (string, bool, error) { + // Validated before anything is uploaded: a folder missing its entry point + // or class is only rejected when a run executes, long after a version has + // been published and an eval bound to it. + pkg, err := evalcore.LoadCodeEvaluator(decl.Name, dir) + if err != nil { + return "", false, err + } + + digest, err := project.FingerprintPath(dir) + if err != nil { + return "", false, err + } + + key := project.FingerprintKey("evaluator", decl.Name) + recordedVersion := r.ec.getEnvValue(ctx, versionKey("evaluator", decl.Name)) + if r.ec.getEnvValue(ctx, key) == digest && recordedVersion != "" { + // Unchanged since the last deploy, but that alone does not make the + // recorded version safe to reuse: someone may have published a newer + // one outside the repo, and binding the eval to the older one would + // quietly evaluate with superseded code. + if err := r.checkEvaluatorDrift(ctx, decl.Name, recordedVersion); err != nil { + return "", false, err + } + return recordedVersion, false, nil + } + + opts, err := codeEvaluatorOptions(pkg, codeEvaluatorFlags{}) + if err != nil { + return "", false, err + } + + created, err := r.ec.evalClient.UploadCodeEvaluatorVersion( + ctx, pkg, opts, ProjectEndpointAPIVersion, + ) + if err != nil { + return "", false, err + } + + _ = r.ec.setEnvValue(ctx, key, digest) + _ = r.ec.setEnvValue(ctx, versionKey("evaluator", decl.Name), created.Version) + return created.Version, true, nil +} + +// checkEvaluatorDrift fails when the service holds a newer version than the +// one recorded at the last deploy. +// +// Publishing is not destructive — versions are immutable — so the remedy is to +// sync with what is on the project, not to overwrite it. +func (r *evalReconciler) checkEvaluatorDrift( + ctx context.Context, + name, recorded string, +) error { + recordedNumber, err := strconv.Atoi(recorded) + if err != nil { + return nil + } + latest := r.ec.evalClient.LatestEvaluatorVersionNumber( + ctx, name, ProjectEndpointAPIVersion, + ) + if latest <= recordedNumber { + return nil + } + return fmt.Errorf( + "evaluator %q is at version %d on the project but %s was recorded at the last deploy; "+ + "someone published a version outside this repo. "+ + "Pull the newer code locally, or delete version %d, then deploy again", + name, latest, recorded, latest) +} + // EnsureEval creates the group when it has never been deployed, or when an // upstream artifact changed. Groups are immutable, so a change means a new // group and a new id. diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/code_evaluator.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/code_evaluator.go new file mode 100644 index 00000000000..5ee2492faed --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/code_evaluator.go @@ -0,0 +1,364 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package eval_api + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "os" + "strconv" + "strings" + + "azureaieval/internal/pkg/evalcore" +) + +// CodeDefinitionType is the discriminator the service uses to deserialize a +// code evaluator definition. +// +// The wire shape is snake_case with a lowercase discriminator, matching +// CodeBasedEvaluatorDefinition in the Foundry data-plane OpenAPI document +// (`type` enum ["code"], plus code_text, entry_point, blob_uri, init_parameters, +// data_schema and metrics). An earlier draft documented a camelCase body with +// `type: "CodeBased"`; that shape is not what the deployed service accepts. +const CodeDefinitionType = "code" + +// evaluatorTypeCustom marks an evaluator as authored by the project rather +// than shipped by the platform. +const evaluatorTypeCustom = "custom" + +// foundryFeaturesHeader opts a request in to preview behaviour. blob_uri and +// entry_point are both declared as preview properties on the code definition, +// so the header is sent with every call that sets one. +const ( + foundryFeaturesHeader = "Foundry-Features" + foundryFeatureEvalsV1 = "Evaluations=V1Preview" + pendingUploadTypeBlob = "BlobReference" + defaultCodeMetricName = "result" + defaultCodeMetricType = "continuous" + defaultMetricDirection = "increase" + firstEvaluatorVersion = "1" + blobTypeHeader = "x-ms-blob-type" + blobTypeBlock = "BlockBlob" + octetStreamContentType = "application/octet-stream" +) + +// DefaultCodeMetrics is used when neither the folder nor the caller declares +// any. +// +// The service rejects a code definition carrying no metrics, and the documented +// evaluator output is a JSON object whose `result` field holds the score, so +// this describes exactly that. It is a default, not a constraint: any declared +// metrics replace it wholesale. +var DefaultCodeMetrics = json.RawMessage(fmt.Sprintf( + `{%q:{"type":%q,"desirable_direction":%q,"is_primary":true}}`, + defaultCodeMetricName, defaultCodeMetricType, defaultMetricDirection, +)) + +// CodeEvaluatorOptions carries the parts of an evaluator version that do not +// come from the Python source itself. +type CodeEvaluatorOptions struct { + DisplayName string + Description string + Categories []string + InitParameters json.RawMessage + DataSchema json.RawMessage + Metrics json.RawMessage +} + +// codeDefinition is the wire body of a code evaluator definition. +// +// The contract also allows code_text in place of blob_uri, and requires exactly +// one of the two. Only blob_uri is modelled here, because only blob_uri is +// sent; a field that is never populated would suggest a supported alternative +// that has not been exercised. +type codeDefinition struct { + Type string `json:"type"` + EntryPoint string `json:"entry_point,omitempty"` + BlobURI string `json:"blob_uri,omitempty"` + InitParameters json.RawMessage `json:"init_parameters,omitempty"` + DataSchema json.RawMessage `json:"data_schema,omitempty"` + Metrics json.RawMessage `json:"metrics,omitempty"` +} + +// createEvaluatorVersionRequest is the POST body for a new evaluator version. +// The service assigns the version; it is not carried here. +type createEvaluatorVersionRequest struct { + Name string `json:"name,omitempty"` + DisplayName string `json:"display_name,omitempty"` + Description string `json:"description,omitempty"` + EvaluatorType string `json:"evaluator_type,omitempty"` + Categories []string `json:"categories,omitempty"` + Definition *codeDefinition `json:"definition"` +} + +// pendingUploadRequest starts an upload for one evaluator version. +type pendingUploadRequest struct { + PendingUploadType string `json:"pendingUploadType"` +} + +// PendingUploadResponse is the reply to startPendingUpload: a container to +// write into, and the SAS that authorizes writing. +type PendingUploadResponse struct { + BlobReference *BlobReference `json:"blobReference,omitempty"` + PendingUploadID string `json:"pendingUploadId,omitempty"` + Version string `json:"version,omitempty"` +} + +// BlobReference is a storage location plus the credential to reach it. +type BlobReference struct { + BlobURI string `json:"blobUri,omitempty"` + StorageAccountARM string `json:"storageAccountArmId,omitempty"` + Credential *BlobCredential `json:"credential,omitempty"` +} + +// BlobCredential holds the SAS granted for an upload. +type BlobCredential struct { + Type string `json:"type,omitempty"` + SASUri string `json:"sasUri,omitempty"` +} + +// UploadURI returns the container URI carrying the SAS token, or empty when +// the service granted no credential. +func (p *PendingUploadResponse) UploadURI() string { + if p == nil || p.BlobReference == nil || p.BlobReference.Credential == nil { + return "" + } + return p.BlobReference.Credential.SASUri +} + +// ContainerURI returns the container URI without the SAS token. This is what +// the evaluator definition records, because the definition is persisted and a +// SAS in it would expire. +func (p *PendingUploadResponse) ContainerURI() string { + if p == nil || p.BlobReference == nil { + return "" + } + return p.BlobReference.BlobURI +} + +// UploadCodeEvaluatorVersion publishes a folder of Python as a new version of +// a code evaluator. +// +// Every package goes through storage, including a package of one file. The +// contract offers code_text as an alternative and it would save a round trip, +// but nothing observable confirms the executor runs it: the hand-off to the +// evaluation runtime drops both code_text and blob_uri and refetches from the +// catalog, so RAISvc does not reveal which one it prefers. blob_uri is the one +// with a demonstrated consumer, which enumerates the container and reads the +// files back. Choosing the unproven path would trade a saved upload for an +// evaluator that registers cleanly and then fails when it is finally run, +// which is far harder to diagnose than a slow publish. Revisit once the live +// test has actually exercised inline source. +func (c *EvalClient) UploadCodeEvaluatorVersion( + ctx context.Context, + pkg *evalcore.CodeEvaluatorPackage, + opts CodeEvaluatorOptions, + apiVersion string, +) (*EvaluatorVersion, error) { + if pkg == nil { + return nil, fmt.Errorf("no evaluator package to publish") + } + + definition := &codeDefinition{ + Type: CodeDefinitionType, + EntryPoint: pkg.EntryPoint, + InitParameters: opts.InitParameters, + DataSchema: opts.DataSchema, + Metrics: opts.Metrics, + } + if len(definition.Metrics) == 0 { + definition.Metrics = DefaultCodeMetrics + } + + blobURI, err := c.uploadCodeEvaluatorFiles(ctx, pkg, apiVersion) + if err != nil { + return nil, err + } + definition.BlobURI = blobURI + + body := &createEvaluatorVersionRequest{ + Name: pkg.Name, + DisplayName: opts.DisplayName, + Description: opts.Description, + EvaluatorType: evaluatorTypeCustom, + Categories: opts.Categories, + Definition: definition, + } + + path := pathEvaluators + "/" + url.PathEscape(pkg.Name) + "/versions" + respBody, err := c.doRequestWithHeaders( + ctx, http.MethodPost, path, nil, body, apiVersion, previewHeaders(), + ) + if err != nil { + return nil, err + } + + var created EvaluatorVersion + if len(respBody) > 0 { + if err := json.Unmarshal(respBody, &created); err != nil { + return nil, fmt.Errorf("failed to parse response: %w", err) + } + } + if created.Name == "" { + created.Name = pkg.Name + } + return &created, nil +} + +// uploadCodeEvaluatorFiles writes every file in the package to the container +// the service provisions for the version being created, and returns the +// container URI to record on the definition. +func (c *EvalClient) uploadCodeEvaluatorFiles( + ctx context.Context, + pkg *evalcore.CodeEvaluatorPackage, + apiVersion string, +) (string, error) { + version := c.NextEvaluatorVersion(ctx, pkg.Name, apiVersion) + + pending, err := c.StartEvaluatorPendingUpload(ctx, pkg.Name, version, apiVersion) + if err != nil { + return "", fmt.Errorf("starting the upload for evaluator %q: %w", pkg.Name, err) + } + + uploadURI := pending.UploadURI() + if uploadURI == "" { + return "", fmt.Errorf( + "the service returned no upload credential for evaluator %q", pkg.Name) + } + containerURI := pending.ContainerURI() + if containerURI == "" { + return "", fmt.Errorf( + "the service returned no storage location for evaluator %q", pkg.Name) + } + + for _, file := range pkg.Files { + content, err := os.ReadFile(file.AbsPath) + if err != nil { + return "", fmt.Errorf("reading %s: %w", file.RelPath, err) + } + if err := uploadBlob(ctx, uploadURI, file.RelPath, content); err != nil { + return "", fmt.Errorf("uploading %s: %w", file.RelPath, err) + } + } + + return containerURI, nil +} + +// StartEvaluatorPendingUpload provisions the storage an evaluator version's +// code is written to. +func (c *EvalClient) StartEvaluatorPendingUpload( + ctx context.Context, + name string, + version string, + apiVersion string, +) (*PendingUploadResponse, error) { + path := fmt.Sprintf( + "%s/%s/versions/%s/startPendingUpload", + pathEvaluators, url.PathEscape(name), url.PathEscape(version), + ) + respBody, err := c.doRequestWithHeaders( + ctx, http.MethodPost, path, nil, + &pendingUploadRequest{PendingUploadType: pendingUploadTypeBlob}, + apiVersion, previewHeaders(), + ) + if err != nil { + return nil, err + } + + var pending PendingUploadResponse + if len(respBody) > 0 { + if err := json.Unmarshal(respBody, &pending); err != nil { + return nil, fmt.Errorf("failed to parse response: %w", err) + } + } + return &pending, nil +} + +// NextEvaluatorVersion reports the version the service will assign to the next +// create. +// +// The upload has to name a version before the version exists, because storage +// is provisioned per version while the create that assigns it comes last. The +// service auto-increments, so the next one is the highest registered plus one. +// An unknown evaluator has none, which is version 1. +func (c *EvalClient) NextEvaluatorVersion( + ctx context.Context, + name string, + apiVersion string, +) string { + list, err := c.ListEvaluatorVersions(ctx, name, apiVersion) + if err != nil || list == nil || len(list.Value) == 0 { + return firstEvaluatorVersion + } + latest := pickLatestVersion(list.Value) + number, err := strconv.Atoi(latest) + if err != nil { + return firstEvaluatorVersion + } + return strconv.Itoa(number + 1) +} + +// LatestEvaluatorVersionNumber reports the newest registered version as an +// integer, or 0 when the evaluator is unknown or its versions are not numeric. +func (c *EvalClient) LatestEvaluatorVersionNumber( + ctx context.Context, + name string, + apiVersion string, +) int { + list, err := c.ListEvaluatorVersions(ctx, name, apiVersion) + if err != nil || list == nil || len(list.Value) == 0 { + return 0 + } + number, err := strconv.Atoi(pickLatestVersion(list.Value)) + if err != nil { + return 0 + } + return number +} + +// previewHeaders returns the opt-in header for the preview properties the code +// definition relies on. +func previewHeaders() map[string]string { + return map[string]string{foundryFeaturesHeader: foundryFeatureEvalsV1} +} + +// uploadBlob writes one file into a container using a container-level SAS. +// +// A plain client is used rather than the pipeline: the SAS in the URL is the +// credential, and the pipeline's bearer token policy would attach a Foundry +// token to a storage request that has no use for it. +func uploadBlob(ctx context.Context, containerSASUri, blobName string, data []byte) error { + u, err := url.Parse(containerSASUri) + if err != nil { + return fmt.Errorf("invalid container SAS URI: %w", err) + } + u.Path = strings.TrimSuffix(u.Path, "/") + "/" + blobName + + req, err := http.NewRequestWithContext(ctx, http.MethodPut, u.String(), bytes.NewReader(data)) + if err != nil { + return fmt.Errorf("failed to create upload request: %w", err) + } + req.Header.Set(blobTypeHeader, blobTypeBlock) + req.Header.Set("Content-Type", octetStreamContentType) + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return fmt.Errorf("failed to upload blob: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusCreated && resp.StatusCode != http.StatusOK { + // The body is Azure Storage XML, which says more than the status alone, + // but it is capped: a rejected upload can answer with a long document. + detail, _ := io.ReadAll(io.LimitReader(resp.Body, 512)) + return fmt.Errorf("blob upload failed with status %d: %s", + resp.StatusCode, strings.TrimSpace(string(detail))) + } + return nil +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/operations.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/operations.go index 5b2a74923bf..4ba2d628994 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/operations.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/operations.go @@ -284,6 +284,20 @@ func (c *EvalClient) doRequest( query map[string]string, body any, apiVersion string, +) ([]byte, error) { + return c.doRequestWithHeaders(ctx, method, path, query, body, apiVersion, nil) +} + +// doRequestWithHeaders is doRequest with extra request headers, which the +// preview evaluator operations need to opt in to the properties they set. +func (c *EvalClient) doRequestWithHeaders( + ctx context.Context, + method string, + path string, + query map[string]string, + body any, + apiVersion string, + headers map[string]string, ) ([]byte, error) { u, err := url.Parse(c.endpoint) if err != nil { @@ -304,6 +318,9 @@ func (c *EvalClient) doRequest( if err != nil { return nil, fmt.Errorf("failed to create request: %w", err) } + for k, v := range headers { + req.Raw().Header.Set(k, v) + } log.Printf("[eval_api] %s %s", method, u.Redacted()) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/codefolder.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/codefolder.go new file mode 100644 index 00000000000..a62391aa96f --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/codefolder.go @@ -0,0 +1,356 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package evalcore + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "io/fs" + "os" + "path/filepath" + "regexp" + "sort" + "strings" + "unicode" +) + +// CodeEvaluatorMetadataFile is the optional descriptor a folder can carry so +// the schemas travel with the code instead of having to be repeated on the +// command line every time it is published. +// +// It is not part of the service contract — the service only ever sees the +// definition assembled from it — so it is read leniently and its absence is +// normal. +const CodeEvaluatorMetadataFile = "evaluator.json" + +// pythonExt is the only source extension a code evaluator entry point can have. +const pythonExt = ".py" + +// excludedDirNames are directories that never carry evaluator source: build +// caches, version control, and dependency trees. Any dot-prefixed directory is +// excluded as well, which is checked separately. +var excludedDirNames = map[string]bool{ + "__pycache__": true, + ".git": true, + ".venv": true, + "venv": true, + "node_modules": true, +} + +// excludedFileExts are compiled Python artifacts. They are derived from the +// sources beside them, so uploading them adds nothing and — because they embed +// a timestamp — would make the folder fingerprint change on every rebuild. +var excludedFileExts = map[string]bool{ + ".pyc": true, + ".pyo": true, +} + +// CodeFile is one file in an evaluator package. +type CodeFile struct { + // RelPath is the path relative to the package root, always + // slash-separated. Windows and Linux must agree on it: it is both the blob + // name the file is uploaded under and part of the fingerprint, so a + // backslash here would republish the whole package on a change of machine. + RelPath string + + // AbsPath is where the file is read from on this machine. It is + // deliberately not part of the fingerprint. + AbsPath string +} + +// CodeEvaluatorMetadata is the optional descriptor read from +// CodeEvaluatorMetadataFile. Every field is optional; the raw JSON fields are +// passed to the service untouched so a schema this extension does not model +// still reaches it intact. +type CodeEvaluatorMetadata struct { + DisplayName string `json:"display_name,omitempty"` + Description string `json:"description,omitempty"` + Categories []string `json:"categories,omitempty"` + InitParameters json.RawMessage `json:"init_parameters,omitempty"` + DataSchema json.RawMessage `json:"data_schema,omitempty"` + Metrics json.RawMessage `json:"metrics,omitempty"` +} + +// CodeEvaluatorPackage is a validated folder ready to publish. +type CodeEvaluatorPackage struct { + // Name is the evaluator name the package was validated against. + Name string + // Root is the folder on disk. + Root string + // Files are the files to upload, in fingerprint order. + Files []CodeFile + // EntryPoint is the Python file holding the evaluator class. + EntryPoint string + // ClassName is the class the runtime instantiates. + ClassName string + // Metadata is the folder's descriptor, or nil when it carries none. + Metadata *CodeEvaluatorMetadata +} + +// IsCodeEvaluatorSource reports whether a declared source names a folder, and +// therefore a code evaluator rather than a rubric. +// +// The path is stat-ed rather than pattern-matched: a trailing separator is not +// required in YAML and `.json` in a folder name would misclassify it. +func IsCodeEvaluatorSource(path string) bool { + if path == "" { + return false + } + info, err := os.Stat(path) + return err == nil && info.IsDir() +} + +// EvaluatorClassName derives the class the runtime looks for from the +// evaluator name: the name in PascalCase, suffixed with Evaluator. +// +// The suffix is not appended twice, so both spellings customers use resolve to +// the same class — `answer_length` and `answer_length_evaluator` both mean +// AnswerLengthEvaluator. +func EvaluatorClassName(name string) string { + parts := strings.FieldsFunc(name, func(r rune) bool { + return r == '_' || r == '-' || r == '.' || r == ' ' + }) + + var b strings.Builder + for _, part := range parts { + runes := []rune(part) + b.WriteRune(unicode.ToUpper(runes[0])) + b.WriteString(string(runes[1:])) + } + + pascal := b.String() + if strings.HasSuffix(pascal, "Evaluator") { + return pascal + } + return pascal + "Evaluator" +} + +// WalkCodeFolder lists the files that make up an evaluator package, excluding +// build caches, version control, dependency trees, and compiled Python. +// +// The result is sorted by RelPath so the upload order and the fingerprint do +// not depend on directory iteration order, which the filesystem does not +// promise to keep stable. +func WalkCodeFolder(dir string) ([]CodeFile, error) { + info, err := os.Stat(dir) + if err != nil { + return nil, fmt.Errorf("reading evaluator folder %q: %w", dir, err) + } + if !info.IsDir() { + return nil, fmt.Errorf( + "evaluator source %q is a file; a code evaluator is published from a folder", dir) + } + + var files []CodeFile + walkErr := filepath.WalkDir(dir, func(path string, entry fs.DirEntry, err error) error { + if err != nil { + return err + } + + rel, relErr := filepath.Rel(dir, path) + if relErr != nil { + return relErr + } + rel = filepath.ToSlash(rel) + if rel == "." { + return nil + } + + if entry.IsDir() { + if isExcludedDir(entry.Name()) { + return fs.SkipDir + } + return nil + } + + // Symlinks, sockets and devices carry no content that can be uploaded, + // and WalkDir does not follow them, so they would otherwise be + // published as zero bytes. + if !entry.Type().IsRegular() { + return nil + } + // Dot-prefixed files are excluded for the same reason as dot-prefixed + // directories, and one reason more: a folder kept next to an evaluator + // tends to collect `.env`, `.netrc` and `.pypirc`, and publishing the + // package would copy those secrets into blob storage. Nothing a Python + // evaluator needs at runtime is named with a leading dot. + if strings.HasPrefix(entry.Name(), ".") { + return nil + } + if excludedFileExts[strings.ToLower(filepath.Ext(rel))] { + return nil + } + + files = append(files, CodeFile{RelPath: rel, AbsPath: path}) + return nil + }) + if walkErr != nil { + return nil, fmt.Errorf("reading evaluator folder %q: %w", dir, walkErr) + } + + sortCodeFiles(files) + return files, nil +} + +// isExcludedDir reports whether a directory is skipped along with everything +// under it. +func isExcludedDir(name string) bool { + return excludedDirNames[name] || strings.HasPrefix(name, ".") +} + +func sortCodeFiles(files []CodeFile) { + sort.Slice(files, func(i, j int) bool { return files[i].RelPath < files[j].RelPath }) +} + +// FingerprintCodeFolder hashes a folder so a later deploy can tell whether the +// evaluator changed without downloading anything from the service. +func FingerprintCodeFolder(dir string) (string, error) { + files, err := WalkCodeFolder(dir) + if err != nil { + return "", err + } + return FingerprintCodeFiles(files) +} + +// FingerprintCodeFiles hashes an already-walked package. +// +// Both the relative path and the content of every file are hashed, so renaming +// a file registers as a change even when the bytes are identical. The input is +// sorted first: the digest must describe the package, not the order the caller +// happened to hand the files over in. +// +// Only RelPath — never AbsPath — feeds the hash, and RelPath is normalized to +// forward slashes, so the same package hashes the same on Windows and Linux. +func FingerprintCodeFiles(files []CodeFile) (string, error) { + ordered := make([]CodeFile, len(files)) + copy(ordered, files) + sortCodeFiles(ordered) + + outer := sha256.New() + for _, file := range ordered { + content, err := os.ReadFile(file.AbsPath) + if err != nil { + return "", fmt.Errorf("hashing %q: %w", file.AbsPath, err) + } + inner := sha256.Sum256(content) + // Path and content digest are written on separate lines, which keeps + // the encoding unambiguous: a file's content digest is fixed width, so + // no path can be read as part of it. + fmt.Fprintf(outer, "%s\n%s\n", file.RelPath, hex.EncodeToString(inner[:])) + } + return hex.EncodeToString(outer.Sum(nil)), nil +} + +// LoadCodeEvaluator validates a folder against the packaging convention and +// returns the package to publish. +// +// The checks are done here rather than left to the service because the service +// only discovers a missing entry point when a run executes, long after a +// version has been published and an eval bound to it. +func LoadCodeEvaluator(name, dir string) (*CodeEvaluatorPackage, error) { + if name == "" { + return nil, fmt.Errorf("an evaluator name is required to validate the folder layout") + } + + files, err := WalkCodeFolder(dir) + if err != nil { + return nil, err + } + if len(files) == 0 { + return nil, fmt.Errorf( + "evaluator folder %q holds no files to publish", dir) + } + + entryPoint := name + pythonExt + className := EvaluatorClassName(name) + + entry, ok := findFile(files, entryPoint) + if !ok { + return nil, fmt.Errorf( + "evaluator %q needs %s in %s, holding a class named %s. The folder holds %s", + name, entryPoint, dir, className, describeFiles(files)) + } + + source, err := os.ReadFile(entry.AbsPath) + if err != nil { + return nil, fmt.Errorf("reading %s: %w", entry.RelPath, err) + } + if !declaresClass(source, className) { + return nil, fmt.Errorf( + "%s does not declare a class named %s. A code evaluator is a class with that "+ + "exact name and a __call__ method that takes **kwargs and returns a JSON "+ + "object, for example:\n\nclass %s:\n def __call__(self, **kwargs):\n"+ + " return {\"result\": 1}", + filepath.Join(dir, entry.RelPath), className, className) + } + + metadata, err := readCodeEvaluatorMetadata(files) + if err != nil { + return nil, err + } + + return &CodeEvaluatorPackage{ + Name: name, + Root: dir, + Files: files, + EntryPoint: entryPoint, + ClassName: className, + Metadata: metadata, + }, nil +} + +func findFile(files []CodeFile, relPath string) (CodeFile, bool) { + for _, file := range files { + if file.RelPath == relPath { + return file, true + } + } + return CodeFile{}, false +} + +// describeFiles renders the folder's contents for an error message, capped so +// a large package does not bury the advice that follows it. +func describeFiles(files []CodeFile) string { + const limit = 10 + names := make([]string, 0, limit) + for i, file := range files { + if i == limit { + return fmt.Sprintf("%s and %d more", strings.Join(names, ", "), len(files)-limit) + } + names = append(names, file.RelPath) + } + return strings.Join(names, ", ") +} + +// declaresClass reports whether the source declares the named class at any +// indentation, in either the bare or the inheriting form. +func declaresClass(source []byte, className string) bool { + pattern := `(?m)^[ \t]*class[ \t]+` + regexp.QuoteMeta(className) + `[ \t]*[(:]` + matched, err := regexp.Match(pattern, source) + return err == nil && matched +} + +// readCodeEvaluatorMetadata reads the optional folder descriptor. A folder +// without one is normal, so absence is not an error; malformed JSON is, since +// silently ignoring it would publish an evaluator missing the schemas the +// author wrote down. +func readCodeEvaluatorMetadata(files []CodeFile) (*CodeEvaluatorMetadata, error) { + entry, ok := findFile(files, CodeEvaluatorMetadataFile) + if !ok { + return nil, nil + } + + raw, err := os.ReadFile(entry.AbsPath) + if err != nil { + return nil, fmt.Errorf("reading %s: %w", entry.RelPath, err) + } + + var metadata CodeEvaluatorMetadata + if err := json.Unmarshal(raw, &metadata); err != nil { + return nil, fmt.Errorf("%s is not valid JSON: %w", entry.RelPath, err) + } + return &metadata, nil +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/codefolder_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/codefolder_test.go new file mode 100644 index 00000000000..b7e956bc3f9 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/codefolder_test.go @@ -0,0 +1,329 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package evalcore + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +// writeFile creates a file and every directory above it. +func writeFile(t *testing.T, root, rel, content string) string { + t.Helper() + path := filepath.Join(root, filepath.FromSlash(rel)) + require.NoError(t, os.MkdirAll(filepath.Dir(path), 0o755)) + require.NoError(t, os.WriteFile(path, []byte(content), 0o600)) + return path +} + +// evaluatorSource is a minimal evaluator matching the packaging convention. +func evaluatorSource(className string) string { + return "class " + className + ":\n" + + " def __call__(self, **kwargs):\n" + + " return {\"result\": len(kwargs.get(\"response\", \"\"))}\n" +} + +// relPaths reads the walk result into a comparable list. +func relPaths(files []CodeFile) []string { + out := make([]string, 0, len(files)) + for _, f := range files { + out = append(out, f.RelPath) + } + return out +} + +func TestWalkCodeFolder_ExcludesBuildAndDependencyTrees(t *testing.T) { + dir := t.TempDir() + + writeFile(t, dir, "answer_length.py", evaluatorSource("AnswerLengthEvaluator")) + writeFile(t, dir, "helpers/text.py", "def clean(s): return s.strip()\n") + writeFile(t, dir, "README.md", "docs\n") + + // Every one of these must be skipped, along with everything under it. + writeFile(t, dir, "__pycache__/answer_length.cpython-311.pyc", "cache") + writeFile(t, dir, ".git/config", "[core]") + writeFile(t, dir, ".venv/lib/site.py", "venv") + writeFile(t, dir, "venv/lib/site.py", "venv") + writeFile(t, dir, "node_modules/pkg/index.js", "js") + writeFile(t, dir, ".mypy_cache/report.json", "{}") + writeFile(t, dir, "helpers/__pycache__/text.cpython-311.pyc", "cache") + // Compiled artifacts are skipped wherever they sit, not just in caches. + writeFile(t, dir, "stale.pyc", "cache") + writeFile(t, dir, "stale.pyo", "cache") + + files, err := WalkCodeFolder(dir) + require.NoError(t, err) + + require.Equal(t, + []string{"README.md", "answer_length.py", "helpers/text.py"}, + relPaths(files), + "only source and data files belong in the package, in sorted order") +} + +// Dot-prefixed files are excluded because an evaluator folder living in a repo +// collects credential files, and publishing the package would copy them into +// blob storage. Excluding the directories alone is not enough: the ones that +// hold secrets sit at the root, next to the source. +func TestWalkCodeFolder_ExcludesDotFiles(t *testing.T) { + dir := t.TempDir() + + writeFile(t, dir, "answer_length.py", evaluatorSource("AnswerLengthEvaluator")) + writeFile(t, dir, ".env", "AZURE_OPENAI_API_KEY=super-secret\n") + writeFile(t, dir, ".netrc", "machine example.com password hunter2\n") + writeFile(t, dir, ".pypirc", "[pypi]\npassword = leaked\n") + writeFile(t, dir, "helpers/.env", "NESTED_SECRET=1\n") + + files, err := WalkCodeFolder(dir) + require.NoError(t, err) + + require.Equal(t, []string{"answer_length.py"}, relPaths(files), + "no dot-prefixed file may reach the upload, at any depth") + + for _, f := range files { + require.NotContains(t, f.RelPath, ".env") + require.NotContains(t, f.RelPath, ".netrc") + require.NotContains(t, f.RelPath, ".pypirc") + } +} + +// Relative paths are the blob names and part of the fingerprint, so they must +// not depend on the host's path separator. +func TestWalkCodeFolder_UsesForwardSlashes(t *testing.T) { + dir := t.TempDir() + writeFile(t, dir, "a/b/c.py", "x = 1\n") + + files, err := WalkCodeFolder(dir) + require.NoError(t, err) + require.Len(t, files, 1) + require.Equal(t, "a/b/c.py", files[0].RelPath) + require.NotContains(t, files[0].RelPath, "\\") +} + +func TestWalkCodeFolder_RejectsAFile(t *testing.T) { + dir := t.TempDir() + path := writeFile(t, dir, "rubric.json", "{}") + + _, err := WalkCodeFolder(path) + require.Error(t, err) + require.Contains(t, err.Error(), "folder") +} + +func TestEvaluatorClassName(t *testing.T) { + cases := map[string]string{ + "answer_length": "AnswerLengthEvaluator", + "answer-length": "AnswerLengthEvaluator", + "answer_length_evaluator": "AnswerLengthEvaluator", + "tone": "ToneEvaluator", + "ToneEvaluator": "ToneEvaluator", + "my.custom_check": "MyCustomCheckEvaluator", + } + for name, want := range cases { + require.Equal(t, want, EvaluatorClassName(name), "for %q", name) + } +} + +func TestLoadCodeEvaluator_AcceptsAConventionalFolder(t *testing.T) { + dir := t.TempDir() + writeFile(t, dir, "answer_length.py", evaluatorSource("AnswerLengthEvaluator")) + writeFile(t, dir, "helpers.py", "def n(s): return len(s)\n") + + pkg, err := LoadCodeEvaluator("answer_length", dir) + require.NoError(t, err) + require.Equal(t, "answer_length.py", pkg.EntryPoint) + require.Equal(t, "AnswerLengthEvaluator", pkg.ClassName) + require.Equal(t, []string{"answer_length.py", "helpers.py"}, relPaths(pkg.Files)) + require.Nil(t, pkg.Metadata) +} + +// The class may inherit, and may be indented inside a conditional. +func TestLoadCodeEvaluator_AcceptsInheritingClass(t *testing.T) { + dir := t.TempDir() + writeFile(t, dir, "tone.py", "import abc\n\nclass ToneEvaluator(abc.ABC):\n pass\n") + + pkg, err := LoadCodeEvaluator("tone", dir) + require.NoError(t, err) + require.Equal(t, "ToneEvaluator", pkg.ClassName) +} + +func TestLoadCodeEvaluator_ReportsMissingEntryPoint(t *testing.T) { + dir := t.TempDir() + writeFile(t, dir, "main.py", evaluatorSource("AnswerLengthEvaluator")) + + _, err := LoadCodeEvaluator("answer_length", dir) + require.Error(t, err) + // The message has to say what is missing, what it must hold, and what was + // actually found — a bare "not found" leaves the author guessing. + require.Contains(t, err.Error(), "answer_length.py") + require.Contains(t, err.Error(), "AnswerLengthEvaluator") + require.Contains(t, err.Error(), "main.py") +} + +func TestLoadCodeEvaluator_ReportsMissingClass(t *testing.T) { + dir := t.TempDir() + writeFile(t, dir, "tone.py", "class SomethingElse:\n pass\n") + + _, err := LoadCodeEvaluator("tone", dir) + require.Error(t, err) + require.Contains(t, err.Error(), "ToneEvaluator") + require.Contains(t, err.Error(), "__call__") +} + +func TestLoadCodeEvaluator_RejectsAnEmptyFolder(t *testing.T) { + dir := t.TempDir() + writeFile(t, dir, "__pycache__/x.pyc", "cache") + + _, err := LoadCodeEvaluator("tone", dir) + require.Error(t, err) + require.Contains(t, err.Error(), "no files") +} + +func TestLoadCodeEvaluator_ReadsFolderMetadata(t *testing.T) { + dir := t.TempDir() + writeFile(t, dir, "tone.py", evaluatorSource("ToneEvaluator")) + writeFile(t, dir, CodeEvaluatorMetadataFile, `{ + "display_name": "Tone", + "description": "Scores tone.", + "metrics": {"result": {"type": "ordinal", "min_value": 1, "max_value": 5}}, + "data_schema": {"type": "object", "properties": {"response": {"type": "string"}}} + }`) + + pkg, err := LoadCodeEvaluator("tone", dir) + require.NoError(t, err) + require.NotNil(t, pkg.Metadata) + require.Equal(t, "Tone", pkg.Metadata.DisplayName) + require.Contains(t, string(pkg.Metadata.Metrics), "ordinal") + require.Contains(t, string(pkg.Metadata.DataSchema), "response") +} + +// A descriptor that cannot be read is a mistake worth reporting: publishing +// without the schemas it declares would register an evaluator the author did +// not describe. +func TestLoadCodeEvaluator_RejectsMalformedMetadata(t *testing.T) { + dir := t.TempDir() + writeFile(t, dir, "tone.py", evaluatorSource("ToneEvaluator")) + writeFile(t, dir, CodeEvaluatorMetadataFile, "{not json") + + _, err := LoadCodeEvaluator("tone", dir) + require.Error(t, err) + require.Contains(t, err.Error(), CodeEvaluatorMetadataFile) +} + +func TestIsCodeEvaluatorSource(t *testing.T) { + dir := t.TempDir() + file := writeFile(t, dir, "rubric.json", "{}") + // A folder can be named like a file, so the decision cannot be made by + // looking at the string. + misleading := filepath.Join(dir, "looks_like.json") + require.NoError(t, os.MkdirAll(misleading, 0o755)) + + require.True(t, IsCodeEvaluatorSource(dir)) + require.True(t, IsCodeEvaluatorSource(misleading)) + require.False(t, IsCodeEvaluatorSource(file)) + require.False(t, IsCodeEvaluatorSource(filepath.Join(dir, "absent"))) + require.False(t, IsCodeEvaluatorSource("")) +} + +func TestFingerprintCodeFolder_IsStableAndSensitive(t *testing.T) { + dir := t.TempDir() + writeFile(t, dir, "answer_length.py", evaluatorSource("AnswerLengthEvaluator")) + writeFile(t, dir, "helpers/text.py", "def clean(s): return s.strip()\n") + + first, err := FingerprintCodeFolder(dir) + require.NoError(t, err) + + again, err := FingerprintCodeFolder(dir) + require.NoError(t, err) + require.Equal(t, first, again, "an unchanged folder must hash the same") + + // One byte. + writeFile(t, dir, "helpers/text.py", "def clean(s): return s.rstrip()\n") + changed, err := FingerprintCodeFolder(dir) + require.NoError(t, err) + require.NotEqual(t, first, changed, "changed content must hash differently") +} + +// Renaming a file changes the package even when every byte is preserved: the +// entry point is resolved by name and imports are written against it. +func TestFingerprintCodeFolder_NoticesARename(t *testing.T) { + dir := t.TempDir() + writeFile(t, dir, "tone.py", evaluatorSource("ToneEvaluator")) + writeFile(t, dir, "helpers.py", "X = 1\n") + + before, err := FingerprintCodeFolder(dir) + require.NoError(t, err) + + require.NoError(t, os.Rename( + filepath.Join(dir, "helpers.py"), filepath.Join(dir, "util.py"))) + + after, err := FingerprintCodeFolder(dir) + require.NoError(t, err) + require.NotEqual(t, before, after) +} + +// The filesystem does not promise a stable iteration order, so the digest must +// describe the package rather than the order it was handed over in. The input +// here is the production walk's own output, permuted — not a hand-built list. +func TestFingerprintCodeFiles_IgnoresInputOrder(t *testing.T) { + dir := t.TempDir() + writeFile(t, dir, "tone.py", evaluatorSource("ToneEvaluator")) + writeFile(t, dir, "helpers/a.py", "A = 1\n") + writeFile(t, dir, "helpers/b.py", "B = 2\n") + writeFile(t, dir, "data/prompts.txt", "hello\n") + + files, err := WalkCodeFolder(dir) + require.NoError(t, err) + require.Len(t, files, 4) + + fromFolder, err := FingerprintCodeFolder(dir) + require.NoError(t, err) + + reversed := make([]CodeFile, 0, len(files)) + for i := len(files) - 1; i >= 0; i-- { + reversed = append(reversed, files[i]) + } + fromReversed, err := FingerprintCodeFiles(reversed) + require.NoError(t, err) + require.Equal(t, fromFolder, fromReversed, "file order must not change the digest") + + rotated := append(append([]CodeFile{}, files[2:]...), files[:2]...) + fromRotated, err := FingerprintCodeFiles(rotated) + require.NoError(t, err) + require.Equal(t, fromFolder, fromRotated) +} + +// Where the folder sits must not affect the digest: two checkouts of the same +// repo, or the same repo on two machines, have to agree or every deploy +// republishes. +func TestFingerprintCodeFolder_IgnoresLocation(t *testing.T) { + build := func(root string) string { + writeFile(t, root, "tone.py", evaluatorSource("ToneEvaluator")) + writeFile(t, root, "helpers/a.py", "A = 1\n") + digest, err := FingerprintCodeFolder(root) + require.NoError(t, err) + return digest + } + + require.Equal(t, build(t.TempDir()), build(t.TempDir())) +} + +// Excluded content must not feed the digest, or a rebuild that only refreshes +// __pycache__ would look like an evaluator change and publish a version. +func TestFingerprintCodeFolder_IgnoresExcludedContent(t *testing.T) { + dir := t.TempDir() + writeFile(t, dir, "tone.py", evaluatorSource("ToneEvaluator")) + + before, err := FingerprintCodeFolder(dir) + require.NoError(t, err) + + writeFile(t, dir, "__pycache__/tone.cpython-311.pyc", strings.Repeat("x", 64)) + writeFile(t, dir, ".venv/lib/site.py", "noise") + + after, err := FingerprintCodeFolder(dir) + require.NoError(t, err) + require.Equal(t, before, after) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go index bfa923167c3..1748b98c0b4 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go @@ -33,6 +33,11 @@ type DatasetDecl struct { // EvaluatorDecl declares a custom evaluator. Built-ins are referenced directly // from an eval and never declared here. +// +// Source decides which kind of evaluator this is: a folder holds Python and +// publishes a code evaluator, a file holds a rubric. The distinction is made +// by stat-ing the path on deploy, not by its spelling, so a folder does not +// need a trailing separator to be recognised. type EvaluatorDecl struct { Name string `yaml:"name" json:"name"` Source string `yaml:"source,omitempty" json:"source,omitempty"` diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go index 4fb967c30a8..3e68eccd52c 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go @@ -13,6 +13,8 @@ import ( "path/filepath" "strings" + "azureaieval/internal/pkg/evalcore" + "github.com/azure/azure-dev/cli/azd/pkg/azdext" "github.com/azure/azure-dev/cli/azd/pkg/foundry" "google.golang.org/protobuf/types/known/structpb" @@ -313,6 +315,22 @@ func Fingerprint(path string) (string, error) { return hex.EncodeToString(sum[:]), nil } +// FingerprintPath hashes whatever a declared source names. +// +// A rubric evaluator is one JSON file; a code evaluator is a folder of Python. +// Both need change detection with the same meaning, so the artifact's shape is +// resolved by stat-ing it rather than by asking the caller to know. +func FingerprintPath(path string) (string, error) { + info, err := os.Stat(path) + if err != nil { + return "", fmt.Errorf("hashing %q: %w", path, err) + } + if info.IsDir() { + return evalcore.FingerprintCodeFolder(path) + } + return Fingerprint(path) +} + // FingerprintGroup hashes an eval's own declaration. // // Change detection on upstream artifacts is not sufficient: editing a group's From 41d1f8bc5f988ead41b4ef5fb57e80354bb0f033 Mon Sep 17 00:00:00 2001 From: mohessie Date: Thu, 30 Jul 2026 01:52:12 +0300 Subject: [PATCH 062/320] Fix code evaluator publishing against the live service The live round-trip had never been executed. Running it against a real project found three defects that reading the contract could not. pendingUploadType was sent as "BlobReference". The enum value is "TemporaryBlobReference", so the body failed to bind and the service answered 400 "The request field is required" - a null model, reported as a missing field rather than an invalid one. startPendingUpload answers with TemporaryDataReferenceResponseDto, which carries the location under blobReferenceForConsumption. Only blobReference was modelled, which datasets use, so the reply parsed into an empty struct and the upload failed for want of a credential. Both names are now accepted. Storage is reserved under a version guessed from the registered ones, and that list is eventually consistent: immediately after a publish it still reports the evaluator as unknown. The guess then collided with an existing version, which the service refuses with a 409 wrapped in a 500. A collision now advances the guess and retries instead of failing. Also relax the version assertion, which required the created version to equal the one storage was reserved under. There is no create-at-version route - the create assigns its own number and the reservation can take one too, so the two legitimately differ. Probing the service shows a reservation returns a container named by GUID, not by version, so a definition always points at exactly the bytes uploaded for it and the divergence is cosmetic. The test now asserts what actually matters: a second publish creates a new version, and the two versions keep separate storage. Verified against a live Foundry project: all three code evaluator round trips pass, and the full live suite is green. --- .../internal/cmd/evaluator_code_live_test.go | 36 +++++-- .../internal/pkg/eval_api/code_evaluator.go | 95 ++++++++++++++++--- 2 files changed, 112 insertions(+), 19 deletions(-) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator_code_live_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator_code_live_test.go index f9c20c6d07d..ee06d909750 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator_code_live_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator_code_live_test.go @@ -211,17 +211,39 @@ func TestLiveCodeEvaluatorPublishesNextVersion(t *testing.T) { _ = client.DeleteEvaluatorVersion( context.Background(), name, first.Version, ProjectEndpointAPIVersion) }) - require.Equal(t, predicted, first.Version, - "the service assigned a version other than the one storage was reserved under") - - // Publishing again must land on the next version, not overwrite the first. - predictedSecond := client.NextEvaluatorVersion(ctx, name, ProjectEndpointAPIVersion) + require.NotEmpty(t, first.Version, "the service must assign a version") + + // Publishing again must land on a new version, not overwrite the first. + // + // The version is deliberately not asserted to equal the one storage was + // reserved under. There is no create-at-version route: the create assigns + // its own number, and reserving storage can itself take a number, so the + // two legitimately differ. It does not matter, because a reservation + // returns a container named by GUID rather than by version, so a published + // definition always points at exactly the bytes uploaded for it. What must + // hold is that a second publish does not land on the first version. second, err := client.UploadCodeEvaluatorVersion(ctx, pkg, opts, ProjectEndpointAPIVersion) require.NoError(t, err) t.Cleanup(func() { _ = client.DeleteEvaluatorVersion( context.Background(), name, second.Version, ProjectEndpointAPIVersion) }) - require.Equal(t, predictedSecond, second.Version) - require.NotEqual(t, first.Version, second.Version) + require.NotEqual(t, first.Version, second.Version, + "a second publish must create a new version rather than replace the first") + + require.NotEqual(t, + blobURIOnService(t, client, name, first.Version), + blobURIOnService(t, client, name, second.Version), + "each version must keep its own storage, or republishing would rewrite the older one") +} + +// blobURIOnService reads back the storage location recorded on a version. +func blobURIOnService( + t *testing.T, + client *eval_api.EvalClient, + name string, + version string, +) string { + t.Helper() + return stringField(t, codeDefinitionOnService(t, client, name, version), "blob_uri") } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/code_evaluator.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/code_evaluator.go index 5ee2492faed..856df8ea19d 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/code_evaluator.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/code_evaluator.go @@ -38,7 +38,7 @@ const evaluatorTypeCustom = "custom" const ( foundryFeaturesHeader = "Foundry-Features" foundryFeatureEvalsV1 = "Evaluations=V1Preview" - pendingUploadTypeBlob = "BlobReference" + pendingUploadTypeBlob = "TemporaryBlobReference" defaultCodeMetricName = "result" defaultCodeMetricType = "continuous" defaultMetricDirection = "increase" @@ -46,6 +46,9 @@ const ( blobTypeHeader = "x-ms-blob-type" blobTypeBlock = "BlockBlob" octetStreamContentType = "application/octet-stream" + // maxVersionProbes bounds how far past a stale version guess the publish + // will step before giving up. + maxVersionProbes = 5 ) // DefaultCodeMetrics is used when neither the folder nor the caller declares @@ -104,10 +107,17 @@ type pendingUploadRequest struct { // PendingUploadResponse is the reply to startPendingUpload: a container to // write into, and the SAS that authorizes writing. +// +// The evaluator endpoint answers with TemporaryDataReferenceResponseDto, which +// carries the location under blobReferenceForConsumption. Datasets answer with +// blobReference instead, so both are modelled and whichever arrives is used: +// the two resources share this step but not the name they return it under. type PendingUploadResponse struct { - BlobReference *BlobReference `json:"blobReference,omitempty"` - PendingUploadID string `json:"pendingUploadId,omitempty"` - Version string `json:"version,omitempty"` + BlobReference *BlobReference `json:"blobReference,omitempty"` + BlobReferenceConsumption *BlobReference `json:"blobReferenceForConsumption,omitempty"` + PendingUploadID string `json:"pendingUploadId,omitempty"` + TemporaryDataReferenceID string `json:"temporaryDataReferenceId,omitempty"` + Version string `json:"version,omitempty"` } // BlobReference is a storage location plus the credential to reach it. @@ -123,23 +133,36 @@ type BlobCredential struct { SASUri string `json:"sasUri,omitempty"` } +// reference returns whichever of the two blob references the service populated. +func (p *PendingUploadResponse) reference() *BlobReference { + if p == nil { + return nil + } + if p.BlobReferenceConsumption != nil { + return p.BlobReferenceConsumption + } + return p.BlobReference +} + // UploadURI returns the container URI carrying the SAS token, or empty when // the service granted no credential. func (p *PendingUploadResponse) UploadURI() string { - if p == nil || p.BlobReference == nil || p.BlobReference.Credential == nil { + ref := p.reference() + if ref == nil || ref.Credential == nil { return "" } - return p.BlobReference.Credential.SASUri + return ref.Credential.SASUri } // ContainerURI returns the container URI without the SAS token. This is what // the evaluator definition records, because the definition is persisted and a // SAS in it would expire. func (p *PendingUploadResponse) ContainerURI() string { - if p == nil || p.BlobReference == nil { + ref := p.reference() + if ref == nil { return "" } - return p.BlobReference.BlobURI + return ref.BlobURI } // UploadCodeEvaluatorVersion publishes a folder of Python as a new version of @@ -219,11 +242,9 @@ func (c *EvalClient) uploadCodeEvaluatorFiles( pkg *evalcore.CodeEvaluatorPackage, apiVersion string, ) (string, error) { - version := c.NextEvaluatorVersion(ctx, pkg.Name, apiVersion) - - pending, err := c.StartEvaluatorPendingUpload(ctx, pkg.Name, version, apiVersion) + pending, err := c.reserveEvaluatorStorage(ctx, pkg.Name, apiVersion) if err != nil { - return "", fmt.Errorf("starting the upload for evaluator %q: %w", pkg.Name, err) + return "", err } uploadURI := pending.UploadURI() @@ -250,6 +271,56 @@ func (c *EvalClient) uploadCodeEvaluatorFiles( return containerURI, nil } +// reserveEvaluatorStorage provisions storage for the version being published, +// stepping past versions that are already taken. +// +// Storage has to be reserved under a version number before the version exists, +// and the number is guessed by reading the ones already registered. That read +// is eventually consistent: immediately after a publish it can still report +// the evaluator as unknown, which makes the guess collide with a version that +// is already there. The service answers a collision with a conflict, so the +// guess is advanced and tried again rather than failing the publish. +// +// Only a conflict is retried. Any other failure is the caller's to see. +func (c *EvalClient) reserveEvaluatorStorage( + ctx context.Context, + name string, + apiVersion string, +) (*PendingUploadResponse, error) { + version, err := strconv.Atoi(c.NextEvaluatorVersion(ctx, name, apiVersion)) + if err != nil { + version = 1 + } + + var lastErr error + for attempt := 0; attempt < maxVersionProbes; attempt++ { + pending, err := c.StartEvaluatorPendingUpload( + ctx, name, strconv.Itoa(version+attempt), apiVersion) + if err == nil { + return pending, nil + } + lastErr = err + if !isVersionConflict(err) { + break + } + } + return nil, fmt.Errorf("starting the upload for evaluator %q: %w", name, lastErr) +} + +// isVersionConflict reports whether a failed reservation was refused because +// the version already exists. +// +// The conflict is not surfaced as one: the service wraps the downstream 409 in +// a 500 whose message quotes the original status, so the status code on the +// response cannot be used and the message is what is left to read. +func isVersionConflict(err error) bool { + if err == nil { + return false + } + text := err.Error() + return strings.Contains(text, "409") || strings.Contains(text, "Conflict") +} + // StartEvaluatorPendingUpload provisions the storage an evaluator version's // code is written to. func (c *EvalClient) StartEvaluatorPendingUpload( From 32b01bab37636c9ceea7d79142e11c247060c6c4 Mon Sep 17 00:00:00 2001 From: mohessie Date: Thu, 30 Jul 2026 03:22:28 +0300 Subject: [PATCH 063/320] Publish code evaluators as a single script, not a folder A code evaluator runs as an OpenAI python grader, and the grader contract settles the shape. GraderPython in RAISvc (Contracts/UnifiedEvaluationV2/Grader.cs) carries Source - one string, "the source code of the python script" - plus ImageTag, PassThreshold and InitializationParameters. There is no folder, archive, file list or entry point, so a folder was never expressible. Live testing had already shown the consequences before the contract explained them: a definition carrying blob_uri alone registers cleanly and then fails the run with "Invalid grader source: top-level grade() function not found in source", while code_text alone completes and scores. A grader importing a sibling module fails; the same logic with the helper inlined passes. So --folder becomes --file, the definition sends code_text, and the evaluator-side blob upload is removed rather than left as a path that cannot reach the executor. Dataset uploads are untouched - they use a separate copy of that flow and still need it. Dependencies now have a supported route: --image-tag, which is how a grader gets anything it cannot inline. Validation follows the executor rather than the documentation. The folder convention required a {Name}Evaluator class with __call__; the runtime calls a top-level grade(). The class was never invoked, so requiring it only guaranteed a late failure. A script is now checked for a top-level grade(), anchored so a method of that name inside a class is correctly rejected. The evaluator.json descriptor is dropped. It was invented here, it has no counterpart in the SDK, and it conflicts with the spec.yaml convention the shipping evaluators in azureml-assets use. Schemas come from flags until that is settled. Also close a propagation race. After a publish, the version listing stays stale for ~650ms-1.4s while a direct read goes consistent in ~330ms, and eval creation follows the listing - so publishing an evaluator and immediately creating an eval that names it fails with "The evaluator X was not found". The reconciler now waits on both reads. The same lag makes two publishes seconds apart both return version 1, because the service's auto-increment reads the same stale listing. One earlier finding is reversed by this. A schema-less evaluator was recorded as refused; that refusal was the propagation 404 arriving at the same call and being read as a mapping error. With the race gone it is accepted with an empty data_mapping, and the test now says so. Verified live: a published script is referenced by an eval and scores its samples - completed, passed=2, errored=0. --- .../internal/cmd/evaluator.go | 125 +++--- .../internal/cmd/evaluator_code_live_test.go | 189 +++++----- .../internal/cmd/evaluator_code_test.go | 140 +++---- .../internal/cmd/reconciler.go | 117 +++++- .../internal/pkg/eval_api/code_evaluator.go | 325 ++-------------- .../internal/pkg/evalcore/codefolder.go | 356 ------------------ .../internal/pkg/evalcore/codefolder_test.go | 329 ---------------- .../internal/pkg/evalcore/codescript.go | 113 ++++++ .../internal/pkg/evalcore/codescript_test.go | 128 +++++++ .../internal/project/eval_config.go | 9 +- .../internal/project/service_target_eval.go | 21 +- .../tests/live/code_evaluator_run_test.go | 347 +++++++++++++++++ 12 files changed, 958 insertions(+), 1241 deletions(-) delete mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/codefolder.go delete mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/codefolder_test.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/codescript.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/codescript_test.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/tests/live/code_evaluator_run_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator.go index 6e15b0a6442..ee601bb16de 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator.go @@ -33,13 +33,14 @@ func newEvaluatorCommand() *cobra.Command { // version every time, so there is nothing for a separate `update` to do. // // An evaluator is either a rubric — a JSON file of weighted dimensions — or -// code — a folder of Python. They are different definition types on the wire, -// so exactly one of the two sources has to be named. +// code — one self-contained Python script. They are different definition types +// on the wire, so exactly one of the two sources has to be named. func newEvaluatorCreateCommand() *cobra.Command { var ( name string rubric string - folder string + file string + imageTag string initParams string dataSchema string metrics string @@ -48,28 +49,37 @@ func newEvaluatorCreateCommand() *cobra.Command { use := "create" short := "Register a rubric or code evaluator, publishing a new version." + long := short + "\n\n" + + "A rubric (--rubric) is a JSON file of weighted dimensions.\n\n" + + "A code evaluator (--file) is a single Python script declaring a top-level\n" + + "grade(sample, item) function that returns a float. It runs as a python\n" + + "grader, which is handed the script's source and nothing else: there is no\n" + + "package and no import path, so a helper module beside the script cannot be\n" + + "imported. Dependencies come from the image named by --image-tag." cmd := &cobra.Command{ Use: use, Short: short, + Long: long, RunE: func(cmd *cobra.Command, args []string) error { if name == "" { return requireFlag("name") } flags := codeEvaluatorFlags{ + imageTag: imageTag, initParams: initParams, dataSchema: dataSchema, metrics: metrics, endpoint: endpointFlg, } - if err := validateEvaluatorSource(rubric, folder, flags); err != nil { + if err := validateEvaluatorSource(rubric, file, flags); err != nil { return err } ctx := cmd.Context() - if folder != "" { - return runEvaluatorCreateFromFolder(cmd, name, folder, flags) + if file != "" { + return runEvaluatorCreateFromFile(cmd, name, file, flags) } raw, err := os.ReadFile(rubric) @@ -106,23 +116,24 @@ func newEvaluatorCreateCommand() *cobra.Command { cmd.Flags().StringVar(&name, "name", "", "Name of the evaluator.") cmd.Flags().StringVar(&rubric, "rubric", "", "Path to the rubric JSON file.") - cmd.Flags().StringVar(&folder, "folder", "", - "Path to a folder of Python holding the evaluator code.") + cmd.Flags().StringVar(&file, "file", "", + "Path to a single Python script declaring a top-level grade(sample, item) function.") + cmd.Flags().StringVar(&imageTag, "image-tag", "", + "Container image the evaluator runs in. Its packages are the only "+ + "dependencies the script can import beyond the standard library.") cmd.Flags().StringVar(&initParams, "init-params", "", - "Path to a JSON Schema for the evaluator's initialization parameters. "+ - "Overrides the folder's "+evalcore.CodeEvaluatorMetadataFile+".") + "Path to a JSON Schema for the evaluator's initialization parameters.") cmd.Flags().StringVar(&dataSchema, "data-schema", "", - "Path to a JSON Schema for the evaluator's input data. "+ - "Overrides the folder's "+evalcore.CodeEvaluatorMetadataFile+".") + "Path to a JSON Schema for the evaluator's input data.") cmd.Flags().StringVar(&metrics, "metrics", "", - "Path to a JSON object describing the metrics the evaluator produces. "+ - "Overrides the folder's "+evalcore.CodeEvaluatorMetadataFile+".") + "Path to a JSON object describing the metrics the evaluator produces.") cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") return cmd } -// codeEvaluatorFlags are the optional overrides for a code evaluator. +// codeEvaluatorFlags are the optional settings for a code evaluator. type codeEvaluatorFlags struct { + imageTag string initParams string dataSchema string metrics string @@ -130,59 +141,62 @@ type codeEvaluatorFlags struct { } // validateEvaluatorSource enforces that exactly one source is named, and that -// the schema overrides are only used with the source they apply to. +// the code-only settings are only used with the source they apply to. // // Deliberately checked here rather than with MarkFlagsMutuallyExclusive: that // only rejects the "both" case, and its message names a flag group rather than // saying what the two flags mean. Both mistakes deserve advice, and this is // testable without driving cobra. -func validateEvaluatorSource(rubric, folder string, flags codeEvaluatorFlags) error { +func validateEvaluatorSource(rubric, file string, flags codeEvaluatorFlags) error { switch { - case rubric == "" && folder == "": + case rubric == "" && file == "": return fmt.Errorf( - "one of --rubric or --folder is required: --rubric takes a JSON file of " + - "weighted dimensions, --folder takes a directory of Python") - case rubric != "" && folder != "": + "one of --rubric or --file is required: --rubric takes a JSON file of " + + "weighted dimensions, --file takes a single Python script") + case rubric != "" && file != "": return fmt.Errorf( - "--rubric and --folder cannot be used together: an evaluator is either a " + + "--rubric and --file cannot be used together: an evaluator is either a " + "rubric or code, not both") } - // A rubric's schemas are fixed by the service, so these would be accepted - // and then quietly dropped — the worst kind of no-op, because the author - // believes the evaluator was published carrying them. - if folder == "" { + // A rubric's schemas are fixed by the service and a rubric runs no code, so + // these would be accepted and then quietly dropped — the worst kind of + // no-op, because the author believes the evaluator was published carrying + // them. + if file == "" { for _, named := range []struct { flag string value string }{ + {"image-tag", flags.imageTag}, {"init-params", flags.initParams}, {"data-schema", flags.dataSchema}, {"metrics", flags.metrics}, } { if named.value != "" { return fmt.Errorf( - "--%s applies to a code evaluator and needs --folder; "+ - "a rubric's schemas are set by the service", named.flag) + "--%s applies to a code evaluator and needs --file; "+ + "a rubric runs no code and its schemas are set by the service", + named.flag) } } } return nil } -// runEvaluatorCreateFromFolder validates the folder, then publishes it. -func runEvaluatorCreateFromFolder( +// runEvaluatorCreateFromFile validates the script, then publishes it. +func runEvaluatorCreateFromFile( cmd *cobra.Command, name string, - folder string, + file string, flags codeEvaluatorFlags, ) error { - pkg, err := evalcore.LoadCodeEvaluator(name, folder) + script, err := evalcore.LoadCodeEvaluator(name, file) if err != nil { return err } - opts, err := codeEvaluatorOptions(pkg, flags) + opts, err := codeEvaluatorOptions(flags) if err != nil { return err } @@ -194,8 +208,8 @@ func runEvaluatorCreateFromFolder( } defer ec.Close() - created, err := ec.evalClient.UploadCodeEvaluatorVersion( - ctx, pkg, opts, ProjectEndpointAPIVersion, + created, err := ec.evalClient.CreateCodeEvaluatorVersion( + ctx, script, opts, ProjectEndpointAPIVersion, ) if err != nil { return fmt.Errorf("publishing evaluator %q: %w", name, err) @@ -205,32 +219,21 @@ func runEvaluatorCreateFromFolder( return emitJSON(cmd.OutOrStdout(), created) } fmt.Fprintf(cmd.OutOrStdout(), - "Published evaluator %s version %s from %d file(s) in %s\n", - created.Name, created.Version, len(pkg.Files), folder) + "Published evaluator %s version %s from %s\n", + created.Name, created.Version, file) return nil } -// codeEvaluatorOptions resolves the evaluator's schemas, preferring an -// explicit flag over whatever the folder declares. +// codeEvaluatorOptions resolves the evaluator's schemas from the flags. // -// The folder is the better place for them — they describe the code and belong -// beside it — but a folder that has none must still be publishable without -// editing it, which is what the flags are for. -func codeEvaluatorOptions( - pkg *evalcore.CodeEvaluatorPackage, - flags codeEvaluatorFlags, -) (eval_api.CodeEvaluatorOptions, error) { - var opts eval_api.CodeEvaluatorOptions - if md := pkg.Metadata; md != nil { - opts.DisplayName = md.DisplayName - opts.Description = md.Description - opts.Categories = md.Categories - opts.InitParameters = md.InitParameters - opts.DataSchema = md.DataSchema - opts.Metrics = md.Metrics - } - - for _, override := range []struct { +// They are not read from the script and not read from a descriptor beside it: +// the grader is handed one file of source, so anything the service needs that +// is not Python has to be named on the command line or carried in the eval +// config. +func codeEvaluatorOptions(flags codeEvaluatorFlags) (eval_api.CodeEvaluatorOptions, error) { + opts := eval_api.CodeEvaluatorOptions{ImageTag: flags.imageTag} + + for _, declared := range []struct { path string flag string field *json.RawMessage @@ -239,14 +242,14 @@ func codeEvaluatorOptions( {flags.dataSchema, "data-schema", &opts.DataSchema}, {flags.metrics, "metrics", &opts.Metrics}, } { - if override.path == "" { + if declared.path == "" { continue } - raw, err := readJSONObject(override.path) + raw, err := readJSONObject(declared.path) if err != nil { - return opts, fmt.Errorf("--%s %q: %w", override.flag, override.path, err) + return opts, fmt.Errorf("--%s %q: %w", declared.flag, declared.path, err) } - *override.field = raw + *declared.field = raw } return opts, nil diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator_code_live_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator_code_live_test.go index ee06d909750..df32ba7cd8d 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator_code_live_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator_code_live_test.go @@ -40,28 +40,19 @@ func liveCodeEvaluatorName(t *testing.T, suffix string) string { return fmt.Sprintf("azdcode_%s_%d", suffix, time.Now().UnixNano()) } -// writeLiveEvaluator lays out a folder to the packaging convention and returns -// it. The source is written for the derived class name so the production +// writeLiveEvaluator writes a self-contained evaluator script and returns its +// path. It goes through the production loader afterwards, so the shipping // validation is exercised rather than bypassed. -func writeLiveEvaluator(t *testing.T, name string, extraFiles map[string]string) string { +func writeLiveEvaluator(t *testing.T, name string) string { t.Helper() dir := t.TempDir() + path := filepath.Join(dir, name+".py") - className := evalcore.EvaluatorClassName(name) - entry := fmt.Sprintf(`class %s: - def __call__(self, **kwargs): - return {"result": float(len(kwargs.get("response", "")))} -`, className) - - require.NoError(t, os.WriteFile( - filepath.Join(dir, name+".py"), []byte(entry), 0o600)) - - for rel, content := range extraFiles { - path := filepath.Join(dir, filepath.FromSlash(rel)) - require.NoError(t, os.MkdirAll(filepath.Dir(path), 0o755)) - require.NoError(t, os.WriteFile(path, []byte(content), 0o600)) - } - return dir + source := `def grade(sample, item) -> float: + return float(len((item or {}).get("response", ""))) +` + require.NoError(t, os.WriteFile(path, []byte(source), 0o600)) + return path } // codeDefinitionOnService reads the registered version back and returns its @@ -101,30 +92,31 @@ func stringField(t *testing.T, definition map[string]json.RawMessage, key string return value } -// TestLiveCodeEvaluatorSingleFileRoundTrip publishes a one-file evaluator and -// asserts it comes back as a code definition pointing at storage. +// TestLiveCodeEvaluatorRoundTrip publishes a script and asserts it comes back +// as a code definition carrying the source inline. // -// A single file takes the same path as a folder. The contract also accepts -// inline source through code_text, which would save the upload, but nothing -// observable confirms the executor runs it, so the CLI does not send it. If -// that is ever settled, this is the test that should change first. -func TestLiveCodeEvaluatorSingleFileRoundTrip(t *testing.T) { +// code_text is the only source property that reaches the executor: the +// definition is consumed as an OpenAI python grader, whose contract is a +// single Source string. A version registered with blob_uri instead publishes +// cleanly and then fails every run with "top-level grade() function not found +// in source", so what matters here is that the source itself round-trips. +func TestLiveCodeEvaluatorRoundTrip(t *testing.T) { client, _ := liveEvalClient(t) ctx := context.Background() - name := liveCodeEvaluatorName(t, "single") - dir := writeLiveEvaluator(t, name, nil) + name := liveCodeEvaluatorName(t, "roundtrip") + path := writeLiveEvaluator(t, name) - // The shipping loader, not a hand-built package: this test has to fail if - // the production path stops producing a valid package. - pkg, err := evalcore.LoadCodeEvaluator(name, dir) + // The shipping loader, not a hand-built script: this test has to fail if + // the production path stops producing a publishable script. + script, err := evalcore.LoadCodeEvaluator(name, path) require.NoError(t, err) - require.Len(t, pkg.Files, 1) + require.Contains(t, script.Source, "def grade(") - opts, err := codeEvaluatorOptions(pkg, codeEvaluatorFlags{}) + opts, err := codeEvaluatorOptions(codeEvaluatorFlags{}) require.NoError(t, err) - created, err := client.UploadCodeEvaluatorVersion(ctx, pkg, opts, ProjectEndpointAPIVersion) + created, err := client.CreateCodeEvaluatorVersion(ctx, script, opts, ProjectEndpointAPIVersion) require.NoError(t, err, "the service rejected the code evaluator body") require.NotEmpty(t, created.Version) t.Cleanup(func() { @@ -136,76 +128,79 @@ func TestLiveCodeEvaluatorSingleFileRoundTrip(t *testing.T) { require.Equal(t, eval_api.CodeDefinitionType, stringField(t, definition, "type"), "the discriminator must round-trip as the lowercase snake_case value") - require.NotEmpty(t, stringField(t, definition, "blob_uri"), - "a published evaluator must record the storage location it was uploaded to") + require.Contains(t, stringField(t, definition, "code_text"), "def grade(", + "the source must round-trip inline; an empty code_text means the grader "+ + "would be handed nothing to run") require.Contains(t, definition, "metrics", "a code definition must carry metrics; the service rejects one without") } -// TestLiveCodeEvaluatorFolderRoundTrip publishes a multi-file evaluator and -// asserts the service records the storage location it handed out. +// TestLiveCodeEvaluatorCarriesSchemasAndImage proves the settings that only +// reach the service through flags survive the round trip. // -// This is the path that exercises startPendingUpload, the SAS write, and the -// blob_uri property, none of which the single-file case touches. -func TestLiveCodeEvaluatorFolderRoundTrip(t *testing.T) { +// They cannot come from anywhere else. The grader is handed one file of +// source, so a descriptor beside the script would never travel with it, and an +// image tag dropped on the way would leave an evaluator whose imports fail at +// run time with no sign of why. +func TestLiveCodeEvaluatorCarriesSchemasAndImage(t *testing.T) { client, _ := liveEvalClient(t) ctx := context.Background() - name := liveCodeEvaluatorName(t, "folder") - dir := writeLiveEvaluator(t, name, map[string]string{ - "helpers/text.py": "def clean(value):\n return value.strip()\n", - // Must be excluded from both the upload and the fingerprint. - "__pycache__/stale.pyc": "cache", - }) + name := liveCodeEvaluatorName(t, "schemas") + path := writeLiveEvaluator(t, name) - pkg, err := evalcore.LoadCodeEvaluator(name, dir) + script, err := evalcore.LoadCodeEvaluator(name, path) require.NoError(t, err) - require.Len(t, pkg.Files, 2, "the compiled artifact must not be part of the package") - opts, err := codeEvaluatorOptions(pkg, codeEvaluatorFlags{}) + dir := t.TempDir() + schemaPath := filepath.Join(dir, "schema.json") + require.NoError(t, os.WriteFile(schemaPath, []byte( + `{"type":"object","properties":{"response":{"type":"string"}},"required":["response"]}`, + ), 0o600)) + + opts, err := codeEvaluatorOptions(codeEvaluatorFlags{dataSchema: schemaPath}) require.NoError(t, err) + require.NotEmpty(t, opts.DataSchema) - created, err := client.UploadCodeEvaluatorVersion(ctx, pkg, opts, ProjectEndpointAPIVersion) - require.NoError(t, err, "the service rejected the uploaded code evaluator") - require.NotEmpty(t, created.Version) + created, err := client.CreateCodeEvaluatorVersion(ctx, script, opts, ProjectEndpointAPIVersion) + require.NoError(t, err, "the service rejected a definition carrying a data schema") t.Cleanup(func() { _ = client.DeleteEvaluatorVersion( context.Background(), name, created.Version, ProjectEndpointAPIVersion) }) definition := codeDefinitionOnService(t, client, name, created.Version) - - require.Equal(t, eval_api.CodeDefinitionType, stringField(t, definition, "type")) - require.NotEmpty(t, stringField(t, definition, "blob_uri"), - "a multi-file evaluator must round-trip carrying the storage location; "+ - "an empty blob_uri means the service dropped the preview property") - require.Contains(t, definition, "metrics") + require.Contains(t, definition, "data_schema", + "the declared data schema must round-trip; without it the criteria builder "+ + "derives no data_mapping and the eval cannot be created") + require.Contains(t, string(definition["data_schema"]), "response") } -// TestLiveCodeEvaluatorPublishesNextVersion proves the version the upload -// reserves storage under is the one the create then assigns. +// TestLiveCodeEvaluatorPublishesANewVersion proves a second publish does not +// overwrite the first. +// +// Versions are immutable and evals bind to one, so a publish that replaced the +// previous version would silently change what every existing eval evaluates. // -// Storage is provisioned per version before the version exists, so the client -// has to predict it. A drift between the two would leave the code in one -// version's container and the definition on another. -func TestLiveCodeEvaluatorPublishesNextVersion(t *testing.T) { +// The wait between the two publishes is not padding. The service assigns the +// next version from its own listing, and that listing lags the create by about +// a second: two publishes issued back to back were both answered with version +// 1, the second overwriting the first. The reconciler waits for a published +// version to appear in the listing before it moves on, so this waits the same +// way — the assertion is about publishing twice, not about racing the service. +func TestLiveCodeEvaluatorPublishesANewVersion(t *testing.T) { client, _ := liveEvalClient(t) ctx := context.Background() name := liveCodeEvaluatorName(t, "versions") - dir := writeLiveEvaluator(t, name, map[string]string{ - "helpers.py": "VALUE = 1\n", - }) + path := writeLiveEvaluator(t, name) - pkg, err := evalcore.LoadCodeEvaluator(name, dir) + script, err := evalcore.LoadCodeEvaluator(name, path) require.NoError(t, err) - opts, err := codeEvaluatorOptions(pkg, codeEvaluatorFlags{}) + opts, err := codeEvaluatorOptions(codeEvaluatorFlags{}) require.NoError(t, err) - predicted := client.NextEvaluatorVersion(ctx, name, ProjectEndpointAPIVersion) - require.Equal(t, "1", predicted, "an unpublished evaluator starts at version 1") - - first, err := client.UploadCodeEvaluatorVersion(ctx, pkg, opts, ProjectEndpointAPIVersion) + first, err := client.CreateCodeEvaluatorVersion(ctx, script, opts, ProjectEndpointAPIVersion) require.NoError(t, err) t.Cleanup(func() { _ = client.DeleteEvaluatorVersion( @@ -213,16 +208,9 @@ func TestLiveCodeEvaluatorPublishesNextVersion(t *testing.T) { }) require.NotEmpty(t, first.Version, "the service must assign a version") - // Publishing again must land on a new version, not overwrite the first. - // - // The version is deliberately not asserted to equal the one storage was - // reserved under. There is no create-at-version route: the create assigns - // its own number, and reserving storage can itself take a number, so the - // two legitimately differ. It does not matter, because a reservation - // returns a container named by GUID rather than by version, so a published - // definition always points at exactly the bytes uploaded for it. What must - // hold is that a second publish does not land on the first version. - second, err := client.UploadCodeEvaluatorVersion(ctx, pkg, opts, ProjectEndpointAPIVersion) + awaitVersionListed(t, client, name, first.Version) + + second, err := client.CreateCodeEvaluatorVersion(ctx, script, opts, ProjectEndpointAPIVersion) require.NoError(t, err) t.Cleanup(func() { _ = client.DeleteEvaluatorVersion( @@ -230,20 +218,29 @@ func TestLiveCodeEvaluatorPublishesNextVersion(t *testing.T) { }) require.NotEqual(t, first.Version, second.Version, "a second publish must create a new version rather than replace the first") - - require.NotEqual(t, - blobURIOnService(t, client, name, first.Version), - blobURIOnService(t, client, name, second.Version), - "each version must keep its own storage, or republishing would rewrite the older one") } -// blobURIOnService reads back the storage location recorded on a version. -func blobURIOnService( - t *testing.T, - client *eval_api.EvalClient, - name string, - version string, -) string { +// awaitVersionListed blocks until a published version shows up in the version +// listing, which is the view the service's own version assignment reads. +func awaitVersionListed(t *testing.T, client *eval_api.EvalClient, name, version string) { t.Helper() - return stringField(t, codeDefinitionOnService(t, client, name, version), "blob_uri") + ctx := context.Background() + + start := time.Now() + for { + list, err := client.ListEvaluatorVersions(ctx, name, ProjectEndpointAPIVersion) + if err == nil && list != nil { + for _, entry := range list.Value { + if entry.Version == version { + t.Logf("version %s listed after %s", + version, time.Since(start).Round(time.Millisecond)) + return + } + } + } + if time.Since(start) > 30*time.Second { + t.Fatalf("version %s of evaluator %s never appeared in the listing", version, name) + } + time.Sleep(250 * time.Millisecond) + } } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator_code_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator_code_test.go index a919bd56a90..0b1ff9205c2 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator_code_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator_code_test.go @@ -25,9 +25,10 @@ func writeTestFile(t *testing.T, root, rel, content string) string { return path } -const toneEvaluatorSource = "class ToneEvaluator:\n" + - " def __call__(self, **kwargs):\n" + - " return {\"result\": 1}\n" +// toneEvaluatorSource is the shape the grader requires: one top-level +// grade(sample, item) returning a float. +const toneEvaluatorSource = "def grade(sample, item) -> float:\n" + + " return float(len((item or {}).get(\"response\", \"\")))\n" // An evaluator is either a rubric or code. Naming both, or neither, is a // mistake the command has to name precisely — the two flags take different @@ -36,22 +37,23 @@ func TestValidateEvaluatorSource(t *testing.T) { err := validateEvaluatorSource("", "", codeEvaluatorFlags{}) require.Error(t, err) require.Contains(t, err.Error(), "--rubric") - require.Contains(t, err.Error(), "--folder") + require.Contains(t, err.Error(), "--file") require.Contains(t, err.Error(), "required") - err = validateEvaluatorSource("rubric.json", "./evaluator", codeEvaluatorFlags{}) + err = validateEvaluatorSource("rubric.json", "evaluator.py", codeEvaluatorFlags{}) require.Error(t, err) require.Contains(t, err.Error(), "cannot be used together") require.NoError(t, validateEvaluatorSource("rubric.json", "", codeEvaluatorFlags{})) - require.NoError(t, validateEvaluatorSource("", "./evaluator", codeEvaluatorFlags{})) + require.NoError(t, validateEvaluatorSource("", "evaluator.py", codeEvaluatorFlags{})) } -// The schema overrides describe a code evaluator. Accepting them beside a +// The code-only settings describe a python grader. Accepting them beside a // rubric and dropping them would leave the author believing the evaluator was -// published carrying schemas it never had. +// published carrying an image and schemas it never had. func TestValidateEvaluatorSource_RejectsCodeFlagsOnARubric(t *testing.T) { for flag, flags := range map[string]codeEvaluatorFlags{ + "image-tag": {imageTag: "python:3.11"}, "init-params": {initParams: "init.json"}, "data-schema": {dataSchema: "schema.json"}, "metrics": {metrics: "metrics.json"}, @@ -59,10 +61,10 @@ func TestValidateEvaluatorSource_RejectsCodeFlagsOnARubric(t *testing.T) { err := validateEvaluatorSource("rubric.json", "", flags) require.Error(t, err, "for --%s", flag) require.Contains(t, err.Error(), "--"+flag) - require.Contains(t, err.Error(), "--folder") + require.Contains(t, err.Error(), "--file") - require.NoError(t, validateEvaluatorSource("", "./evaluator", flags), - "--%s is valid with --folder", flag) + require.NoError(t, validateEvaluatorSource("", "evaluator.py", flags), + "--%s is valid with --file", flag) } } @@ -70,7 +72,7 @@ func TestValidateEvaluatorSource_RejectsCodeFlagsOnARubric(t *testing.T) { // future refactor cannot leave the flags declared but unvalidated. func TestEvaluatorCreateRejectsBothSources(t *testing.T) { cmd := newEvaluatorCreateCommand() - cmd.SetArgs([]string{"--name", "tone", "--rubric", "r.json", "--folder", "./x"}) + cmd.SetArgs([]string{"--name", "tone", "--rubric", "r.json", "--file", "tone.py"}) cmd.SetOut(io.Discard) cmd.SetErr(io.Discard) cmd.SilenceUsage = true @@ -89,72 +91,80 @@ func TestEvaluatorCreateRejectsNeitherSource(t *testing.T) { err := cmd.Execute() require.Error(t, err) - require.Contains(t, err.Error(), "one of --rubric or --folder") + require.Contains(t, err.Error(), "one of --rubric or --file") } -// The folder is the natural home for the schemas, but a folder that carries -// none still has to be publishable without editing it. -func TestCodeEvaluatorOptions_FlagsOverrideFolderMetadata(t *testing.T) { +// A script with no top-level grade() must be refused before a version is +// published, and the refusal must come from the command rather than from a run +// that fails minutes later. +func TestEvaluatorCreateRejectsAScriptWithoutGrade(t *testing.T) { dir := t.TempDir() - writeTestFile(t, dir, "tone.py", toneEvaluatorSource) - writeTestFile(t, dir, evalcore.CodeEvaluatorMetadataFile, `{ - "display_name": "Tone", - "metrics": {"result": {"type": "ordinal"}}, - "data_schema": {"type": "object", "properties": {"a": {"type": "string"}}} - }`) - - pkg, err := evalcore.LoadCodeEvaluator("tone", dir) - require.NoError(t, err) + path := writeTestFile(t, dir, "tone.py", + "class ToneEvaluator:\n def __call__(self, **kwargs):\n return {\"result\": 1}\n") - // Nothing overridden: the folder wins. - opts, err := codeEvaluatorOptions(pkg, codeEvaluatorFlags{}) + cmd := newEvaluatorCreateCommand() + cmd.SetArgs([]string{"--name", "tone", "--file", path}) + cmd.SetOut(io.Discard) + cmd.SetErr(io.Discard) + cmd.SilenceUsage = true + + err := cmd.Execute() + require.Error(t, err) + require.Contains(t, err.Error(), "grade(sample, item)") +} + +// The flags are the only place a code evaluator's schemas can come from: the +// grader is handed one file of source, so nothing that is not Python can +// travel with it. +func TestCodeEvaluatorOptions_ReadsTheFlags(t *testing.T) { + empty, err := codeEvaluatorOptions(codeEvaluatorFlags{}) require.NoError(t, err) - require.Equal(t, "Tone", opts.DisplayName) - require.Contains(t, string(opts.Metrics), "ordinal") - require.Contains(t, string(opts.DataSchema), `"a"`) - require.Empty(t, opts.InitParameters) + require.Empty(t, empty.ImageTag) + require.Empty(t, empty.Metrics) + require.Empty(t, empty.DataSchema) + require.Empty(t, empty.InitParameters) - overrides := t.TempDir() - metricsPath := writeTestFile(t, overrides, "metrics.json", + dir := t.TempDir() + metricsPath := writeTestFile(t, dir, "metrics.json", `{"result":{"type":"continuous"}}`) - initPath := writeTestFile(t, overrides, "init.json", + initPath := writeTestFile(t, dir, "init.json", `{"type":"object","properties":{"deployment_name":{"type":"string"}}}`) + schemaPath := writeTestFile(t, dir, "schema.json", + `{"type":"object","properties":{"response":{"type":"string"}}}`) - opts, err = codeEvaluatorOptions(pkg, codeEvaluatorFlags{ + opts, err := codeEvaluatorOptions(codeEvaluatorFlags{ + imageTag: "mcr.microsoft.com/azureml/evaluator:latest", metrics: metricsPath, initParams: initPath, + dataSchema: schemaPath, }) require.NoError(t, err) + require.Equal(t, "mcr.microsoft.com/azureml/evaluator:latest", opts.ImageTag) require.Contains(t, string(opts.Metrics), "continuous") - require.NotContains(t, string(opts.Metrics), "ordinal") require.Contains(t, string(opts.InitParameters), "deployment_name") - // Untouched by the overrides. - require.Contains(t, string(opts.DataSchema), `"a"`) + require.Contains(t, string(opts.DataSchema), "response") } // A typo in a schema file must be reported against the flag that named it, // not discovered by the service after a version has been published. -func TestCodeEvaluatorOptions_RejectsMalformedOverride(t *testing.T) { +func TestCodeEvaluatorOptions_RejectsMalformedInput(t *testing.T) { dir := t.TempDir() - writeTestFile(t, dir, "tone.py", toneEvaluatorSource) - pkg, err := evalcore.LoadCodeEvaluator("tone", dir) - require.NoError(t, err) bad := writeTestFile(t, dir, "metrics.json", "[1,2,3]") - _, err = codeEvaluatorOptions(pkg, codeEvaluatorFlags{metrics: bad}) + _, err := codeEvaluatorOptions(codeEvaluatorFlags{metrics: bad}) require.Error(t, err) require.Contains(t, err.Error(), "--metrics") require.Contains(t, err.Error(), "JSON object") - _, err = codeEvaluatorOptions(pkg, codeEvaluatorFlags{ + _, err = codeEvaluatorOptions(codeEvaluatorFlags{ dataSchema: filepath.Join(dir, "absent.json"), }) require.Error(t, err) require.Contains(t, err.Error(), "--data-schema") } -// The service rejects a code definition carrying no metrics, so a folder that -// declares none still has to publish with one. +// The service rejects a code definition carrying no metrics, so a script +// published without any still has to publish with one. func TestDefaultCodeMetricsIsAJSONObject(t *testing.T) { var metrics map[string]map[string]any require.NoError(t, json.Unmarshal(eval_api.DefaultCodeMetrics, &metrics)) @@ -163,32 +173,30 @@ func TestDefaultCodeMetricsIsAJSONObject(t *testing.T) { require.Equal(t, "continuous", metrics["result"]["type"]) } -// Change detection has to work for both kinds of evaluator source, and the -// reconciler decides which by stat-ing the path. -func TestFingerprintPath_HandlesFilesAndFolders(t *testing.T) { +// Both kinds of evaluator source are one file, and the reconciler tells them +// apart by extension rather than by stat-ing the path. +func TestEvaluatorSourceClassificationAndFingerprint(t *testing.T) { root := t.TempDir() - file := writeTestFile(t, root, "rubric.json", `{"dimensions":[]}`) - fileDigest, err := project.FingerprintPath(file) - require.NoError(t, err) - plainDigest, err := project.Fingerprint(file) - require.NoError(t, err) - require.Equal(t, plainDigest, fileDigest, - "a file must hash the same through either entry point") + rubric := writeTestFile(t, root, "rubric.json", `{"dimensions":[]}`) + script := writeTestFile(t, root, "tone.py", toneEvaluatorSource) - dir := t.TempDir() - writeTestFile(t, dir, "tone.py", toneEvaluatorSource) - folderDigest, err := project.FingerprintPath(dir) + require.False(t, evalcore.IsCodeEvaluatorSource(rubric)) + require.True(t, evalcore.IsCodeEvaluatorSource(script)) + + rubricDigest, err := project.Fingerprint(rubric) + require.NoError(t, err) + scriptDigest, err := project.Fingerprint(script) require.NoError(t, err) - require.NotEmpty(t, folderDigest) - require.NotEqual(t, fileDigest, folderDigest) + require.NotEqual(t, rubricDigest, scriptDigest) - writeTestFile(t, dir, "helpers.py", "X = 1\n") - changed, err := project.FingerprintPath(dir) + // Editing the script must be noticed, or a deploy would reuse a version + // holding the old source. + writeTestFile(t, root, "tone.py", toneEvaluatorSource+"\n# tweak\n") + changed, err := project.Fingerprint(script) require.NoError(t, err) - require.NotEqual(t, folderDigest, changed, - "adding a file to the folder must change the digest") + require.NotEqual(t, scriptDigest, changed) - _, err = project.FingerprintPath(filepath.Join(root, "absent")) + _, err = project.Fingerprint(filepath.Join(root, "absent.py")) require.Error(t, err) } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go index 3c5c93234f9..687d1936baf 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go @@ -13,6 +13,7 @@ import ( "reflect" "strconv" "strings" + "time" "azureaieval/internal/pkg/dataset_api" "azureaieval/internal/pkg/evalcore" @@ -227,12 +228,12 @@ func (r *evalReconciler) latestDatasetVersion(ctx context.Context, name string) // EnsureEvaluator publishes a new version when the local definition differs // from what the service holds. // -// The two kinds of evaluator are told apart by what the source names, not by -// its spelling: a folder is code, a file is a rubric. They also detect change -// differently. A rubric definition comes back inline, so it is compared -// directly; code does not — the service returns a storage URI, never the -// source — so a fingerprint of the folder is kept in the azd environment, the -// same way datasets work. +// The two kinds of evaluator are told apart by the source's extension: `.py` +// is code, anything else is a rubric. They also detect change differently. A +// rubric definition comes back inline, so it is compared directly; a code +// definition's source is not read back in a form worth comparing, so a +// fingerprint of the script is kept in the azd environment, the same way +// datasets work. func (r *evalReconciler) EnsureEvaluator( ctx context.Context, decl project.EvaluatorDecl, @@ -250,11 +251,10 @@ func (r *evalReconciler) EnsureEvaluator( return versionFromRaw(raw, decl.Version), false, nil } - info, err := os.Stat(localPath) - if err != nil { + if _, err := os.Stat(localPath); err != nil { return "", false, fmt.Errorf("evaluator source %q: %w", localPath, err) } - if info.IsDir() { + if evalcore.IsCodeEvaluatorSource(localPath) { return r.ensureCodeEvaluator(ctx, decl, localPath) } @@ -283,11 +283,12 @@ func (r *evalReconciler) EnsureEvaluator( if err != nil { return "", false, err } + r.awaitEvaluatorReadable(ctx, decl.Name, created.Version) return created.Version, true, nil } -// ensureCodeEvaluator publishes a folder of Python only when its content -// changed since the last deploy. +// ensureCodeEvaluator publishes a Python script only when its content changed +// since the last deploy. // // Every publish is a new immutable version, so without this a repeated // `azd up` would leave a trail of identical versions and force every eval @@ -295,17 +296,17 @@ func (r *evalReconciler) EnsureEvaluator( func (r *evalReconciler) ensureCodeEvaluator( ctx context.Context, decl project.EvaluatorDecl, - dir string, + path string, ) (string, bool, error) { - // Validated before anything is uploaded: a folder missing its entry point - // or class is only rejected when a run executes, long after a version has + // Validated before anything is published: a script with no top-level + // grade() is only rejected when a run executes, long after a version has // been published and an eval bound to it. - pkg, err := evalcore.LoadCodeEvaluator(decl.Name, dir) + script, err := evalcore.LoadCodeEvaluator(decl.Name, path) if err != nil { return "", false, err } - digest, err := project.FingerprintPath(dir) + digest, err := project.Fingerprint(path) if err != nil { return "", false, err } @@ -323,23 +324,101 @@ func (r *evalReconciler) ensureCodeEvaluator( return recordedVersion, false, nil } - opts, err := codeEvaluatorOptions(pkg, codeEvaluatorFlags{}) + opts, err := codeEvaluatorOptions(codeEvaluatorFlags{}) if err != nil { return "", false, err } - created, err := r.ec.evalClient.UploadCodeEvaluatorVersion( - ctx, pkg, opts, ProjectEndpointAPIVersion, + created, err := r.ec.evalClient.CreateCodeEvaluatorVersion( + ctx, script, opts, ProjectEndpointAPIVersion, ) if err != nil { return "", false, err } + r.awaitEvaluatorReadable(ctx, decl.Name, created.Version) + _ = r.ec.setEnvValue(ctx, key, digest) _ = r.ec.setEnvValue(ctx, versionKey("evaluator", decl.Name), created.Version) return created.Version, true, nil } +// evaluatorPropagation bounds the wait for a freshly published evaluator to +// become usable. +// +// A create returns before the version is resolvable everywhere, and the very +// next step of a deploy is EnsureEval, which names the evaluator in a testing +// criterion. Creating the eval inside that window fails with "The evaluator X +// was not found" — a confusing error, because the evaluator was published +// seconds earlier and is plainly there by the time anyone looks. The observed +// gap is under a second, so the poll is frequent and the cap is generous +// enough to absorb a slow day without stalling a deploy on an evaluator that +// is genuinely missing. +const ( + evaluatorPropagationTimeout = 30 * time.Second + evaluatorPropagationInterval = 250 * time.Millisecond +) + +// awaitEvaluatorReadable polls until a published version is resolvable, or the +// cap passes. +// +// Two reads have to agree, because they are not backed by the same view. The +// direct read goes consistent almost immediately; the version listing lags it +// by seconds, the same way the dataset listing does. A live publish was +// observed reading back at 03:06:58 and still failing eval creation at +// 03:06:59, so waiting on the direct read alone leaves exactly the race this +// exists to close. The listing is the slower of the two and therefore the one +// worth waiting on. +// +// A timeout is not an error. The wait is a courtesy that makes the common case +// reliable; if it never succeeds, the create that follows will report the real +// problem with far more context than a wait that gave up could. +func (r *evalReconciler) awaitEvaluatorReadable(ctx context.Context, name, version string) { + if version == "" { + return + } + deadline := time.Now().Add(evaluatorPropagationTimeout) + for { + if r.evaluatorVersionResolvable(ctx, name, version) { + return + } + if time.Now().After(deadline) { + return + } + select { + case <-ctx.Done(): + return + case <-time.After(evaluatorPropagationInterval): + } + } +} + +// evaluatorVersionResolvable reports whether a version can be both read +// directly and found in the listing. +func (r *evalReconciler) evaluatorVersionResolvable( + ctx context.Context, + name, version string, +) bool { + if _, err := r.ec.evalClient.GetEvaluatorRaw( + ctx, name, version, ProjectEndpointAPIVersion, + ); err != nil { + return false + } + + list, err := r.ec.evalClient.ListEvaluatorVersions( + ctx, name, ProjectEndpointAPIVersion, + ) + if err != nil || list == nil { + return false + } + for _, entry := range list.Value { + if entry.Version == version { + return true + } + } + return false +} + // checkEvaluatorDrift fails when the service holds a newer version than the // one recorded at the last deploy. // diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/code_evaluator.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/code_evaluator.go index 856df8ea19d..f9e1305eea6 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/code_evaluator.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/code_evaluator.go @@ -4,16 +4,12 @@ package eval_api import ( - "bytes" "context" "encoding/json" "fmt" - "io" "net/http" "net/url" - "os" "strconv" - "strings" "azureaieval/internal/pkg/evalcore" ) @@ -23,7 +19,7 @@ import ( // // The wire shape is snake_case with a lowercase discriminator, matching // CodeBasedEvaluatorDefinition in the Foundry data-plane OpenAPI document -// (`type` enum ["code"], plus code_text, entry_point, blob_uri, init_parameters, +// (`type` enum ["code"], plus code_text, image_tag, init_parameters, // data_schema and metrics). An earlier draft documented a camelCase body with // `type: "CodeBased"`; that shape is not what the deployed service accepts. const CodeDefinitionType = "code" @@ -32,27 +28,18 @@ const CodeDefinitionType = "code" // than shipped by the platform. const evaluatorTypeCustom = "custom" -// foundryFeaturesHeader opts a request in to preview behaviour. blob_uri and -// entry_point are both declared as preview properties on the code definition, -// so the header is sent with every call that sets one. +// foundryFeaturesHeader opts a request in to preview behaviour. The code +// definition's properties are declared as preview, so the header is sent with +// every call that sets one. const ( foundryFeaturesHeader = "Foundry-Features" foundryFeatureEvalsV1 = "Evaluations=V1Preview" - pendingUploadTypeBlob = "TemporaryBlobReference" defaultCodeMetricName = "result" defaultCodeMetricType = "continuous" defaultMetricDirection = "increase" - firstEvaluatorVersion = "1" - blobTypeHeader = "x-ms-blob-type" - blobTypeBlock = "BlockBlob" - octetStreamContentType = "application/octet-stream" - // maxVersionProbes bounds how far past a stale version guess the publish - // will step before giving up. - maxVersionProbes = 5 ) -// DefaultCodeMetrics is used when neither the folder nor the caller declares -// any. +// DefaultCodeMetrics is used when the caller declares none. // // The service rejects a code definition carrying no metrics, and the documented // evaluator output is a JSON object whose `result` field holds the score, so @@ -69,6 +56,7 @@ type CodeEvaluatorOptions struct { DisplayName string Description string Categories []string + ImageTag string InitParameters json.RawMessage DataSchema json.RawMessage Metrics json.RawMessage @@ -76,14 +64,21 @@ type CodeEvaluatorOptions struct { // codeDefinition is the wire body of a code evaluator definition. // -// The contract also allows code_text in place of blob_uri, and requires exactly -// one of the two. Only blob_uri is modelled here, because only blob_uri is -// sent; a field that is never populated would suggest a supported alternative -// that has not been exercised. +// code_text carries the whole evaluator. The contract's other source property, +// blob_uri, is deliberately absent: the definition is consumed as an OpenAI +// python grader, whose contract (GraderPython) is a single `Source` string +// with no notion of a folder, archive, file list or entry point. A definition +// published with blob_uri alone registers cleanly and then fails the run with +// "Invalid grader source: top-level grade() function not found in source", +// because nothing reads the blob back into Source. +// +// image_tag is how a grader gets dependencies. Only one file is ever sent, so +// a helper module cannot travel with it and anything beyond the standard +// library has to already be in the image. type codeDefinition struct { Type string `json:"type"` - EntryPoint string `json:"entry_point,omitempty"` - BlobURI string `json:"blob_uri,omitempty"` + CodeText string `json:"code_text,omitempty"` + ImageTag string `json:"image_tag,omitempty"` InitParameters json.RawMessage `json:"init_parameters,omitempty"` DataSchema json.RawMessage `json:"data_schema,omitempty"` Metrics json.RawMessage `json:"metrics,omitempty"` @@ -100,97 +95,27 @@ type createEvaluatorVersionRequest struct { Definition *codeDefinition `json:"definition"` } -// pendingUploadRequest starts an upload for one evaluator version. -type pendingUploadRequest struct { - PendingUploadType string `json:"pendingUploadType"` -} - -// PendingUploadResponse is the reply to startPendingUpload: a container to -// write into, and the SAS that authorizes writing. -// -// The evaluator endpoint answers with TemporaryDataReferenceResponseDto, which -// carries the location under blobReferenceForConsumption. Datasets answer with -// blobReference instead, so both are modelled and whichever arrives is used: -// the two resources share this step but not the name they return it under. -type PendingUploadResponse struct { - BlobReference *BlobReference `json:"blobReference,omitempty"` - BlobReferenceConsumption *BlobReference `json:"blobReferenceForConsumption,omitempty"` - PendingUploadID string `json:"pendingUploadId,omitempty"` - TemporaryDataReferenceID string `json:"temporaryDataReferenceId,omitempty"` - Version string `json:"version,omitempty"` -} - -// BlobReference is a storage location plus the credential to reach it. -type BlobReference struct { - BlobURI string `json:"blobUri,omitempty"` - StorageAccountARM string `json:"storageAccountArmId,omitempty"` - Credential *BlobCredential `json:"credential,omitempty"` -} - -// BlobCredential holds the SAS granted for an upload. -type BlobCredential struct { - Type string `json:"type,omitempty"` - SASUri string `json:"sasUri,omitempty"` -} - -// reference returns whichever of the two blob references the service populated. -func (p *PendingUploadResponse) reference() *BlobReference { - if p == nil { - return nil - } - if p.BlobReferenceConsumption != nil { - return p.BlobReferenceConsumption - } - return p.BlobReference -} - -// UploadURI returns the container URI carrying the SAS token, or empty when -// the service granted no credential. -func (p *PendingUploadResponse) UploadURI() string { - ref := p.reference() - if ref == nil || ref.Credential == nil { - return "" - } - return ref.Credential.SASUri -} - -// ContainerURI returns the container URI without the SAS token. This is what -// the evaluator definition records, because the definition is persisted and a -// SAS in it would expire. -func (p *PendingUploadResponse) ContainerURI() string { - ref := p.reference() - if ref == nil { - return "" - } - return ref.BlobURI -} - -// UploadCodeEvaluatorVersion publishes a folder of Python as a new version of -// a code evaluator. +// CreateCodeEvaluatorVersion publishes a Python script as a new version of a +// code evaluator. // -// Every package goes through storage, including a package of one file. The -// contract offers code_text as an alternative and it would save a round trip, -// but nothing observable confirms the executor runs it: the hand-off to the -// evaluation runtime drops both code_text and blob_uri and refetches from the -// catalog, so RAISvc does not reveal which one it prefers. blob_uri is the one -// with a demonstrated consumer, which enumerates the container and reads the -// files back. Choosing the unproven path would trade a saved upload for an -// evaluator that registers cleanly and then fails when it is finally run, -// which is far harder to diagnose than a slow publish. Revisit once the live -// test has actually exercised inline source. -func (c *EvalClient) UploadCodeEvaluatorVersion( +// The source is sent inline. There is no upload step and no storage to +// reserve: the executor is handed a string of source, so a blob it would never +// read adds a round trip, a SAS write, and a failure mode in exchange for +// nothing that reaches the grader. +func (c *EvalClient) CreateCodeEvaluatorVersion( ctx context.Context, - pkg *evalcore.CodeEvaluatorPackage, + script *evalcore.CodeEvaluatorScript, opts CodeEvaluatorOptions, apiVersion string, ) (*EvaluatorVersion, error) { - if pkg == nil { - return nil, fmt.Errorf("no evaluator package to publish") + if script == nil { + return nil, fmt.Errorf("no evaluator script to publish") } definition := &codeDefinition{ Type: CodeDefinitionType, - EntryPoint: pkg.EntryPoint, + CodeText: script.Source, + ImageTag: opts.ImageTag, InitParameters: opts.InitParameters, DataSchema: opts.DataSchema, Metrics: opts.Metrics, @@ -199,14 +124,8 @@ func (c *EvalClient) UploadCodeEvaluatorVersion( definition.Metrics = DefaultCodeMetrics } - blobURI, err := c.uploadCodeEvaluatorFiles(ctx, pkg, apiVersion) - if err != nil { - return nil, err - } - definition.BlobURI = blobURI - body := &createEvaluatorVersionRequest{ - Name: pkg.Name, + Name: script.Name, DisplayName: opts.DisplayName, Description: opts.Description, EvaluatorType: evaluatorTypeCustom, @@ -214,7 +133,7 @@ func (c *EvalClient) UploadCodeEvaluatorVersion( Definition: definition, } - path := pathEvaluators + "/" + url.PathEscape(pkg.Name) + "/versions" + path := pathEvaluators + "/" + url.PathEscape(script.Name) + "/versions" respBody, err := c.doRequestWithHeaders( ctx, http.MethodPost, path, nil, body, apiVersion, previewHeaders(), ) @@ -229,152 +148,11 @@ func (c *EvalClient) UploadCodeEvaluatorVersion( } } if created.Name == "" { - created.Name = pkg.Name + created.Name = script.Name } return &created, nil } -// uploadCodeEvaluatorFiles writes every file in the package to the container -// the service provisions for the version being created, and returns the -// container URI to record on the definition. -func (c *EvalClient) uploadCodeEvaluatorFiles( - ctx context.Context, - pkg *evalcore.CodeEvaluatorPackage, - apiVersion string, -) (string, error) { - pending, err := c.reserveEvaluatorStorage(ctx, pkg.Name, apiVersion) - if err != nil { - return "", err - } - - uploadURI := pending.UploadURI() - if uploadURI == "" { - return "", fmt.Errorf( - "the service returned no upload credential for evaluator %q", pkg.Name) - } - containerURI := pending.ContainerURI() - if containerURI == "" { - return "", fmt.Errorf( - "the service returned no storage location for evaluator %q", pkg.Name) - } - - for _, file := range pkg.Files { - content, err := os.ReadFile(file.AbsPath) - if err != nil { - return "", fmt.Errorf("reading %s: %w", file.RelPath, err) - } - if err := uploadBlob(ctx, uploadURI, file.RelPath, content); err != nil { - return "", fmt.Errorf("uploading %s: %w", file.RelPath, err) - } - } - - return containerURI, nil -} - -// reserveEvaluatorStorage provisions storage for the version being published, -// stepping past versions that are already taken. -// -// Storage has to be reserved under a version number before the version exists, -// and the number is guessed by reading the ones already registered. That read -// is eventually consistent: immediately after a publish it can still report -// the evaluator as unknown, which makes the guess collide with a version that -// is already there. The service answers a collision with a conflict, so the -// guess is advanced and tried again rather than failing the publish. -// -// Only a conflict is retried. Any other failure is the caller's to see. -func (c *EvalClient) reserveEvaluatorStorage( - ctx context.Context, - name string, - apiVersion string, -) (*PendingUploadResponse, error) { - version, err := strconv.Atoi(c.NextEvaluatorVersion(ctx, name, apiVersion)) - if err != nil { - version = 1 - } - - var lastErr error - for attempt := 0; attempt < maxVersionProbes; attempt++ { - pending, err := c.StartEvaluatorPendingUpload( - ctx, name, strconv.Itoa(version+attempt), apiVersion) - if err == nil { - return pending, nil - } - lastErr = err - if !isVersionConflict(err) { - break - } - } - return nil, fmt.Errorf("starting the upload for evaluator %q: %w", name, lastErr) -} - -// isVersionConflict reports whether a failed reservation was refused because -// the version already exists. -// -// The conflict is not surfaced as one: the service wraps the downstream 409 in -// a 500 whose message quotes the original status, so the status code on the -// response cannot be used and the message is what is left to read. -func isVersionConflict(err error) bool { - if err == nil { - return false - } - text := err.Error() - return strings.Contains(text, "409") || strings.Contains(text, "Conflict") -} - -// StartEvaluatorPendingUpload provisions the storage an evaluator version's -// code is written to. -func (c *EvalClient) StartEvaluatorPendingUpload( - ctx context.Context, - name string, - version string, - apiVersion string, -) (*PendingUploadResponse, error) { - path := fmt.Sprintf( - "%s/%s/versions/%s/startPendingUpload", - pathEvaluators, url.PathEscape(name), url.PathEscape(version), - ) - respBody, err := c.doRequestWithHeaders( - ctx, http.MethodPost, path, nil, - &pendingUploadRequest{PendingUploadType: pendingUploadTypeBlob}, - apiVersion, previewHeaders(), - ) - if err != nil { - return nil, err - } - - var pending PendingUploadResponse - if len(respBody) > 0 { - if err := json.Unmarshal(respBody, &pending); err != nil { - return nil, fmt.Errorf("failed to parse response: %w", err) - } - } - return &pending, nil -} - -// NextEvaluatorVersion reports the version the service will assign to the next -// create. -// -// The upload has to name a version before the version exists, because storage -// is provisioned per version while the create that assigns it comes last. The -// service auto-increments, so the next one is the highest registered plus one. -// An unknown evaluator has none, which is version 1. -func (c *EvalClient) NextEvaluatorVersion( - ctx context.Context, - name string, - apiVersion string, -) string { - list, err := c.ListEvaluatorVersions(ctx, name, apiVersion) - if err != nil || list == nil || len(list.Value) == 0 { - return firstEvaluatorVersion - } - latest := pickLatestVersion(list.Value) - number, err := strconv.Atoi(latest) - if err != nil { - return firstEvaluatorVersion - } - return strconv.Itoa(number + 1) -} - // LatestEvaluatorVersionNumber reports the newest registered version as an // integer, or 0 when the evaluator is unknown or its versions are not numeric. func (c *EvalClient) LatestEvaluatorVersionNumber( @@ -398,38 +176,3 @@ func (c *EvalClient) LatestEvaluatorVersionNumber( func previewHeaders() map[string]string { return map[string]string{foundryFeaturesHeader: foundryFeatureEvalsV1} } - -// uploadBlob writes one file into a container using a container-level SAS. -// -// A plain client is used rather than the pipeline: the SAS in the URL is the -// credential, and the pipeline's bearer token policy would attach a Foundry -// token to a storage request that has no use for it. -func uploadBlob(ctx context.Context, containerSASUri, blobName string, data []byte) error { - u, err := url.Parse(containerSASUri) - if err != nil { - return fmt.Errorf("invalid container SAS URI: %w", err) - } - u.Path = strings.TrimSuffix(u.Path, "/") + "/" + blobName - - req, err := http.NewRequestWithContext(ctx, http.MethodPut, u.String(), bytes.NewReader(data)) - if err != nil { - return fmt.Errorf("failed to create upload request: %w", err) - } - req.Header.Set(blobTypeHeader, blobTypeBlock) - req.Header.Set("Content-Type", octetStreamContentType) - - resp, err := http.DefaultClient.Do(req) - if err != nil { - return fmt.Errorf("failed to upload blob: %w", err) - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusCreated && resp.StatusCode != http.StatusOK { - // The body is Azure Storage XML, which says more than the status alone, - // but it is capped: a rejected upload can answer with a long document. - detail, _ := io.ReadAll(io.LimitReader(resp.Body, 512)) - return fmt.Errorf("blob upload failed with status %d: %s", - resp.StatusCode, strings.TrimSpace(string(detail))) - } - return nil -} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/codefolder.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/codefolder.go deleted file mode 100644 index a62391aa96f..00000000000 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/codefolder.go +++ /dev/null @@ -1,356 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package evalcore - -import ( - "crypto/sha256" - "encoding/hex" - "encoding/json" - "fmt" - "io/fs" - "os" - "path/filepath" - "regexp" - "sort" - "strings" - "unicode" -) - -// CodeEvaluatorMetadataFile is the optional descriptor a folder can carry so -// the schemas travel with the code instead of having to be repeated on the -// command line every time it is published. -// -// It is not part of the service contract — the service only ever sees the -// definition assembled from it — so it is read leniently and its absence is -// normal. -const CodeEvaluatorMetadataFile = "evaluator.json" - -// pythonExt is the only source extension a code evaluator entry point can have. -const pythonExt = ".py" - -// excludedDirNames are directories that never carry evaluator source: build -// caches, version control, and dependency trees. Any dot-prefixed directory is -// excluded as well, which is checked separately. -var excludedDirNames = map[string]bool{ - "__pycache__": true, - ".git": true, - ".venv": true, - "venv": true, - "node_modules": true, -} - -// excludedFileExts are compiled Python artifacts. They are derived from the -// sources beside them, so uploading them adds nothing and — because they embed -// a timestamp — would make the folder fingerprint change on every rebuild. -var excludedFileExts = map[string]bool{ - ".pyc": true, - ".pyo": true, -} - -// CodeFile is one file in an evaluator package. -type CodeFile struct { - // RelPath is the path relative to the package root, always - // slash-separated. Windows and Linux must agree on it: it is both the blob - // name the file is uploaded under and part of the fingerprint, so a - // backslash here would republish the whole package on a change of machine. - RelPath string - - // AbsPath is where the file is read from on this machine. It is - // deliberately not part of the fingerprint. - AbsPath string -} - -// CodeEvaluatorMetadata is the optional descriptor read from -// CodeEvaluatorMetadataFile. Every field is optional; the raw JSON fields are -// passed to the service untouched so a schema this extension does not model -// still reaches it intact. -type CodeEvaluatorMetadata struct { - DisplayName string `json:"display_name,omitempty"` - Description string `json:"description,omitempty"` - Categories []string `json:"categories,omitempty"` - InitParameters json.RawMessage `json:"init_parameters,omitempty"` - DataSchema json.RawMessage `json:"data_schema,omitempty"` - Metrics json.RawMessage `json:"metrics,omitempty"` -} - -// CodeEvaluatorPackage is a validated folder ready to publish. -type CodeEvaluatorPackage struct { - // Name is the evaluator name the package was validated against. - Name string - // Root is the folder on disk. - Root string - // Files are the files to upload, in fingerprint order. - Files []CodeFile - // EntryPoint is the Python file holding the evaluator class. - EntryPoint string - // ClassName is the class the runtime instantiates. - ClassName string - // Metadata is the folder's descriptor, or nil when it carries none. - Metadata *CodeEvaluatorMetadata -} - -// IsCodeEvaluatorSource reports whether a declared source names a folder, and -// therefore a code evaluator rather than a rubric. -// -// The path is stat-ed rather than pattern-matched: a trailing separator is not -// required in YAML and `.json` in a folder name would misclassify it. -func IsCodeEvaluatorSource(path string) bool { - if path == "" { - return false - } - info, err := os.Stat(path) - return err == nil && info.IsDir() -} - -// EvaluatorClassName derives the class the runtime looks for from the -// evaluator name: the name in PascalCase, suffixed with Evaluator. -// -// The suffix is not appended twice, so both spellings customers use resolve to -// the same class — `answer_length` and `answer_length_evaluator` both mean -// AnswerLengthEvaluator. -func EvaluatorClassName(name string) string { - parts := strings.FieldsFunc(name, func(r rune) bool { - return r == '_' || r == '-' || r == '.' || r == ' ' - }) - - var b strings.Builder - for _, part := range parts { - runes := []rune(part) - b.WriteRune(unicode.ToUpper(runes[0])) - b.WriteString(string(runes[1:])) - } - - pascal := b.String() - if strings.HasSuffix(pascal, "Evaluator") { - return pascal - } - return pascal + "Evaluator" -} - -// WalkCodeFolder lists the files that make up an evaluator package, excluding -// build caches, version control, dependency trees, and compiled Python. -// -// The result is sorted by RelPath so the upload order and the fingerprint do -// not depend on directory iteration order, which the filesystem does not -// promise to keep stable. -func WalkCodeFolder(dir string) ([]CodeFile, error) { - info, err := os.Stat(dir) - if err != nil { - return nil, fmt.Errorf("reading evaluator folder %q: %w", dir, err) - } - if !info.IsDir() { - return nil, fmt.Errorf( - "evaluator source %q is a file; a code evaluator is published from a folder", dir) - } - - var files []CodeFile - walkErr := filepath.WalkDir(dir, func(path string, entry fs.DirEntry, err error) error { - if err != nil { - return err - } - - rel, relErr := filepath.Rel(dir, path) - if relErr != nil { - return relErr - } - rel = filepath.ToSlash(rel) - if rel == "." { - return nil - } - - if entry.IsDir() { - if isExcludedDir(entry.Name()) { - return fs.SkipDir - } - return nil - } - - // Symlinks, sockets and devices carry no content that can be uploaded, - // and WalkDir does not follow them, so they would otherwise be - // published as zero bytes. - if !entry.Type().IsRegular() { - return nil - } - // Dot-prefixed files are excluded for the same reason as dot-prefixed - // directories, and one reason more: a folder kept next to an evaluator - // tends to collect `.env`, `.netrc` and `.pypirc`, and publishing the - // package would copy those secrets into blob storage. Nothing a Python - // evaluator needs at runtime is named with a leading dot. - if strings.HasPrefix(entry.Name(), ".") { - return nil - } - if excludedFileExts[strings.ToLower(filepath.Ext(rel))] { - return nil - } - - files = append(files, CodeFile{RelPath: rel, AbsPath: path}) - return nil - }) - if walkErr != nil { - return nil, fmt.Errorf("reading evaluator folder %q: %w", dir, walkErr) - } - - sortCodeFiles(files) - return files, nil -} - -// isExcludedDir reports whether a directory is skipped along with everything -// under it. -func isExcludedDir(name string) bool { - return excludedDirNames[name] || strings.HasPrefix(name, ".") -} - -func sortCodeFiles(files []CodeFile) { - sort.Slice(files, func(i, j int) bool { return files[i].RelPath < files[j].RelPath }) -} - -// FingerprintCodeFolder hashes a folder so a later deploy can tell whether the -// evaluator changed without downloading anything from the service. -func FingerprintCodeFolder(dir string) (string, error) { - files, err := WalkCodeFolder(dir) - if err != nil { - return "", err - } - return FingerprintCodeFiles(files) -} - -// FingerprintCodeFiles hashes an already-walked package. -// -// Both the relative path and the content of every file are hashed, so renaming -// a file registers as a change even when the bytes are identical. The input is -// sorted first: the digest must describe the package, not the order the caller -// happened to hand the files over in. -// -// Only RelPath — never AbsPath — feeds the hash, and RelPath is normalized to -// forward slashes, so the same package hashes the same on Windows and Linux. -func FingerprintCodeFiles(files []CodeFile) (string, error) { - ordered := make([]CodeFile, len(files)) - copy(ordered, files) - sortCodeFiles(ordered) - - outer := sha256.New() - for _, file := range ordered { - content, err := os.ReadFile(file.AbsPath) - if err != nil { - return "", fmt.Errorf("hashing %q: %w", file.AbsPath, err) - } - inner := sha256.Sum256(content) - // Path and content digest are written on separate lines, which keeps - // the encoding unambiguous: a file's content digest is fixed width, so - // no path can be read as part of it. - fmt.Fprintf(outer, "%s\n%s\n", file.RelPath, hex.EncodeToString(inner[:])) - } - return hex.EncodeToString(outer.Sum(nil)), nil -} - -// LoadCodeEvaluator validates a folder against the packaging convention and -// returns the package to publish. -// -// The checks are done here rather than left to the service because the service -// only discovers a missing entry point when a run executes, long after a -// version has been published and an eval bound to it. -func LoadCodeEvaluator(name, dir string) (*CodeEvaluatorPackage, error) { - if name == "" { - return nil, fmt.Errorf("an evaluator name is required to validate the folder layout") - } - - files, err := WalkCodeFolder(dir) - if err != nil { - return nil, err - } - if len(files) == 0 { - return nil, fmt.Errorf( - "evaluator folder %q holds no files to publish", dir) - } - - entryPoint := name + pythonExt - className := EvaluatorClassName(name) - - entry, ok := findFile(files, entryPoint) - if !ok { - return nil, fmt.Errorf( - "evaluator %q needs %s in %s, holding a class named %s. The folder holds %s", - name, entryPoint, dir, className, describeFiles(files)) - } - - source, err := os.ReadFile(entry.AbsPath) - if err != nil { - return nil, fmt.Errorf("reading %s: %w", entry.RelPath, err) - } - if !declaresClass(source, className) { - return nil, fmt.Errorf( - "%s does not declare a class named %s. A code evaluator is a class with that "+ - "exact name and a __call__ method that takes **kwargs and returns a JSON "+ - "object, for example:\n\nclass %s:\n def __call__(self, **kwargs):\n"+ - " return {\"result\": 1}", - filepath.Join(dir, entry.RelPath), className, className) - } - - metadata, err := readCodeEvaluatorMetadata(files) - if err != nil { - return nil, err - } - - return &CodeEvaluatorPackage{ - Name: name, - Root: dir, - Files: files, - EntryPoint: entryPoint, - ClassName: className, - Metadata: metadata, - }, nil -} - -func findFile(files []CodeFile, relPath string) (CodeFile, bool) { - for _, file := range files { - if file.RelPath == relPath { - return file, true - } - } - return CodeFile{}, false -} - -// describeFiles renders the folder's contents for an error message, capped so -// a large package does not bury the advice that follows it. -func describeFiles(files []CodeFile) string { - const limit = 10 - names := make([]string, 0, limit) - for i, file := range files { - if i == limit { - return fmt.Sprintf("%s and %d more", strings.Join(names, ", "), len(files)-limit) - } - names = append(names, file.RelPath) - } - return strings.Join(names, ", ") -} - -// declaresClass reports whether the source declares the named class at any -// indentation, in either the bare or the inheriting form. -func declaresClass(source []byte, className string) bool { - pattern := `(?m)^[ \t]*class[ \t]+` + regexp.QuoteMeta(className) + `[ \t]*[(:]` - matched, err := regexp.Match(pattern, source) - return err == nil && matched -} - -// readCodeEvaluatorMetadata reads the optional folder descriptor. A folder -// without one is normal, so absence is not an error; malformed JSON is, since -// silently ignoring it would publish an evaluator missing the schemas the -// author wrote down. -func readCodeEvaluatorMetadata(files []CodeFile) (*CodeEvaluatorMetadata, error) { - entry, ok := findFile(files, CodeEvaluatorMetadataFile) - if !ok { - return nil, nil - } - - raw, err := os.ReadFile(entry.AbsPath) - if err != nil { - return nil, fmt.Errorf("reading %s: %w", entry.RelPath, err) - } - - var metadata CodeEvaluatorMetadata - if err := json.Unmarshal(raw, &metadata); err != nil { - return nil, fmt.Errorf("%s is not valid JSON: %w", entry.RelPath, err) - } - return &metadata, nil -} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/codefolder_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/codefolder_test.go deleted file mode 100644 index b7e956bc3f9..00000000000 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/codefolder_test.go +++ /dev/null @@ -1,329 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package evalcore - -import ( - "os" - "path/filepath" - "strings" - "testing" - - "github.com/stretchr/testify/require" -) - -// writeFile creates a file and every directory above it. -func writeFile(t *testing.T, root, rel, content string) string { - t.Helper() - path := filepath.Join(root, filepath.FromSlash(rel)) - require.NoError(t, os.MkdirAll(filepath.Dir(path), 0o755)) - require.NoError(t, os.WriteFile(path, []byte(content), 0o600)) - return path -} - -// evaluatorSource is a minimal evaluator matching the packaging convention. -func evaluatorSource(className string) string { - return "class " + className + ":\n" + - " def __call__(self, **kwargs):\n" + - " return {\"result\": len(kwargs.get(\"response\", \"\"))}\n" -} - -// relPaths reads the walk result into a comparable list. -func relPaths(files []CodeFile) []string { - out := make([]string, 0, len(files)) - for _, f := range files { - out = append(out, f.RelPath) - } - return out -} - -func TestWalkCodeFolder_ExcludesBuildAndDependencyTrees(t *testing.T) { - dir := t.TempDir() - - writeFile(t, dir, "answer_length.py", evaluatorSource("AnswerLengthEvaluator")) - writeFile(t, dir, "helpers/text.py", "def clean(s): return s.strip()\n") - writeFile(t, dir, "README.md", "docs\n") - - // Every one of these must be skipped, along with everything under it. - writeFile(t, dir, "__pycache__/answer_length.cpython-311.pyc", "cache") - writeFile(t, dir, ".git/config", "[core]") - writeFile(t, dir, ".venv/lib/site.py", "venv") - writeFile(t, dir, "venv/lib/site.py", "venv") - writeFile(t, dir, "node_modules/pkg/index.js", "js") - writeFile(t, dir, ".mypy_cache/report.json", "{}") - writeFile(t, dir, "helpers/__pycache__/text.cpython-311.pyc", "cache") - // Compiled artifacts are skipped wherever they sit, not just in caches. - writeFile(t, dir, "stale.pyc", "cache") - writeFile(t, dir, "stale.pyo", "cache") - - files, err := WalkCodeFolder(dir) - require.NoError(t, err) - - require.Equal(t, - []string{"README.md", "answer_length.py", "helpers/text.py"}, - relPaths(files), - "only source and data files belong in the package, in sorted order") -} - -// Dot-prefixed files are excluded because an evaluator folder living in a repo -// collects credential files, and publishing the package would copy them into -// blob storage. Excluding the directories alone is not enough: the ones that -// hold secrets sit at the root, next to the source. -func TestWalkCodeFolder_ExcludesDotFiles(t *testing.T) { - dir := t.TempDir() - - writeFile(t, dir, "answer_length.py", evaluatorSource("AnswerLengthEvaluator")) - writeFile(t, dir, ".env", "AZURE_OPENAI_API_KEY=super-secret\n") - writeFile(t, dir, ".netrc", "machine example.com password hunter2\n") - writeFile(t, dir, ".pypirc", "[pypi]\npassword = leaked\n") - writeFile(t, dir, "helpers/.env", "NESTED_SECRET=1\n") - - files, err := WalkCodeFolder(dir) - require.NoError(t, err) - - require.Equal(t, []string{"answer_length.py"}, relPaths(files), - "no dot-prefixed file may reach the upload, at any depth") - - for _, f := range files { - require.NotContains(t, f.RelPath, ".env") - require.NotContains(t, f.RelPath, ".netrc") - require.NotContains(t, f.RelPath, ".pypirc") - } -} - -// Relative paths are the blob names and part of the fingerprint, so they must -// not depend on the host's path separator. -func TestWalkCodeFolder_UsesForwardSlashes(t *testing.T) { - dir := t.TempDir() - writeFile(t, dir, "a/b/c.py", "x = 1\n") - - files, err := WalkCodeFolder(dir) - require.NoError(t, err) - require.Len(t, files, 1) - require.Equal(t, "a/b/c.py", files[0].RelPath) - require.NotContains(t, files[0].RelPath, "\\") -} - -func TestWalkCodeFolder_RejectsAFile(t *testing.T) { - dir := t.TempDir() - path := writeFile(t, dir, "rubric.json", "{}") - - _, err := WalkCodeFolder(path) - require.Error(t, err) - require.Contains(t, err.Error(), "folder") -} - -func TestEvaluatorClassName(t *testing.T) { - cases := map[string]string{ - "answer_length": "AnswerLengthEvaluator", - "answer-length": "AnswerLengthEvaluator", - "answer_length_evaluator": "AnswerLengthEvaluator", - "tone": "ToneEvaluator", - "ToneEvaluator": "ToneEvaluator", - "my.custom_check": "MyCustomCheckEvaluator", - } - for name, want := range cases { - require.Equal(t, want, EvaluatorClassName(name), "for %q", name) - } -} - -func TestLoadCodeEvaluator_AcceptsAConventionalFolder(t *testing.T) { - dir := t.TempDir() - writeFile(t, dir, "answer_length.py", evaluatorSource("AnswerLengthEvaluator")) - writeFile(t, dir, "helpers.py", "def n(s): return len(s)\n") - - pkg, err := LoadCodeEvaluator("answer_length", dir) - require.NoError(t, err) - require.Equal(t, "answer_length.py", pkg.EntryPoint) - require.Equal(t, "AnswerLengthEvaluator", pkg.ClassName) - require.Equal(t, []string{"answer_length.py", "helpers.py"}, relPaths(pkg.Files)) - require.Nil(t, pkg.Metadata) -} - -// The class may inherit, and may be indented inside a conditional. -func TestLoadCodeEvaluator_AcceptsInheritingClass(t *testing.T) { - dir := t.TempDir() - writeFile(t, dir, "tone.py", "import abc\n\nclass ToneEvaluator(abc.ABC):\n pass\n") - - pkg, err := LoadCodeEvaluator("tone", dir) - require.NoError(t, err) - require.Equal(t, "ToneEvaluator", pkg.ClassName) -} - -func TestLoadCodeEvaluator_ReportsMissingEntryPoint(t *testing.T) { - dir := t.TempDir() - writeFile(t, dir, "main.py", evaluatorSource("AnswerLengthEvaluator")) - - _, err := LoadCodeEvaluator("answer_length", dir) - require.Error(t, err) - // The message has to say what is missing, what it must hold, and what was - // actually found — a bare "not found" leaves the author guessing. - require.Contains(t, err.Error(), "answer_length.py") - require.Contains(t, err.Error(), "AnswerLengthEvaluator") - require.Contains(t, err.Error(), "main.py") -} - -func TestLoadCodeEvaluator_ReportsMissingClass(t *testing.T) { - dir := t.TempDir() - writeFile(t, dir, "tone.py", "class SomethingElse:\n pass\n") - - _, err := LoadCodeEvaluator("tone", dir) - require.Error(t, err) - require.Contains(t, err.Error(), "ToneEvaluator") - require.Contains(t, err.Error(), "__call__") -} - -func TestLoadCodeEvaluator_RejectsAnEmptyFolder(t *testing.T) { - dir := t.TempDir() - writeFile(t, dir, "__pycache__/x.pyc", "cache") - - _, err := LoadCodeEvaluator("tone", dir) - require.Error(t, err) - require.Contains(t, err.Error(), "no files") -} - -func TestLoadCodeEvaluator_ReadsFolderMetadata(t *testing.T) { - dir := t.TempDir() - writeFile(t, dir, "tone.py", evaluatorSource("ToneEvaluator")) - writeFile(t, dir, CodeEvaluatorMetadataFile, `{ - "display_name": "Tone", - "description": "Scores tone.", - "metrics": {"result": {"type": "ordinal", "min_value": 1, "max_value": 5}}, - "data_schema": {"type": "object", "properties": {"response": {"type": "string"}}} - }`) - - pkg, err := LoadCodeEvaluator("tone", dir) - require.NoError(t, err) - require.NotNil(t, pkg.Metadata) - require.Equal(t, "Tone", pkg.Metadata.DisplayName) - require.Contains(t, string(pkg.Metadata.Metrics), "ordinal") - require.Contains(t, string(pkg.Metadata.DataSchema), "response") -} - -// A descriptor that cannot be read is a mistake worth reporting: publishing -// without the schemas it declares would register an evaluator the author did -// not describe. -func TestLoadCodeEvaluator_RejectsMalformedMetadata(t *testing.T) { - dir := t.TempDir() - writeFile(t, dir, "tone.py", evaluatorSource("ToneEvaluator")) - writeFile(t, dir, CodeEvaluatorMetadataFile, "{not json") - - _, err := LoadCodeEvaluator("tone", dir) - require.Error(t, err) - require.Contains(t, err.Error(), CodeEvaluatorMetadataFile) -} - -func TestIsCodeEvaluatorSource(t *testing.T) { - dir := t.TempDir() - file := writeFile(t, dir, "rubric.json", "{}") - // A folder can be named like a file, so the decision cannot be made by - // looking at the string. - misleading := filepath.Join(dir, "looks_like.json") - require.NoError(t, os.MkdirAll(misleading, 0o755)) - - require.True(t, IsCodeEvaluatorSource(dir)) - require.True(t, IsCodeEvaluatorSource(misleading)) - require.False(t, IsCodeEvaluatorSource(file)) - require.False(t, IsCodeEvaluatorSource(filepath.Join(dir, "absent"))) - require.False(t, IsCodeEvaluatorSource("")) -} - -func TestFingerprintCodeFolder_IsStableAndSensitive(t *testing.T) { - dir := t.TempDir() - writeFile(t, dir, "answer_length.py", evaluatorSource("AnswerLengthEvaluator")) - writeFile(t, dir, "helpers/text.py", "def clean(s): return s.strip()\n") - - first, err := FingerprintCodeFolder(dir) - require.NoError(t, err) - - again, err := FingerprintCodeFolder(dir) - require.NoError(t, err) - require.Equal(t, first, again, "an unchanged folder must hash the same") - - // One byte. - writeFile(t, dir, "helpers/text.py", "def clean(s): return s.rstrip()\n") - changed, err := FingerprintCodeFolder(dir) - require.NoError(t, err) - require.NotEqual(t, first, changed, "changed content must hash differently") -} - -// Renaming a file changes the package even when every byte is preserved: the -// entry point is resolved by name and imports are written against it. -func TestFingerprintCodeFolder_NoticesARename(t *testing.T) { - dir := t.TempDir() - writeFile(t, dir, "tone.py", evaluatorSource("ToneEvaluator")) - writeFile(t, dir, "helpers.py", "X = 1\n") - - before, err := FingerprintCodeFolder(dir) - require.NoError(t, err) - - require.NoError(t, os.Rename( - filepath.Join(dir, "helpers.py"), filepath.Join(dir, "util.py"))) - - after, err := FingerprintCodeFolder(dir) - require.NoError(t, err) - require.NotEqual(t, before, after) -} - -// The filesystem does not promise a stable iteration order, so the digest must -// describe the package rather than the order it was handed over in. The input -// here is the production walk's own output, permuted — not a hand-built list. -func TestFingerprintCodeFiles_IgnoresInputOrder(t *testing.T) { - dir := t.TempDir() - writeFile(t, dir, "tone.py", evaluatorSource("ToneEvaluator")) - writeFile(t, dir, "helpers/a.py", "A = 1\n") - writeFile(t, dir, "helpers/b.py", "B = 2\n") - writeFile(t, dir, "data/prompts.txt", "hello\n") - - files, err := WalkCodeFolder(dir) - require.NoError(t, err) - require.Len(t, files, 4) - - fromFolder, err := FingerprintCodeFolder(dir) - require.NoError(t, err) - - reversed := make([]CodeFile, 0, len(files)) - for i := len(files) - 1; i >= 0; i-- { - reversed = append(reversed, files[i]) - } - fromReversed, err := FingerprintCodeFiles(reversed) - require.NoError(t, err) - require.Equal(t, fromFolder, fromReversed, "file order must not change the digest") - - rotated := append(append([]CodeFile{}, files[2:]...), files[:2]...) - fromRotated, err := FingerprintCodeFiles(rotated) - require.NoError(t, err) - require.Equal(t, fromFolder, fromRotated) -} - -// Where the folder sits must not affect the digest: two checkouts of the same -// repo, or the same repo on two machines, have to agree or every deploy -// republishes. -func TestFingerprintCodeFolder_IgnoresLocation(t *testing.T) { - build := func(root string) string { - writeFile(t, root, "tone.py", evaluatorSource("ToneEvaluator")) - writeFile(t, root, "helpers/a.py", "A = 1\n") - digest, err := FingerprintCodeFolder(root) - require.NoError(t, err) - return digest - } - - require.Equal(t, build(t.TempDir()), build(t.TempDir())) -} - -// Excluded content must not feed the digest, or a rebuild that only refreshes -// __pycache__ would look like an evaluator change and publish a version. -func TestFingerprintCodeFolder_IgnoresExcludedContent(t *testing.T) { - dir := t.TempDir() - writeFile(t, dir, "tone.py", evaluatorSource("ToneEvaluator")) - - before, err := FingerprintCodeFolder(dir) - require.NoError(t, err) - - writeFile(t, dir, "__pycache__/tone.cpython-311.pyc", strings.Repeat("x", 64)) - writeFile(t, dir, ".venv/lib/site.py", "noise") - - after, err := FingerprintCodeFolder(dir) - require.NoError(t, err) - require.Equal(t, before, after) -} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/codescript.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/codescript.go new file mode 100644 index 00000000000..2c01e791f09 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/codescript.go @@ -0,0 +1,113 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package evalcore + +import ( + "fmt" + "os" + "path/filepath" + "regexp" + "strings" +) + +// pythonExt is the extension a code evaluator's source carries. It is also +// what tells a code evaluator apart from a rubric, which is `.json`. +const pythonExt = ".py" + +// GradeFunctionName is the function the executor calls. +// +// A code evaluator runs as an OpenAI python grader, and the grader contract is +// a single string of source with one entry point: a top-level +// `grade(sample, item)` returning a float. There is no module, package, or +// import path, so nothing else in the file can be reached. +const GradeFunctionName = "grade" + +// CodeEvaluatorScript is a validated evaluator script ready to publish. +type CodeEvaluatorScript struct { + // Name is the evaluator name the script is published under. + Name string + // Path is where the script was read from, kept for error messages. + Path string + // Source is the whole file, which is what the service is sent. The grader + // takes source, not a location: there is nowhere for a second file to go. + Source string +} + +// IsCodeEvaluatorSource reports whether a declared `source:` names a code +// evaluator rather than a rubric. +// +// The decision is made from the extension alone and never touches the +// filesystem, so it answers the same for a path that has not been created yet +// — a config can be validated before the file it names exists. +func IsCodeEvaluatorSource(path string) bool { + return strings.EqualFold(filepath.Ext(path), pythonExt) +} + +// gradeDeclaration matches a top-level `def grade(` — optionally async, and +// anchored at column zero. +// +// Indentation is what makes this specific rather than a substring search: a +// `grade` nested inside a class is a method, and the grader only ever calls a +// module-level function, so an indented match would pass validation here and +// then fail at run time with "top-level grade() function not found". +var gradeDeclaration = regexp.MustCompile( + `(?m)^(?:async[ \t]+)?def[ \t]+` + regexp.QuoteMeta(GradeFunctionName) + `[ \t]*\(`) + +// LoadCodeEvaluator reads an evaluator script and checks it against the grader +// contract. +// +// The check is done here rather than left to the service because the service +// only discovers a missing entry point when a run executes — long after a +// version has been published and an eval bound to it. The failure it reports +// then is "Invalid grader source: top-level grade() function not found in +// source", which names neither the file nor the evaluator. +func LoadCodeEvaluator(name, path string) (*CodeEvaluatorScript, error) { + if name == "" { + return nil, fmt.Errorf("an evaluator name is required to publish %q", path) + } + if path == "" { + return nil, fmt.Errorf("evaluator %q has no source file to publish", name) + } + + info, err := os.Stat(path) + if err != nil { + return nil, fmt.Errorf("reading evaluator source %q: %w", path, err) + } + if info.IsDir() { + return nil, fmt.Errorf( + "evaluator source %q is a directory. A code evaluator is a single %s file: "+ + "it is published as the source of a python grader, which takes one script "+ + "and cannot import a helper module beside it", path, pythonExt) + } + if !IsCodeEvaluatorSource(path) { + return nil, fmt.Errorf( + "evaluator source %q must be a %s file", path, pythonExt) + } + + source, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("reading evaluator source %q: %w", path, err) + } + if len(strings.TrimSpace(string(source))) == 0 { + return nil, fmt.Errorf("evaluator source %q is empty", path) + } + if !gradeDeclaration.Match(source) { + return nil, fmt.Errorf( + "%s does not declare a top-level %s(sample, item) function. A code evaluator "+ + "runs as a python grader, which calls exactly that and nothing else — a "+ + "class, a differently named function, or one nested inside another will "+ + "not be found. For example:\n\ndef %s(sample, item) -> float:\n"+ + " return float(len(item.get(\"response\", \"\")))\n\n"+ + "The script must also be self-contained: only the standard library and "+ + "whatever the image named by --image-tag provides are importable, so a "+ + "helper file next to it cannot be imported", + path, GradeFunctionName, GradeFunctionName) + } + + return &CodeEvaluatorScript{ + Name: name, + Path: path, + Source: string(source), + }, nil +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/codescript_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/codescript_test.go new file mode 100644 index 00000000000..7cb75014828 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/codescript_test.go @@ -0,0 +1,128 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package evalcore + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" +) + +// gradeSource is the minimal script that satisfies the grader contract. +const gradeSource = `def grade(sample, item) -> float: + return float(len((item or {}).get("response", ""))) +` + +func writeFile(t *testing.T, root, rel, content string) string { + t.Helper() + path := filepath.Join(root, filepath.FromSlash(rel)) + require.NoError(t, os.MkdirAll(filepath.Dir(path), 0o755)) + require.NoError(t, os.WriteFile(path, []byte(content), 0o600)) + return path +} + +// The extension decides which definition type to publish from the source's +// extension alone, and it has to answer for a path that does not exist yet so +// a config can be validated before its files are written. +func TestIsCodeEvaluatorSource(t *testing.T) { + require.True(t, IsCodeEvaluatorSource("tone.py")) + require.True(t, IsCodeEvaluatorSource("evaluators/tone.py")) + require.True(t, IsCodeEvaluatorSource(`evaluators\tone.PY`), + "the extension is matched case-insensitively") + require.True(t, IsCodeEvaluatorSource("/absent/never/created.py"), + "classification must not touch the filesystem") + + require.False(t, IsCodeEvaluatorSource("rubric.json")) + require.False(t, IsCodeEvaluatorSource("evaluators/rubric.json")) + require.False(t, IsCodeEvaluatorSource("evaluator"), + "a folder is no longer a code evaluator; the grader takes one script") + require.False(t, IsCodeEvaluatorSource("tone.python")) + require.False(t, IsCodeEvaluatorSource("")) +} + +func TestLoadCodeEvaluator_AcceptsATopLevelGrade(t *testing.T) { + dir := t.TempDir() + path := writeFile(t, dir, "tone.py", gradeSource) + + script, err := LoadCodeEvaluator("tone", path) + require.NoError(t, err) + require.Equal(t, "tone", script.Name) + require.Equal(t, path, script.Path) + require.Equal(t, gradeSource, script.Source, + "the whole file is what the grader is sent") +} + +// The grader is handed source and calls grade(); an async definition is still +// a top-level grade(). +func TestLoadCodeEvaluator_AcceptsAsyncAndAnnotatedForms(t *testing.T) { + for label, source := range map[string]string{ + "async": "async def grade(sample, item) -> float:\n return 1.0\n", + "spaced": "def grade (sample, item):\n return 1.0\n", + "no-annot": "def grade(sample, item):\n return 1.0\n", + "after-code": "import json\n\n\ndef grade(sample, item):\n return 1.0\n", + } { + dir := t.TempDir() + path := writeFile(t, dir, "tone.py", source) + _, err := LoadCodeEvaluator("tone", path) + require.NoError(t, err, "for %s", label) + } +} + +// Without this the failure surfaces only when a run executes, as "Invalid +// grader source: top-level grade() function not found in source" — long after +// a version has been published and an eval bound to it. +func TestLoadCodeEvaluator_ReportsAMissingGrade(t *testing.T) { + dir := t.TempDir() + path := writeFile(t, dir, "tone.py", + "class ToneEvaluator:\n def __call__(self, **kwargs):\n return {\"result\": 1}\n") + + _, err := LoadCodeEvaluator("tone", path) + require.Error(t, err) + require.Contains(t, err.Error(), "grade(sample, item)") + require.Contains(t, err.Error(), path) +} + +// A grade() nested inside a class is a method. The grader only ever calls a +// module-level function, so an indented match must not pass validation. +func TestLoadCodeEvaluator_RejectsANestedGrade(t *testing.T) { + dir := t.TempDir() + path := writeFile(t, dir, "tone.py", + "class ToneEvaluator:\n def grade(self, sample, item):\n return 1.0\n") + + _, err := LoadCodeEvaluator("tone", path) + require.Error(t, err) + require.Contains(t, err.Error(), "top-level") +} + +func TestLoadCodeEvaluator_RejectsAFolder(t *testing.T) { + dir := t.TempDir() + writeFile(t, dir, "tone.py", gradeSource) + + _, err := LoadCodeEvaluator("tone", dir) + require.Error(t, err) + require.Contains(t, err.Error(), "single") +} + +func TestLoadCodeEvaluator_RejectsBadInput(t *testing.T) { + dir := t.TempDir() + + _, err := LoadCodeEvaluator("", writeFile(t, dir, "tone.py", gradeSource)) + require.Error(t, err, "a name is required to publish under") + + _, err = LoadCodeEvaluator("tone", "") + require.Error(t, err) + + _, err = LoadCodeEvaluator("tone", filepath.Join(dir, "absent.py")) + require.Error(t, err) + + _, err = LoadCodeEvaluator("tone", writeFile(t, dir, "rubric.json", "{}")) + require.Error(t, err, "a rubric is not a code evaluator") + require.Contains(t, err.Error(), ".py") + + _, err = LoadCodeEvaluator("tone", writeFile(t, dir, "empty.py", " \n\n")) + require.Error(t, err) + require.Contains(t, err.Error(), "empty") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go index 1748b98c0b4..336c9b17bdf 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go @@ -34,10 +34,11 @@ type DatasetDecl struct { // EvaluatorDecl declares a custom evaluator. Built-ins are referenced directly // from an eval and never declared here. // -// Source decides which kind of evaluator this is: a folder holds Python and -// publishes a code evaluator, a file holds a rubric. The distinction is made -// by stat-ing the path on deploy, not by its spelling, so a folder does not -// need a trailing separator to be recognised. +// Source decides which kind of evaluator this is, by extension: a `.py` file is +// a single self-contained Python script and publishes a code evaluator, a +// `.json` file holds a rubric. A code evaluator cannot name a folder — it runs +// as a python grader, which is handed one script's source and cannot import a +// helper module beside it. type EvaluatorDecl struct { Name string `yaml:"name" json:"name"` Source string `yaml:"source,omitempty" json:"source,omitempty"` diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go index 3e68eccd52c..f4902f236a3 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go @@ -13,8 +13,6 @@ import ( "path/filepath" "strings" - "azureaieval/internal/pkg/evalcore" - "github.com/azure/azure-dev/cli/azd/pkg/azdext" "github.com/azure/azure-dev/cli/azd/pkg/foundry" "google.golang.org/protobuf/types/known/structpb" @@ -305,7 +303,8 @@ func resolveSource(baseDir, source string) string { // content changed without downloading anything from the service. // // The dataset API returns no content hash or etag, so comparing against the -// service would mean downloading the blob on every deploy. +// service would mean downloading the blob on every deploy. Every artifact this +// applies to — a dataset, a rubric, an evaluator script — is a single file. func Fingerprint(path string) (string, error) { data, err := os.ReadFile(path) if err != nil { @@ -315,22 +314,6 @@ func Fingerprint(path string) (string, error) { return hex.EncodeToString(sum[:]), nil } -// FingerprintPath hashes whatever a declared source names. -// -// A rubric evaluator is one JSON file; a code evaluator is a folder of Python. -// Both need change detection with the same meaning, so the artifact's shape is -// resolved by stat-ing it rather than by asking the caller to know. -func FingerprintPath(path string) (string, error) { - info, err := os.Stat(path) - if err != nil { - return "", fmt.Errorf("hashing %q: %w", path, err) - } - if info.IsDir() { - return evalcore.FingerprintCodeFolder(path) - } - return Fingerprint(path) -} - // FingerprintGroup hashes an eval's own declaration. // // Change detection on upstream artifacts is not sufficient: editing a group's diff --git a/cli/azd/extensions/azure.ai.evaluations/tests/live/code_evaluator_run_test.go b/cli/azd/extensions/azure.ai.evaluations/tests/live/code_evaluator_run_test.go new file mode 100644 index 00000000000..fbdfe1b2b87 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/tests/live/code_evaluator_run_test.go @@ -0,0 +1,347 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +//go:build live + +package live + +import ( + "context" + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "azureaieval/internal/pkg/eval_api" + "azureaieval/internal/pkg/evalcore" + + "github.com/stretchr/testify/require" +) + +// These tests close the gap between publishing a code evaluator and using one. +// +// The publish path was verified on its own first, and passing that proves less +// than it appears to: a script publishes cleanly while carrying no +// data_schema, because only metrics are defaulted. Nothing then tells the +// caller that the evaluator cannot be wired into an eval. The criteria builder +// derives data_mapping from the schema the evaluator publishes, so no schema +// means no mapping, and the service refuses a criterion with none. That +// failure would surface at run time, long after the publish that caused it. +// +// So there are two tests. The first runs a code evaluator end to end and +// requires it to score a sample. The second publishes without a schema and +// records what the service actually does about it, rather than leaving the +// consequence to inference. + +// writeCodeEvaluator writes an evaluator that scores the length of a response. +// +// It is one self-contained script. A code evaluator runs as an OpenAI python +// grader, whose contract is a single Source string with one entry point: a +// top-level grade(sample, item). There is no package and no import path, so a +// helper module beside it could not be reached even if it were published. +func writeCodeEvaluator(t *testing.T, name string) string { + t.Helper() + dir := t.TempDir() + path := filepath.Join(dir, name+".py") + + source := `def grade(sample, item) -> float: + response = (item or {}).get("response", "") + return float(len(response)) +` + require.NoError(t, os.WriteFile(path, []byte(source), 0o600)) + return path +} + +// dataSchemaForResponse is the schema the criteria builder needs to derive a +// data_mapping. Only the caller can supply it: nothing about it is inferable +// from Python source. +func dataSchemaForResponse(t *testing.T) json.RawMessage { + t.Helper() + raw, err := json.Marshal(map[string]any{ + "type": "object", + "properties": map[string]any{"response": map[string]any{"type": "string"}}, + "required": []string{"response"}, + }) + require.NoError(t, err) + return raw +} + +// publishCodeEvaluator registers the script and returns the version, after +// confirming it is readable. +func publishCodeEvaluator( + t *testing.T, + env *liveEnv, + name string, + withSchema bool, +) *eval_api.EvaluatorVersion { + t.Helper() + ctx := context.Background() + + path := writeCodeEvaluator(t, name) + script, err := evalcore.LoadCodeEvaluator(name, path) + require.NoError(t, err, "loading the evaluator script") + + var opts eval_api.CodeEvaluatorOptions + if withSchema { + opts.DataSchema = dataSchemaForResponse(t) + opts.Metrics = json.RawMessage( + `{"result":{"type":"continuous","desirable_direction":"increase","is_primary":true}}`) + } + + version, err := env.evalClient.CreateCodeEvaluatorVersion(ctx, script, opts, projectAPIVersion) + require.NoError(t, err, "publishing the code evaluator") + t.Cleanup(func() { + _ = env.evalClient.DeleteEvaluatorVersion( + context.Background(), name, version.Version, projectAPIVersion) + }) + t.Logf("published code evaluator %s version %s", name, version.Version) + + awaitEvaluatorResolvable(t, env, name, version.Version) + return version +} + +// awaitEvaluatorResolvable waits for a published version the way the +// reconciler does, and reports how long each of the two views took. +// +// The numbers are the point. The direct read goes consistent almost at once +// while the listing lags it, and the eval-create resolver follows the slower +// one: a publish was observed reading back at 03:06:58 and still failing eval +// creation at 03:06:59. Logging both is what keeps the reconciler's tolerance +// honest instead of guessed, and asserting on the listing here is what proves +// the gate it waits on is the right one. +func awaitEvaluatorResolvable(t *testing.T, env *liveEnv, name, version string) { + t.Helper() + ctx := context.Background() + + start := time.Now() + var readable time.Duration + + for { + if readable == 0 { + if _, err := env.evalClient.GetEvaluatorRaw( + ctx, name, version, projectAPIVersion, + ); err == nil { + readable = time.Since(start) + t.Logf("evaluator %s readable after %s", + name, readable.Round(time.Millisecond)) + } + } + if readable != 0 && evaluatorVersionListed(ctx, env, name, version) { + t.Logf("evaluator %s listed after %s", + name, time.Since(start).Round(time.Millisecond)) + return + } + if time.Since(start) > 2*time.Minute { + t.Fatalf("evaluator %s never became resolvable", name) + } + time.Sleep(250 * time.Millisecond) + } +} + +func evaluatorVersionListed( + ctx context.Context, + env *liveEnv, + name, version string, +) bool { + list, err := env.evalClient.ListEvaluatorVersions(ctx, name, projectAPIVersion) + if err != nil || list == nil { + return false + } + for _, entry := range list.Value { + if entry.Version == version { + return true + } + } + return false +} + +// createEvalReferencing creates an eval naming a custom evaluator, tolerating +// the window in which the evaluator is published but not yet resolvable. +// +// The delay is reported so a run that hits it leaves evidence of how long it +// took, which is the number the reconciler's own tolerance has to be built on. +func createEvalReferencing( + t *testing.T, + env *liveEnv, + req *eval_api.CreateOpenAIEvalRequest, + within time.Duration, +) (*eval_api.OpenAIEval, error) { + t.Helper() + ctx := context.Background() + + start := time.Now() + for { + group, err := env.evalClient.CreateOpenAIEval(ctx, req) + if err == nil { + t.Logf("eval accepted the evaluator after %s", time.Since(start).Round(time.Millisecond)) + return group, nil + } + if !strings.Contains(strings.ToLower(err.Error()), "was not found") { + return nil, err + } + if time.Since(start) > within { + t.Logf("the evaluator was still unresolvable after %s", within) + return nil, err + } + time.Sleep(5 * time.Second) + } +} + +// TestLiveCodeEvaluatorScoresARun is the test the publish tests could not be: +// it requires the evaluator to actually run and return a score. +// +// No agent is involved. A code evaluator reads item fields, so the run uses a +// dataset-only source and the rows are supplied inline. That keeps the test +// about the evaluator rather than about a target being reachable. +func TestLiveCodeEvaluatorScoresARun(t *testing.T) { + env := setup(t) + ctx := context.Background() + + name := strings.ReplaceAll(uniqueName("azdcoderun"), "-", "_") + publishCodeEvaluator(t, env, name, true) + + group, err := createEvalReferencing(t, env, &eval_api.CreateOpenAIEvalRequest{ + Name: uniqueName("azd-code-eval"), + DataSourceConfig: &eval_api.DataSourceConfig{ + Type: "custom", + ItemSchema: map[string]any{ + "type": "object", + "properties": map[string]any{"response": map[string]any{"type": "string"}}, + }, + }, + TestingCriteria: []eval_api.TestingCriterion{{ + Type: "azure_ai_evaluator", + Name: name, + EvaluatorName: name, + DataMapping: map[string]string{"response": "{{item.response}}"}, + }}, + }, 3*time.Minute) + require.NoError(t, err, "creating an eval that references the code evaluator") + t.Logf("created eval %s", group.ID) + + ds := eval_api.NewDatasetOnlyDataSource() + ds.SetFileContent([]map[string]any{ + {"response": "a short answer"}, + {"response": "a considerably longer answer than the first one"}, + }) + + run, err := env.evalClient.CreateOpenAIEvalRun(ctx, group.ID, &eval_api.CreateOpenAIEvalRunRequest{ + Name: uniqueName("code-run"), + DataSource: ds, + }) + require.NoError(t, err, "starting the run") + t.Cleanup(func() { + _, _ = env.evalClient.CancelOpenAIEvalRun(context.Background(), group.ID, run.ID) + }) + t.Logf("started run %s", run.ID) + + final := awaitRun(t, env, group.ID, run.ID, 10*time.Minute) + + // Reaching a terminal state is not the same as having evaluated anything: + // a run whose every sample errors still reports completed. + require.Equal(t, "completed", strings.ToLower(final.Status), + "the run must complete rather than fail or cancel") + require.NotNil(t, final.ResultCounts, "a completed run must report counts") + require.Zero(t, final.ResultCounts.Errored, + "an errored sample means the code evaluator did not run") + require.Positive(t, final.ResultCounts.Passed+final.ResultCounts.Failed, + "the run must score at least one sample; scoring nothing means the rows "+ + "never reached the evaluator") +} + +// TestLiveCodeEvaluatorWithoutSchemaIsAccepted pins down what happens to a +// script published with no data_schema, which is the shape most people's +// their first evaluator will produce. +// +// It was expected to be refused. The reasoning was that the criteria builder +// derives data_mapping from the evaluator's schema, so no schema means no +// mapping, and the service rejects a criterion with none. An earlier run +// appeared to confirm it. That was wrong: the refusal was the propagation 404 +// in disguise, read as a mapping error because it arrived at the same call. +// With the publish properly gated, a schema-less evaluator is accepted and an +// empty data_mapping is allowed, so this is not the trap it looked like. +// +// The create is deliberately not retried. publishCodeEvaluator has already +// waited on the same condition the reconciler waits on, so a "was not found" +// here would mean that gate is the wrong one — which is worth failing on, +// because a retry would hide it. +func TestLiveCodeEvaluatorWithoutSchemaIsAccepted(t *testing.T) { + env := setup(t) + ctx := context.Background() + + name := strings.ReplaceAll(uniqueName("azdcodenoschema"), "-", "_") + published := publishCodeEvaluator(t, env, name, false) + require.NotEmpty(t, published.Version, + "a script with no schema still publishes; the schema is not required to register") + + _, err := env.evalClient.CreateOpenAIEval(ctx, &eval_api.CreateOpenAIEvalRequest{ + Name: uniqueName("azd-code-eval-noschema"), + DataSourceConfig: &eval_api.DataSourceConfig{ + Type: "custom", + ItemSchema: map[string]any{ + "type": "object", + "properties": map[string]any{"response": map[string]any{"type": "string"}}, + }, + }, + TestingCriteria: []eval_api.TestingCriterion{{ + Type: "azure_ai_evaluator", + Name: name, + EvaluatorName: name, + // Deliberately empty: this is what the criteria builder produces + // for an evaluator that publishes no data_schema. + DataMapping: map[string]string{}, + }}, + }) + + if err == nil { + t.Log("an evaluator with no data_schema was accepted with an empty data_mapping; " + + "publishing without a schema is not by itself a blocker") + return + } + require.NotContains(t, strings.ToLower(err.Error()), "was not found", + "the evaluator was published and waited for, so a not-found here means the "+ + "propagation gate the reconciler uses does not cover eval creation") + t.Logf("an evaluator with no data_schema was refused: %v", err) + require.Contains(t, strings.ToLower(err.Error()), "mapping", + "the refusal should name the mapping, so the CLI can explain it at publish time") +} + +// awaitRun polls until the run reaches a terminal state or the deadline passes. +func awaitRun( + t *testing.T, + env *liveEnv, + evalID string, + runID string, + within time.Duration, +) *eval_api.OpenAIEvalRun { + t.Helper() + ctx := context.Background() + + terminal := map[string]bool{ + "completed": true, "failed": true, "canceled": true, "cancelled": true, "error": true, + } + deadline := time.Now().Add(within) + for { + current, err := env.evalClient.GetOpenAIEvalRun(ctx, evalID, runID) + require.NoError(t, err, "polling the run") + if terminal[strings.ToLower(current.Status)] { + if current.ResultCounts != nil { + t.Logf("run %s reached %s: passed=%d failed=%d errored=%d", + runID, current.Status, + current.ResultCounts.Passed, + current.ResultCounts.Failed, + current.ResultCounts.Errored) + } + body, _ := json.MarshalIndent(current.PerTestingCriteria, "", " ") + t.Logf("per-criteria results: %s", string(body)) + return current + } + if time.Now().After(deadline) { + t.Fatalf("run %s did not finish within %s (last status %q)", + runID, within, current.Status) + } + time.Sleep(10 * time.Second) + } +} From 53c08cb83a0678d5e459a49b60cca034e5c1d08d Mon Sep 17 00:00:00 2001 From: mohessie Date: Sun, 2 Aug 2026 20:54:23 +0300 Subject: [PATCH 064/320] Cover schedules end to end, and explain the role they need Schedules had eight unit tests, all of them about buildTrigger's output. Whether the service accepts that output was untested, so every trigger shape the CLI can emit is now sent to a real project and required to survive a round trip, alongside create/read/list/delete, --disabled, and the edit the CLI refuses to make. Running them found a prerequisite nothing else in the extension needs. A schedule fires later and runs as the project, not as the caller, so the project's managed identity must hold the Foundry User role on the project. Without it every schedule creation is refused with PermissionDenied, and the raw message does not say why creating a schedule needs a permission that running an eval does not. explainScheduleFailure now says so. The tests skip rather than fail when the role is absent, since that is an environment fact rather than a regression, and the skip names the missing role. The trigger-shape test probes once before its subtests: a parent reporting PASS while every subtest skipped is worse than a failure, because it looks like coverage. Two things the tests had to get right to be worth anything. The eval they schedule is built with the shipping builder, not a hand-written criterion - built-ins do not share an input contract, and the one the listing returns first, builtin.ifeval, needs an instruction_id_list, so a hand-rolled mapping is rejected. And no evaluator is named: which built-ins a project exposes varies, so the first one the builder can satisfy is used. Cleanup waits for a schedule to settle before deleting it. One still being created refuses the delete as a 409, or as a 404 because the trigger behind it does not exist yet, and a project holds very few, so a leaked schedule would break every later run. --- .../internal/cmd/schedule.go | 16 + .../internal/cmd/schedule_live_test.go | 397 ++++++++++++++++++ 2 files changed, 413 insertions(+) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/schedule_live_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/schedule.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/schedule.go index 5a8affe44bd..e3ce51edee8 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/schedule.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/schedule.go @@ -605,6 +605,15 @@ func explainScheduleFailure( "once against its dataset first so the schedule repeats that instead", name) } + if isScheduleRoleMissing(cause) { + return fmt.Errorf( + "saving schedule %q: the project's managed identity needs the Foundry User "+ + "role on the project before it can run scheduled evaluations. A schedule "+ + "runs later, as the project rather than as you, which is why creating one "+ + "needs a role that running an eval yourself does not. Grant it on the "+ + "project and retry", name) + } + list, listErr := ec.evalClient.ListSchedules(ctx, ProjectEndpointAPIVersion) if listErr != nil || list == nil { return fmt.Errorf("saving schedule %q: %w", name, cause) @@ -627,3 +636,10 @@ func isTracesHourlyOnly(err error) bool { return err != nil && strings.Contains(err.Error(), "trace evaluations only support hourly") } + +// isScheduleRoleMissing matches the refusal when the project identity cannot +// run the evaluation the schedule would trigger. +func isScheduleRoleMissing(err error) bool { + return err != nil && + strings.Contains(err.Error(), "lacks Foundry User role") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/schedule_live_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/schedule_live_test.go new file mode 100644 index 00000000000..ebd8da98570 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/schedule_live_test.go @@ -0,0 +1,397 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +//go:build live + +package cmd + +import ( + "context" + "fmt" + "sort" + "testing" + "time" + + "azureaieval/internal/pkg/eval_api" + "azureaieval/internal/pkg/evalcore" + "azureaieval/internal/project" + + "github.com/stretchr/testify/require" +) + +// Schedules are project-scoped and the service tolerates very few of them, so +// everything here runs serially and deletes what it creates. Nothing calls +// t.Parallel(). +// +// These tests exist because the unit tests only prove that buildTrigger +// produces a well-formed struct. Whether the service accepts that struct is a +// different question, and it is the one that matters: every trigger shape the +// CLI can emit is sent here and required to survive a round trip. + +// liveScheduleEval creates a throwaway eval for a schedule to point at. +// +// The criterion comes from the shipping builder rather than a hand-written +// one. Built-ins do not share an input contract — builtin.ifeval, which is +// what the listing happens to return first, needs an instruction_id_list — so +// a hand-rolled mapping is rejected with MissingRequiredDataMapping. Letting +// production shape the request also means this helper cannot drift from it. +// +// No evaluator is named. Which built-ins a project exposes varies, so the +// first one the builder can satisfy is used, and the dataset is given every +// column that evaluator declares. +func liveScheduleEval(t *testing.T, client *eval_api.EvalClient, judge string) string { + t.Helper() + ctx := context.Background() + + ec := &evalContext{evalClient: client} + schemas := ec.evaluatorSchemas(ctx) + require.NotEmpty(t, schemas, "need the published evaluator contracts to build an eval") + + names := make([]string, 0, len(schemas)) + for name := range schemas { + names = append(names, name) + } + sort.Strings(names) + + for _, name := range names { + summary := schemas[name] + columns := map[string]bool{"query": true} + if ds := summary.DataSchema(); ds != nil { + for _, col := range ds.PropertyNames() { + columns[col] = true + } + } + + level := "" + if len(summary.SupportedEvaluationLevels) > 0 { + level = summary.SupportedEvaluationLevels[0] + } + + req, err := buildEvalRequest(&project.Eval{ + Name: fmt.Sprintf("azd-sched-%d", time.Now().UTC().UnixNano()), + Dataset: "inline", + Target: &project.Target{Type: "agent", Name: "probe-agent"}, + Evaluators: []evalcore.EvaluatorRef{{Name: summary.Name}}, + Options: &project.Options{EvalModel: judge, EvaluationLevel: level}, + }, schemas, columns) + if err != nil { + continue + } + + created, err := client.CreateOpenAIEval(ctx, req) + if err != nil { + t.Logf("built-in %s could not back a schedule: %v", name, err) + continue + } + t.Cleanup(func() { + _ = client.DeleteOpenAIEval(context.Background(), created.ID) + }) + t.Logf("scheduling an eval built on %s (%s)", name, created.ID) + return created.ID + } + + t.Fatalf("no built-in produced an eval a schedule could run; tried %d", len(names)) + return "" +} + +// putLiveSchedule creates a schedule and registers its removal. +func putLiveSchedule( + t *testing.T, + client *eval_api.EvalClient, + name string, + evalID string, + trigger *eval_api.ScheduleTrigger, +) (*eval_api.Schedule, error) { + t.Helper() + + saved, err := client.PutSchedule(context.Background(), name, &eval_api.Schedule{ + DisplayName: name, + Enabled: true, + Trigger: trigger, + Task: &eval_api.ScheduleTask{ + Type: eval_api.ScheduleTaskEvaluation, + EvalID: evalID, + EvalRun: &eval_api.CreateOpenAIEvalRunRequest{ + Name: name, + DataSource: datasetOnlyRows([]map[string]any{{"query": "how do I reset my password?"}}), + }, + }, + }, ProjectEndpointAPIVersion) + + if err == nil { + t.Cleanup(func() { removeLiveSchedule(t, client, name) }) + } + return saved, err +} + +// skipIfScheduleRoleMissing stops the test when the project cannot host a +// schedule at all. +// +// A schedule fires later and runs as the project, so creating one requires the +// project's managed identity to hold the Foundry User role on the project — +// a permission no other command in this extension needs. Without it every +// schedule test fails identically and for a reason that has nothing to do with +// the code, so they skip loudly instead of reporting a false regression. +func skipIfScheduleRoleMissing(t *testing.T, err error) { + t.Helper() + if err == nil || !isScheduleRoleMissing(err) { + return + } + t.Skipf("this project cannot host schedules: its managed identity lacks the "+ + "Foundry User role on the project. Grant it and re-run to exercise "+ + "schedules for real. Underlying error: %v", err) +} + +// datasetOnlyRows is the inline data source a scheduled run repeats. +func datasetOnlyRows(rows []map[string]any) *eval_api.EvalRunDataSource { + ds := eval_api.NewDatasetOnlyDataSource() + ds.SetFileContent(rows) + return ds +} + +// removeLiveSchedule deletes a schedule once it is no longer provisioning. +// +// A schedule still being created refuses the delete, as a 409 while it is busy +// or a 404 because the trigger behind it does not exist yet. Waiting for it to +// settle is what makes cleanup reliable, and leaving one behind would break +// every later test in this file, because the project holds very few. +func removeLiveSchedule(t *testing.T, client *eval_api.EvalClient, name string) { + t.Helper() + ctx := context.Background() + + deadline := time.Now().Add(2 * time.Minute) + for { + current, err := client.GetSchedule(ctx, name, ProjectEndpointAPIVersion) + if err != nil { + return // already gone + } + if current.Settled() { + break + } + if time.Now().After(deadline) { + t.Logf("schedule %q never settled (status %q); leaving it", name, current.ProvisioningStatus) + return + } + time.Sleep(3 * time.Second) + } + + if err := client.DeleteSchedule(ctx, name, ProjectEndpointAPIVersion); err != nil { + t.Logf("could not delete schedule %q: %v", name, err) + } +} + +// awaitScheduleSettled blocks until the schedule finishes provisioning. +func awaitScheduleSettled( + t *testing.T, + client *eval_api.EvalClient, + name string, +) *eval_api.Schedule { + t.Helper() + ctx := context.Background() + + deadline := time.Now().Add(2 * time.Minute) + for { + current, err := client.GetSchedule(ctx, name, ProjectEndpointAPIVersion) + require.NoError(t, err, "reading schedule %q back", name) + if current.Settled() { + return current + } + if time.Now().After(deadline) { + t.Fatalf("schedule %q stuck in %q", name, current.ProvisioningStatus) + } + time.Sleep(3 * time.Second) + } +} + +func liveScheduleName(t *testing.T, suffix string) string { + t.Helper() + return fmt.Sprintf("azdsched-%s-%d", suffix, time.Now().UnixNano()) +} + +// TestLiveScheduleLifecycle walks create, read, list and delete. +func TestLiveScheduleLifecycle(t *testing.T) { + client, judge := liveEvalClient(t) + ctx := context.Background() + + evalID := liveScheduleEval(t, client, judge) + name := liveScheduleName(t, "life") + + trigger, err := buildTrigger(triggerFlags{every: "daily", atHours: []int{9}}) + require.NoError(t, err) + + saved, err := putLiveSchedule(t, client, name, evalID, trigger) + skipIfScheduleRoleMissing(t, err) + require.NoError(t, err, "the service rejected a trigger the CLI can produce") + require.NotEmpty(t, saved.ID) + + settled := awaitScheduleSettled(t, client, name) + require.True(t, settled.Enabled, "a schedule created without --disabled must be enabled") + require.NotNil(t, settled.Task, "the schedule must carry the task it runs") + require.Equal(t, evalID, settled.Task.EvalID, + "the schedule must point at the eval it was given") + require.NotNil(t, settled.Trigger) + require.Equal(t, eval_api.TriggerRecurrence, settled.Trigger.Type) + + list, err := client.ListSchedules(ctx, ProjectEndpointAPIVersion) + require.NoError(t, err) + found := false + for i := range list.Value { + if list.Value[i].ID == saved.ID || list.Value[i].DisplayName == name { + found = true + break + } + } + require.True(t, found, "a created schedule must appear in the listing") + + removeLiveSchedule(t, client, name) + + _, err = client.GetSchedule(ctx, name, ProjectEndpointAPIVersion) + require.Error(t, err, "a deleted schedule must not read back") + require.True(t, eval_api.IsNotFound(err), + "deleting should leave a not-found, got %v", err) +} + +// TestLiveScheduleAcceptsEveryTriggerShape sends one schedule per trigger the +// CLI can build and requires the service to accept each. +// +// The unit tests assert the shape of what buildTrigger returns. They cannot +// say whether the service agrees, and a trigger the service rejects is a +// trigger the CLI should never have offered. +func TestLiveScheduleAcceptsEveryTriggerShape(t *testing.T) { + client, judge := liveEvalClient(t) + evalID := liveScheduleEval(t, client, judge) + + // Probe once up front. Skipping inside the subtests instead would leave the + // parent reporting PASS with nothing proven, which is worse than a failure + // because it looks like coverage. + probe := liveScheduleName(t, "probe") + daily, err := buildTrigger(triggerFlags{every: "daily", atHours: []int{4}}) + require.NoError(t, err) + if _, probeErr := putLiveSchedule(t, client, probe, evalID, daily); probeErr != nil { + skipIfScheduleRoleMissing(t, probeErr) + require.NoError(t, probeErr, "could not create the probe schedule") + } + removeLiveSchedule(t, client, probe) + + cases := []struct { + label string + flags triggerFlags + want string + }{ + {"cron", triggerFlags{cron: "0 9 * * *"}, eval_api.TriggerCron}, + {"hourly", triggerFlags{every: "hourly"}, eval_api.TriggerRecurrence}, + {"daily", triggerFlags{every: "daily", atHours: []int{9, 17}}, eval_api.TriggerRecurrence}, + {"weekly", triggerFlags{every: "weekly", onDays: []string{"Monday"}}, eval_api.TriggerRecurrence}, + {"monthly", triggerFlags{every: "monthly", onDaysOfMon: []int{1}}, eval_api.TriggerRecurrence}, + {"interval", triggerFlags{every: "daily", interval: 3}, eval_api.TriggerRecurrence}, + { + "onetime", + triggerFlags{atTime: time.Now().UTC().Add(24 * time.Hour).Format(time.RFC3339)}, + eval_api.TriggerOneTime, + }, + } + + for _, tc := range cases { + t.Run(tc.label, func(t *testing.T) { + trigger, err := buildTrigger(tc.flags) + require.NoError(t, err, "the CLI could not build a %s trigger", tc.label) + require.Equal(t, tc.want, trigger.Type) + + name := liveScheduleName(t, tc.label) + saved, err := putLiveSchedule(t, client, name, evalID, trigger) + skipIfScheduleRoleMissing(t, err) + require.NoError(t, err, "the service rejected the %s trigger", tc.label) + require.NotEmpty(t, saved.ID) + + settled := awaitScheduleSettled(t, client, name) + require.Equal(t, tc.want, settled.Trigger.Type, + "the trigger type must survive the round trip") + removeLiveSchedule(t, client, name) + }) + } +} + +// TestLiveScheduleDisabledStaysDisabled covers --disabled, which is the one +// flag whose whole purpose is a field the service could quietly ignore. +func TestLiveScheduleDisabledStaysDisabled(t *testing.T) { + client, judge := liveEvalClient(t) + evalID := liveScheduleEval(t, client, judge) + name := liveScheduleName(t, "disabled") + + trigger, err := buildTrigger(triggerFlags{every: "daily", atHours: []int{3}}) + require.NoError(t, err) + + saved, err := client.PutSchedule(context.Background(), name, &eval_api.Schedule{ + DisplayName: name, + Enabled: false, + Trigger: trigger, + Task: &eval_api.ScheduleTask{ + Type: eval_api.ScheduleTaskEvaluation, + EvalID: evalID, + EvalRun: &eval_api.CreateOpenAIEvalRunRequest{ + Name: name, + DataSource: datasetOnlyRows([]map[string]any{{"query": "hello"}}), + }, + }, + }, ProjectEndpointAPIVersion) + skipIfScheduleRoleMissing(t, err) + require.NoError(t, err) + t.Cleanup(func() { removeLiveSchedule(t, client, name) }) + require.NotEmpty(t, saved.ID) + + settled := awaitScheduleSettled(t, client, name) + require.False(t, settled.Enabled, + "a schedule created disabled must not come back enabled") +} + +// TestLiveScheduleEditIsRefusedByTheCLI pins the reason `schedule set` refuses +// to reuse a name. +// +// The service takes a PUT on an existing schedule and does not apply it, and a +// replacement can stick in Creating where it can no longer be deleted. The CLI +// therefore refuses before sending. This test records the service behaviour +// the guard exists for, so a change in the service is visible here rather than +// as a stuck schedule in someone's project. +func TestLiveScheduleEditIsRefusedByTheCLI(t *testing.T) { + client, judge := liveEvalClient(t) + evalID := liveScheduleEval(t, client, judge) + name := liveScheduleName(t, "edit") + + daily, err := buildTrigger(triggerFlags{every: "daily", atHours: []int{9}}) + require.NoError(t, err) + _, err = putLiveSchedule(t, client, name, evalID, daily) + skipIfScheduleRoleMissing(t, err) + require.NoError(t, err) + awaitScheduleSettled(t, client, name) + + // The guard in `schedule set` is a GetSchedule that must find this. + existing, err := client.GetSchedule(context.Background(), name, ProjectEndpointAPIVersion) + require.NoError(t, err) + require.NotEmpty(t, existing.ID, + "the CLI decides a name is taken by reading it back, so this must be non-empty") + + weekly, err := buildTrigger(triggerFlags{every: "weekly", onDays: []string{"Friday"}}) + require.NoError(t, err) + _, putErr := client.PutSchedule(context.Background(), name, &eval_api.Schedule{ + DisplayName: name, + Enabled: true, + Trigger: weekly, + Task: &eval_api.ScheduleTask{ + Type: eval_api.ScheduleTaskEvaluation, + EvalID: evalID, + EvalRun: &eval_api.CreateOpenAIEvalRunRequest{ + Name: name, + DataSource: datasetOnlyRows([]map[string]any{{"query": "hello"}}), + }, + }, + }, ProjectEndpointAPIVersion) + + if putErr != nil { + t.Logf("the service refused the edit outright: %v", putErr) + return + } + + after := awaitScheduleSettled(t, client, name) + t.Logf("after editing daily -> weekly the schedule reads back as %q", after.Summary()) + require.Equal(t, eval_api.TriggerRecurrence, after.Trigger.Type) +} From 423035b29fb4779a95bd1b6431020a59218dcd07 Mon Sep 17 00:00:00 2001 From: mohessie Date: Sun, 2 Aug 2026 20:59:35 +0300 Subject: [PATCH 065/320] Move scheduling out of M1 onto its own branch Scheduling works and is now live-tested, but it widens the first release past the loop that matters: define an eval, run it, read the results. It moves to feat/azure-ai-evaluations-schedule, branched from the commit that added its coverage, so it is parked in a known-good state rather than an unverified one. Re-adding it is four files and one line - schedule.go, schedule_test.go, schedule_live_test.go, eval_api/schedules.go, and newScheduleCommand() in root.go. Nothing else ever referenced it. The one entanglement is undone here rather than later: schedules.go also held IsNotFound and IsConflict, and IsNotFound is used by results.go and run_ops.go. Both are generic HTTP predicates with nothing to do with schedules, so they move to eval_api/errors.go, where re-adding schedules will not bring them back. Also add COMMANDS.md: every command, its parameters, an example, and how far each one is actually verified. It records two things the test suite cannot say for itself - no live test drives a CLI command, they all exercise the client layer; and TestLiveRun skips unless AZURE_AI_EVAL_AGENT is set, so agent-target runs are unverified. --- .../azure.ai.evaluations/COMMANDS.md | 193 ++++++ .../azure.ai.evaluations/internal/cmd/root.go | 1 - .../internal/cmd/schedule.go | 645 ------------------ .../internal/cmd/schedule_live_test.go | 397 ----------- .../internal/cmd/schedule_test.go | 168 ----- .../internal/pkg/eval_api/errors.go | 29 + .../internal/pkg/eval_api/schedules.go | 184 ----- 7 files changed, 222 insertions(+), 1395 deletions(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/COMMANDS.md delete mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/schedule.go delete mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/schedule_live_test.go delete mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/schedule_test.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/errors.go delete mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/schedules.go diff --git a/cli/azd/extensions/azure.ai.evaluations/COMMANDS.md b/cli/azd/extensions/azure.ai.evaluations/COMMANDS.md new file mode 100644 index 00000000000..5f58e75f030 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/COMMANDS.md @@ -0,0 +1,193 @@ +# `azd ai eval` — command reference and verification status + +Generated from the built binary at commit `d776507a3`, plus a live run against a real +Foundry project. Not a design document: this records what exists **today** and how much +of it is actually proven. + +## How to read the status column + +| Status | Meaning | +|---|---| +| **LIVE** | A committed live test exercises this path against the real service. | +| **PARTIAL** | The underlying API call is exercised live, but not every flag or branch is. | +| **UNIT** | Unit tests only. No service call is made by this command, or none is covered. | +| **NONE** | No automated coverage. Manually tried at some point, or never run. | + +Two caveats that apply to the whole table, and that the column cannot express: + +1. **No live test drives a CLI command.** Every live test calls the client layer + (`evalClient.CreateOpenAIEval`, `datasetClient.UploadNewVersion`, …) directly. Flag + parsing, prompting, `--no-prompt`, `-o json` rendering and the table output are + covered by unit tests only. So "LIVE" means *the API path this command uses* works, + not that the command itself was run. +2. **`TestLiveRun` skips unless `AZURE_AI_EVAL_AGENT` is set.** It is not set in normal + runs, so the **agent-target** run path is unverified. The dataset-only run path *is* + verified, by `TestLiveCodeEvaluatorScoresARun`. + +Last full live run: **173 passed, 1 skipped, 0 failed**. + +## Global flags + +Available on every command. + +| Flag | Description | +|---|---| +| `-C, --cwd ` | Set the working directory. | +| `--debug` | Debug and diagnostics logging. | +| `-e, --environment ` | azd environment to use. | +| `--no-prompt` | Never prompt. Fails if a required value cannot be resolved. | +| `-o, --output ` | Output format; `json` emits machine-readable output. | + +Most service-touching commands also take `--project-endpoint ` to override the +endpoint resolved from the azd environment. + +## Composite commands + +| Command | Description | Key params | Status | +|---|---|---|---| +| `init` | Scaffold `evals/azure.yaml` + `evals/eval_generate.yaml`. **Makes no service calls.** | `--target`, `--dataset`, `--evaluator` (repeatable), `--judge-model`, `--out-dir` (default `evals`), `--force` | UNIT | +| `generate` | Run the generation jobs, download the rubric and dataset, write `source:` refs into the deploy spec. | `--config` (default `evals/eval_generate.yaml`), `--deploy-config` (default `evals/azure.yaml`), `--target`, `--generation-model`, `--max-samples` (15–1000), `--trace-days`, `--agent-instruction[-file]`, `--dataset`, `--evaluator`, `--no-wait` | NONE | +| `run` | Run an evaluation, creating the eval if it does not exist. | `--config`, `--eval`, `--eval-id`, `--name`, `--level`, `--max-samples`, `--from-traces`, `--trace-window`, `--max-traces`, `--response-id`, `--max-turns`, `--wait` (default true), `--no-wait` | PARTIAL | + +`run` example: + +```console +$ azd ai eval run +Started run evalrun_1f3f909b... on eval eval_9cd479cc... +run reached completed: passed=2 failed=0 errored=0 +``` + +`run` is PARTIAL because the dataset-only path is live-proven while the agent target, +`--from-traces` and `--response-id` are not. + +## `dataset` + +| Command | Description | Key params | Status | +|---|---|---|---| +| `dataset create` | Register a dataset, publishing a new version. | `--name`, `--file` (a `.jsonl` or a directory containing one), `--version` | LIVE | +| `dataset list` | List datasets, or the versions of one. | `--name` | PARTIAL | +| `dataset show` | Show a dataset version. | `--name`, `--version` (omit for latest) | PARTIAL | +| `dataset delete` | Delete a dataset version. | `--name`, `--version` | PARTIAL | + +```console +$ azd ai eval dataset list +NAME VERSION FORMAT URI +support-golden 3 jsonl azureml://.../support-golden/versions/3 +``` + +`TestLiveDatasetLifecycle` covers create, version increment, listing and delete through +the client — hence PARTIAL for the read/delete commands rather than LIVE. + +## `evaluator` + +| Command | Description | Key params | Status | +|---|---|---|---| +| `evaluator create` (rubric) | Register a rubric evaluator. | `--name`, `--rubric ` | NONE | +| `evaluator create` (code) | Register a code evaluator from a **single Python script**. | `--name`, `--file `, `--image-tag`, `--init-params`, `--data-schema`, `--metrics` | LIVE | +| `evaluator list` | List the project's evaluators, versions of one, or the built-ins. | `--name`, `--builtin` | PARTIAL | +| `evaluator show` | Show an evaluator definition. | `--name`, `--version` | NONE | +| `evaluator delete` | Delete an evaluator version. | `--name`, `--version` | PARTIAL | + +```console +$ azd ai eval evaluator create --name answer_length --file ./answer_length.py +Published evaluator answer_length version 1 + +$ azd ai eval evaluator list --builtin +NAME VERSION TYPE +builtin.groundedness 16 builtin +builtin.relevance 12 builtin +``` + +A code evaluator script must declare a **top-level `grade(sample, item)`** returning a +float. It runs as an OpenAI python grader, which receives the script source and nothing +else — there is no import path, so a helper module beside the script cannot be imported. +Dependencies come from `--image-tag`. + +`--rubric`, `evaluator show` are NONE: no live test publishes a rubric or reads a +definition back through them. +`--image-tag` reaches the definition and round-trips, but has **never been exercised +against a real custom image**. + +## `run` subcommands + +| Command | Description | Key params | Status | +|---|---|---|---| +| `run start` | Start a run, creating the eval if needed. Same flags as `run`. | as `run` | PARTIAL | +| `run list` | List runs for an eval. | `[eval-id]`, `--eval`, `--eval-id`, `--limit` | NONE | +| `run show` | Show one run. | `[eval-id]`, `--run-id` (defaults to most recent) | PARTIAL | +| `run cancel` | Cancel an in-flight run. | `[eval-id]`, `--run-id` | PARTIAL | +| `run delete` | Delete a run. | `[eval-id]`, `--run-id` | NONE | + +```console +$ azd ai eval run list +RUN ID NAME STATUS RESULTS +evalrun_1f3f909b... pr-gate-1785370812 completed 2 passed, 0 failed, 0 errored +``` + +Every command taking an eval id accepts it as the argument, as `--eval-id `, or as +`--eval ` to name one from the config. + +## `results` + +| Command | Description | Key params | Status | +|---|---|---|---| +| `results show` | Per-sample results for a run (`output_items`). | ``, `--run-id`, `--failed-only`, `-O/--out-file` | NONE | +| `results export` | Export run results. | ``, `--run-id`, `--format json\|csv`, `-O/--out-file` | NONE | +| `results compare` | Compare runs against a baseline. | `[eval-id]`, `--baseline`, `--treatment` (repeatable), `--name` | NONE | + +The root help still lists `dataset`, `evaluator`, `generate`, `init`, `results` and +`run`. `schedule` is gone from it; see below. + +```console +$ azd ai eval results show +ITEM EVALUATOR RESULT SCORE INPUT REASON +1 answer_length pass 14.0 a short answer - +2 answer_length pass 46.0 a considerably long… - + +$ azd ai eval results compare +METRIC TREATMENT RUN BASELINE TREATMENT DELTA P-VALUE EFFECT +groundedness evalrun_a1b2… 3.80 4.20 +0.40 0.031 small +``` + +`--baseline` defaults to the second most recent completed run and `--treatment` to the +most recent. That auto-selection is untested against real run history. + +**TODO (April, spec review 2026-07-29):** `compare` and `export` belong at the **run** +level, not under `results` — *"compare is not at the items level… export should be at the +run level"*. `results show` should become `run output list`, paginating `output_items`. + +## `schedule` — not on this branch + +Scheduling is implemented and live-tested, but lives on +`feat/azure-ai-evaluations-schedule` rather than here. It is out of M1 so the first +release stays focused on the eval / run / results loop. + +Re-adding it is four files plus one line: `internal/cmd/schedule.go`, +`internal/cmd/schedule_test.go`, `internal/cmd/schedule_live_test.go`, +`internal/pkg/eval_api/schedules.go`, and `newScheduleCommand()` in `root.go`. +Nothing else ever referenced it — the two error helpers it used to carry, +`IsNotFound` and `IsConflict`, now live in `internal/pkg/eval_api/errors.go`, which +is where they belonged anyway. + +What the live tests on that branch establish: + +- Every trigger shape the CLI can emit — cron, hourly, daily, weekly, monthly, + interval, one-time — is accepted by the service and survives a round trip. +- **Schedules need a permission nothing else does.** A schedule fires later and runs + as the project, so the project's managed identity must hold the **Foundry User** + role on the project. Without it every create is refused with `PermissionDenied`. + The tests skip, naming the missing role, rather than reporting a false regression. +- Service constraints: one schedule per project; no in-place edits; a schedule + repeating a `--from-traces` run accepts only `--every hourly`. + +## Summary of gaps + +| Gap | Impact | +|---|---| +| No live test drives a CLI command | Flag parsing, prompting and rendering are unit-tested only | +| `compare` has no live coverage | Named in M1 exit criteria, unproven; needs two completed runs to test | +| `generate` has no live coverage | The most complex composite command | +| `TestLiveRun` skips | Agent-target runs, `--from-traces`, `--response-id` all unverified | +| `--image-tag` never used with a real image | The only supported way to give a code evaluator dependencies | +| `azd up` cannot configure a code evaluator | Reconciler passes empty options; no config fields for `data_schema`, `metrics`, `init_params`, `image_tag` | +| Default metric name `result` is invented | Real evaluators use semantic names (`groundedness`, `relevance`) | diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/root.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/root.go index 118a831fdde..40724c5a0a7 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/root.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/root.go @@ -51,7 +51,6 @@ func NewRootCommand() *cobra.Command { newRunCommand(), newEvaluatorCommand(), newResultsCommand(), - newScheduleCommand(), newListenCommand(), ) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/schedule.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/schedule.go deleted file mode 100644 index e3ce51edee8..00000000000 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/schedule.go +++ /dev/null @@ -1,645 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package cmd - -import ( - "context" - "fmt" - "strconv" - "strings" - "time" - - "azureaieval/internal/pkg/eval_api" - "azureaieval/internal/project" - - "github.com/spf13/cobra" -) - -// newScheduleCommand groups the recurring-evaluation commands. -func newScheduleCommand() *cobra.Command { - cmd := &cobra.Command{ - Use: "schedule", - Short: "Run an eval on a schedule.", - } - cmd.AddCommand( - newScheduleSetCommand(), - newScheduleListCommand(), - newScheduleShowCommand(), - newScheduleDeleteCommand(), - ) - return cmd -} - -// newScheduleSetCommand creates the schedule that runs an eval. -// -// It does not update. The service accepts a PUT over an existing schedule, -// echoes the new body and keeps the old trigger, so an in-place edit would -// report a change that did not happen. Recreating under the same name is not -// an escape either: the replacement never leaves Creating and cannot then be -// deleted. So an existing schedule is refused, and changing one means deleting -// it and creating another under a different name. -func newScheduleSetCommand() *cobra.Command { - var ( - configPath string - groupName string - evalID string - name string - description string - cron string - every string - interval int - atHours []int - onDays []string - onDaysOfMon []int - atTime string - timezone string - startTime string - endTime string - disabled bool - level string - maxSamples int - endpointFlg string - ) - - cmd := &cobra.Command{ - Use: "set [eval-id]", - Short: "Create the schedule that runs an eval.", - Args: cobra.MaximumNArgs(1), - RunE: func(cmd *cobra.Command, args []string) error { - ctx := cmd.Context() - out := cmd.OutOrStdout() - - if len(args) == 1 { - evalID = args[0] - } - - trigger, err := buildTrigger(triggerFlags{ - cron: cron, - every: every, - interval: interval, - atHours: atHours, - onDays: onDays, - onDaysOfMon: onDaysOfMon, - atTime: atTime, - timezone: timezone, - startTime: startTime, - endTime: endTime, - }) - if err != nil { - return err - } - - ec, err := newEvalContext(ctx, endpointFlg) - if err != nil { - return err - } - defer ec.Close() - - // Same resolution as `run`: the config names the group unless - // --eval-id bypasses it, and the run payload carries the target - // and dataset because the group holds neither. - var group *project.Eval - var dataSource *eval_api.EvalRunDataSource - if evalID == "" { - cfg, err := project.LoadEvalConfig(configPath) - if err != nil { - return err - } - if err := cfg.Validate(); err != nil { - return err - } - group, err = cfg.ResolveGroup(groupName) - if err != nil { - return err - } - if err := ec.checkDatasetRegistered(ctx, cfg, group, configPath); err != nil { - return err - } - evalID, err = ec.resolveEvalIDFromConfig( - ctx, group, configPath, resolveLevel(level, group), - len(cfg.Evals) == 1, out, isJSON(cmd)) - if err != nil { - return err - } - dataSource, err = ec.buildRunDataSource( - ctx, group, configPath, resolveMaxSamples(maxSamples, group)) - if err != nil { - return err - } - } else { - dataSource, err = ec.reuseDataSourceFromLastRun(ctx, evalID) - if err != nil { - return err - } - } - - if name == "" { - name = defaultScheduleName(group) - } - if description == "" { - description = fmt.Sprintf("Scheduled evaluation of %s.", evalID) - } - - // An existing schedule cannot be edited: the service takes the PUT - // and ignores it. Recreating under the same name is worse — the - // replacement sticks in Creating and cannot be deleted — so the - // only safe answer is a different name. - if existing, err := ec.evalClient.GetSchedule( - ctx, name, ProjectEndpointAPIVersion); err == nil && existing != nil && existing.ID != "" { - return fmt.Errorf( - "schedule %q already exists, and the service ignores edits to it. "+ - "Delete it with `azd ai eval schedule delete %s` and create the new "+ - "one under a different name; reusing this one leaves it stuck", - name, name) - } - - metadata := map[string]string{} - if lvl := resolveLevel(level, group); lvl != "" { - metadata["evaluation_level"] = lvl - } - - schedule := &eval_api.Schedule{ - DisplayName: name, - Description: description, - Enabled: !disabled, - Trigger: trigger, - Task: &eval_api.ScheduleTask{ - Type: eval_api.ScheduleTaskEvaluation, - EvalID: evalID, - EvalRun: &eval_api.CreateOpenAIEvalRunRequest{ - Name: name, - DataSource: dataSource, - Metadata: metadata, - }, - }, - } - - saved, err := ec.evalClient.PutSchedule(ctx, name, schedule, ProjectEndpointAPIVersion) - if err != nil { - return explainScheduleFailure(ctx, ec, name, err) - } - - if isJSON(cmd) { - return emitJSON(out, saved) - } - state := "enabled" - if !saved.Enabled { - state = "disabled" - } - fmt.Fprintf(out, "Schedule %s (%s) runs %s on %s\n", - saved.ID, state, saved.Summary(), evalID) - return nil - }, - } - - cmd.Flags().StringVar(&configPath, "config", project.DefaultDeployConfig, - "Path to the eval deployment config.") - cmd.Flags().StringVar(&groupName, "eval", "", "Which evals entry to schedule.") - cmd.Flags().StringVar(&evalID, "eval-id", "", "Schedule an existing eval by id, ignoring config.") - cmd.Flags().StringVar(&name, "name", "", "Schedule name. Defaults to the eval name.") - cmd.Flags().StringVar(&description, "description", "", "Schedule description.") - cmd.Flags().StringVar(&cron, "cron", "", `Cron expression, for example "0 9 * * *".`) - cmd.Flags().StringVar(&every, "every", "", - "Recur hourly, daily, weekly or monthly.") - cmd.Flags().IntVar(&interval, "interval", 0, "Repeat every N periods of --every. Defaults to 1.") - cmd.Flags().IntSliceVar(&atHours, "at", nil, "Hours of the day for --every daily, 0-23.") - cmd.Flags().StringSliceVar(&onDays, "on", nil, "Days of the week for --every weekly, for example Monday.") - cmd.Flags().IntSliceVar(&onDaysOfMon, "on-day", nil, "Days of the month for --every monthly, 1-31.") - cmd.Flags().StringVar(&atTime, "at-time", "", "Run once at this RFC3339 time.") - cmd.Flags().StringVar(&timezone, "timezone", "", "Timezone for the trigger. Defaults to UTC.") - cmd.Flags().StringVar(&startTime, "start-time", "", "RFC3339 time before which the schedule does not fire.") - cmd.Flags().StringVar(&endTime, "end-time", "", "RFC3339 time after which the schedule stops firing.") - cmd.Flags().BoolVar(&disabled, "disabled", false, "Create the schedule without enabling it.") - cmd.Flags().StringVar(&level, "level", "", "Evaluation level for the scheduled runs.") - cmd.Flags().IntVar(&maxSamples, "max-samples", 0, "Cap rows sent from a local dataset file.") - cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") - cmd.MarkFlagsMutuallyExclusive("cron", "every", "at-time") - - return cmd -} - -func newScheduleListCommand() *cobra.Command { - var endpointFlg string - - cmd := &cobra.Command{ - Use: "list", - Short: "List the project's schedules.", - RunE: func(cmd *cobra.Command, args []string) error { - ctx := cmd.Context() - out := cmd.OutOrStdout() - - ec, err := newEvalContext(ctx, endpointFlg) - if err != nil { - return err - } - defer ec.Close() - - list, err := ec.evalClient.ListSchedules(ctx, ProjectEndpointAPIVersion) - if err != nil { - return fmt.Errorf("listing schedules: %w", err) - } - if isJSON(cmd) { - var schedules []eval_api.Schedule - if list != nil { - schedules = list.Value - } - return emitJSONList(out, schedules) - } - if list == nil || len(list.Value) == 0 { - fmt.Fprintln(out, "No schedules.") - return nil - } - - rows := make([][]string, 0, len(list.Value)) - for i := range list.Value { - s := &list.Value[i] - evalID := "" - if s.Task != nil { - evalID = s.Task.EvalID - } - rows = append(rows, []string{ - s.ID, - strconv.FormatBool(s.Enabled), - s.ProvisioningStatus, - s.Summary(), - evalID, - }) - } - return emitTable(out, - []string{"NAME", "ENABLED", "STATUS", "TRIGGER", "EVAL"}, rows) - }, - } - cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") - return cmd -} - -func newScheduleShowCommand() *cobra.Command { - var ( - name string - endpointFlg string - ) - - cmd := &cobra.Command{ - Use: "show [name]", - Short: "Show one schedule.", - Args: cobra.MaximumNArgs(1), - RunE: func(cmd *cobra.Command, args []string) error { - ctx := cmd.Context() - out := cmd.OutOrStdout() - - if len(args) == 1 { - name = args[0] - } - if name == "" { - return requireFlag("name") - } - - ec, err := newEvalContext(ctx, endpointFlg) - if err != nil { - return err - } - defer ec.Close() - - s, err := ec.evalClient.GetSchedule(ctx, name, ProjectEndpointAPIVersion) - if err != nil { - return fmt.Errorf("reading schedule %q: %w", name, err) - } - if isJSON(cmd) { - return emitJSON(out, s) - } - - fmt.Fprintf(out, "Schedule %s\n", s.ID) - fmt.Fprintf(out, " enabled: %t\n", s.Enabled) - fmt.Fprintf(out, " status: %s\n", s.ProvisioningStatus) - fmt.Fprintf(out, " trigger: %s\n", s.Summary()) - if s.Trigger != nil && s.Trigger.Timezone != "" { - fmt.Fprintf(out, " timezone: %s\n", s.Trigger.Timezone) - } - if s.Task != nil { - fmt.Fprintf(out, " eval: %s\n", s.Task.EvalID) - } - if s.Description != "" { - fmt.Fprintf(out, " about: %s\n", s.Description) - } - return nil - }, - } - cmd.Flags().StringVar(&name, "name", "", "Schedule name.") - cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") - return cmd -} - -func newScheduleDeleteCommand() *cobra.Command { - var ( - name string - endpointFlg string - ) - - cmd := &cobra.Command{ - Use: "delete [name]", - Short: "Delete a schedule.", - Args: cobra.MaximumNArgs(1), - RunE: func(cmd *cobra.Command, args []string) error { - ctx := cmd.Context() - out := cmd.OutOrStdout() - - if len(args) == 1 { - name = args[0] - } - if name == "" { - return requireFlag("name") - } - - ec, err := newEvalContext(ctx, endpointFlg) - if err != nil { - return err - } - defer ec.Close() - - if err := ec.deleteScheduleWhenSettled(ctx, name); err != nil { - // A name that was never there is the common typo, and the - // service answers it with a full error document wrapping an - // inner 404 from the trigger service. Saying so in one line is - // more use than reproducing that. - if eval_api.IsNotFound(err) { - return fmt.Errorf( - "no schedule named %q in this project; "+ - "`azd ai eval schedule list` shows the ones that exist", name) - } - return fmt.Errorf("deleting schedule %q: %w", name, err) - } - fmt.Fprintf(out, "Deleted schedule %s\n", name) - return nil - }, - } - cmd.Flags().StringVar(&name, "name", "", "Schedule name.") - cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") - return cmd -} - -// triggerFlags carries the schedule's timing flags so buildTrigger can be -// tested without a command. -type triggerFlags struct { - cron string - every string - interval int - atHours []int - onDays []string - onDaysOfMon []int - atTime string - timezone string - startTime string - endTime string -} - -// buildTrigger turns the timing flags into the trigger the API expects. -func buildTrigger(f triggerFlags) (*eval_api.ScheduleTrigger, error) { - tz := f.timezone - if tz == "" { - tz = "UTC" - } - - switch { - case f.cron != "": - return &eval_api.ScheduleTrigger{ - Type: eval_api.TriggerCron, - Expression: f.cron, - StartTime: f.startTime, - EndTime: f.endTime, - Timezone: tz, - }, nil - - case f.atTime != "": - if _, err := time.Parse(time.RFC3339, f.atTime); err != nil { - return nil, fmt.Errorf("--at-time %q is not an RFC3339 time", f.atTime) - } - return &eval_api.ScheduleTrigger{ - Type: eval_api.TriggerOneTime, - ScheduledTime: f.atTime, - Timezone: tz, - }, nil - - case f.every != "": - pattern, err := buildRecurrence(f) - if err != nil { - return nil, err - } - interval := f.interval - if interval <= 0 { - interval = 1 - } - return &eval_api.ScheduleTrigger{ - Type: eval_api.TriggerRecurrence, - Schedule: pattern, - Interval: interval, - StartTime: f.startTime, - EndTime: f.endTime, - Timezone: tz, - }, nil - } - - return nil, fmt.Errorf("a schedule needs a trigger: pass --cron, --every or --at-time") -} - -// buildRecurrence maps --every and its qualifiers onto a recurrence pattern. -// -// Each period reads only its own qualifier, so passing one that does not apply -// is rejected rather than dropped. -func buildRecurrence(f triggerFlags) (*eval_api.RecurrencePattern, error) { - period := strings.ToLower(strings.TrimSpace(f.every)) - - reject := func(flag, applies string) error { - return fmt.Errorf("--%s applies to --every %s, not %s", flag, applies, period) - } - - switch period { - case "hourly": - if len(f.atHours) > 0 { - return nil, reject("at", "daily") - } - if len(f.onDays) > 0 { - return nil, reject("on", "weekly") - } - if len(f.onDaysOfMon) > 0 { - return nil, reject("on-day", "monthly") - } - return &eval_api.RecurrencePattern{Type: eval_api.RecurrenceHourly}, nil - - case "daily": - if len(f.onDays) > 0 { - return nil, reject("on", "weekly") - } - if len(f.onDaysOfMon) > 0 { - return nil, reject("on-day", "monthly") - } - for _, h := range f.atHours { - if h < 0 || h > 23 { - return nil, fmt.Errorf("--at %d is not an hour of the day (0-23)", h) - } - } - return &eval_api.RecurrencePattern{Type: eval_api.RecurrenceDaily, Hours: f.atHours}, nil - - case "weekly": - if len(f.atHours) > 0 { - return nil, reject("at", "daily") - } - if len(f.onDaysOfMon) > 0 { - return nil, reject("on-day", "monthly") - } - days, err := normalizeDaysOfWeek(f.onDays) - if err != nil { - return nil, err - } - return &eval_api.RecurrencePattern{Type: eval_api.RecurrenceWeekly, DaysOfWeek: days}, nil - - case "monthly": - if len(f.atHours) > 0 { - return nil, reject("at", "daily") - } - if len(f.onDays) > 0 { - return nil, reject("on", "weekly") - } - for _, d := range f.onDaysOfMon { - if d < 1 || d > 31 { - return nil, fmt.Errorf("--on-day %d is not a day of the month (1-31)", d) - } - } - return &eval_api.RecurrencePattern{Type: eval_api.RecurrenceMonthly, DaysOfMonth: f.onDaysOfMon}, nil - } - - return nil, fmt.Errorf( - "--every %q is not a recurrence: use hourly, daily, weekly or monthly", f.every) -} - -// normalizeDaysOfWeek accepts day names in any casing and returns the spelling -// the service expects. -func normalizeDaysOfWeek(days []string) ([]string, error) { - if len(days) == 0 { - return nil, nil - } - canonical := map[string]string{} - for d := time.Sunday; d <= time.Saturday; d++ { - canonical[strings.ToLower(d.String())] = d.String() - } - - out := make([]string, 0, len(days)) - for _, raw := range days { - name, ok := canonical[strings.ToLower(strings.TrimSpace(raw))] - if !ok { - return nil, fmt.Errorf("--on %q is not a day of the week", raw) - } - out = append(out, name) - } - return out, nil -} - -// defaultScheduleName derives a schedule name from the group being scheduled. -func defaultScheduleName(group *project.Eval) string { - if group != nil && group.Name != "" { - return group.Name - } - return "eval-" + strconv.FormatInt(time.Now().UTC().Unix(), 10) -} - -// deleteScheduleWhenSettled removes a schedule, waiting out the window where -// the service is still provisioning it. -// -// A schedule that is mid-provision refuses the delete, and does it two ways: -// 409 while it is busy, or 404 because the trigger behind it does not exist -// yet. Either way the caller neither caused it nor can see it, so the wait -// happens here. -func (ec *evalContext) deleteScheduleWhenSettled(ctx context.Context, name string) error { - const attempts = 30 - - for i := 0; i < attempts; i++ { - s, err := ec.evalClient.GetSchedule(ctx, name, ProjectEndpointAPIVersion) - if err != nil || s == nil || s.ID == "" { - // Nothing to wait for: let the delete report what it finds. - break - } - if s.Settled() { - break - } - select { - case <-ctx.Done(): - return ctx.Err() - case <-time.After(4 * time.Second): - } - } - - for i := 0; ; i++ { - err := ec.evalClient.DeleteSchedule(ctx, name, ProjectEndpointAPIVersion) - if err == nil || !eval_api.IsConflict(err) || i == attempts-1 { - return err - } - select { - case <-ctx.Done(): - return ctx.Err() - case <-time.After(4 * time.Second): - } - } -} - -// explainScheduleFailure turns the service's bodiless rejection into the reason -// it actually happened. -// -// The project accepts one schedule at a time and refuses a second with a 400 -// carrying no message, so the count is what explains it. -func explainScheduleFailure( - ctx context.Context, - ec *evalContext, - name string, - cause error, -) error { - // A schedule repeats the group's most recent run, so scheduling a group - // whose last run came from --from-traces creates a trace evaluation, and - // the service allows only an hourly trigger for those. The message it - // returns says so without saying why it thinks the schedule is one, which - // is bewildering when the trigger was the only thing asked for. - if isTracesHourlyOnly(cause) { - return fmt.Errorf( - "saving schedule %q: this eval's most recent run read from traces, and a schedule "+ - "repeats that run, so the service treats it as a scheduled trace evaluation "+ - "and allows only `--every hourly`. Use `--every hourly`, or run the eval "+ - "once against its dataset first so the schedule repeats that instead", name) - } - - if isScheduleRoleMissing(cause) { - return fmt.Errorf( - "saving schedule %q: the project's managed identity needs the Foundry User "+ - "role on the project before it can run scheduled evaluations. A schedule "+ - "runs later, as the project rather than as you, which is why creating one "+ - "needs a role that running an eval yourself does not. Grant it on the "+ - "project and retry", name) - } - - list, listErr := ec.evalClient.ListSchedules(ctx, ProjectEndpointAPIVersion) - if listErr != nil || list == nil { - return fmt.Errorf("saving schedule %q: %w", name, cause) - } - - for i := range list.Value { - if other := list.Value[i].ID; other != "" && other != name { - return fmt.Errorf( - "saving schedule %q: the project already has a schedule, %q, and only one is "+ - "allowed at a time. Delete it first with "+ - "`azd ai eval schedule delete %s`", name, other, other) - } - } - return fmt.Errorf("saving schedule %q: %w", name, cause) -} - -// isTracesHourlyOnly matches the service's refusal of a non-hourly trigger on a -// schedule it considers a trace evaluation. -func isTracesHourlyOnly(err error) bool { - return err != nil && - strings.Contains(err.Error(), "trace evaluations only support hourly") -} - -// isScheduleRoleMissing matches the refusal when the project identity cannot -// run the evaluation the schedule would trigger. -func isScheduleRoleMissing(err error) bool { - return err != nil && - strings.Contains(err.Error(), "lacks Foundry User role") -} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/schedule_live_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/schedule_live_test.go deleted file mode 100644 index ebd8da98570..00000000000 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/schedule_live_test.go +++ /dev/null @@ -1,397 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -//go:build live - -package cmd - -import ( - "context" - "fmt" - "sort" - "testing" - "time" - - "azureaieval/internal/pkg/eval_api" - "azureaieval/internal/pkg/evalcore" - "azureaieval/internal/project" - - "github.com/stretchr/testify/require" -) - -// Schedules are project-scoped and the service tolerates very few of them, so -// everything here runs serially and deletes what it creates. Nothing calls -// t.Parallel(). -// -// These tests exist because the unit tests only prove that buildTrigger -// produces a well-formed struct. Whether the service accepts that struct is a -// different question, and it is the one that matters: every trigger shape the -// CLI can emit is sent here and required to survive a round trip. - -// liveScheduleEval creates a throwaway eval for a schedule to point at. -// -// The criterion comes from the shipping builder rather than a hand-written -// one. Built-ins do not share an input contract — builtin.ifeval, which is -// what the listing happens to return first, needs an instruction_id_list — so -// a hand-rolled mapping is rejected with MissingRequiredDataMapping. Letting -// production shape the request also means this helper cannot drift from it. -// -// No evaluator is named. Which built-ins a project exposes varies, so the -// first one the builder can satisfy is used, and the dataset is given every -// column that evaluator declares. -func liveScheduleEval(t *testing.T, client *eval_api.EvalClient, judge string) string { - t.Helper() - ctx := context.Background() - - ec := &evalContext{evalClient: client} - schemas := ec.evaluatorSchemas(ctx) - require.NotEmpty(t, schemas, "need the published evaluator contracts to build an eval") - - names := make([]string, 0, len(schemas)) - for name := range schemas { - names = append(names, name) - } - sort.Strings(names) - - for _, name := range names { - summary := schemas[name] - columns := map[string]bool{"query": true} - if ds := summary.DataSchema(); ds != nil { - for _, col := range ds.PropertyNames() { - columns[col] = true - } - } - - level := "" - if len(summary.SupportedEvaluationLevels) > 0 { - level = summary.SupportedEvaluationLevels[0] - } - - req, err := buildEvalRequest(&project.Eval{ - Name: fmt.Sprintf("azd-sched-%d", time.Now().UTC().UnixNano()), - Dataset: "inline", - Target: &project.Target{Type: "agent", Name: "probe-agent"}, - Evaluators: []evalcore.EvaluatorRef{{Name: summary.Name}}, - Options: &project.Options{EvalModel: judge, EvaluationLevel: level}, - }, schemas, columns) - if err != nil { - continue - } - - created, err := client.CreateOpenAIEval(ctx, req) - if err != nil { - t.Logf("built-in %s could not back a schedule: %v", name, err) - continue - } - t.Cleanup(func() { - _ = client.DeleteOpenAIEval(context.Background(), created.ID) - }) - t.Logf("scheduling an eval built on %s (%s)", name, created.ID) - return created.ID - } - - t.Fatalf("no built-in produced an eval a schedule could run; tried %d", len(names)) - return "" -} - -// putLiveSchedule creates a schedule and registers its removal. -func putLiveSchedule( - t *testing.T, - client *eval_api.EvalClient, - name string, - evalID string, - trigger *eval_api.ScheduleTrigger, -) (*eval_api.Schedule, error) { - t.Helper() - - saved, err := client.PutSchedule(context.Background(), name, &eval_api.Schedule{ - DisplayName: name, - Enabled: true, - Trigger: trigger, - Task: &eval_api.ScheduleTask{ - Type: eval_api.ScheduleTaskEvaluation, - EvalID: evalID, - EvalRun: &eval_api.CreateOpenAIEvalRunRequest{ - Name: name, - DataSource: datasetOnlyRows([]map[string]any{{"query": "how do I reset my password?"}}), - }, - }, - }, ProjectEndpointAPIVersion) - - if err == nil { - t.Cleanup(func() { removeLiveSchedule(t, client, name) }) - } - return saved, err -} - -// skipIfScheduleRoleMissing stops the test when the project cannot host a -// schedule at all. -// -// A schedule fires later and runs as the project, so creating one requires the -// project's managed identity to hold the Foundry User role on the project — -// a permission no other command in this extension needs. Without it every -// schedule test fails identically and for a reason that has nothing to do with -// the code, so they skip loudly instead of reporting a false regression. -func skipIfScheduleRoleMissing(t *testing.T, err error) { - t.Helper() - if err == nil || !isScheduleRoleMissing(err) { - return - } - t.Skipf("this project cannot host schedules: its managed identity lacks the "+ - "Foundry User role on the project. Grant it and re-run to exercise "+ - "schedules for real. Underlying error: %v", err) -} - -// datasetOnlyRows is the inline data source a scheduled run repeats. -func datasetOnlyRows(rows []map[string]any) *eval_api.EvalRunDataSource { - ds := eval_api.NewDatasetOnlyDataSource() - ds.SetFileContent(rows) - return ds -} - -// removeLiveSchedule deletes a schedule once it is no longer provisioning. -// -// A schedule still being created refuses the delete, as a 409 while it is busy -// or a 404 because the trigger behind it does not exist yet. Waiting for it to -// settle is what makes cleanup reliable, and leaving one behind would break -// every later test in this file, because the project holds very few. -func removeLiveSchedule(t *testing.T, client *eval_api.EvalClient, name string) { - t.Helper() - ctx := context.Background() - - deadline := time.Now().Add(2 * time.Minute) - for { - current, err := client.GetSchedule(ctx, name, ProjectEndpointAPIVersion) - if err != nil { - return // already gone - } - if current.Settled() { - break - } - if time.Now().After(deadline) { - t.Logf("schedule %q never settled (status %q); leaving it", name, current.ProvisioningStatus) - return - } - time.Sleep(3 * time.Second) - } - - if err := client.DeleteSchedule(ctx, name, ProjectEndpointAPIVersion); err != nil { - t.Logf("could not delete schedule %q: %v", name, err) - } -} - -// awaitScheduleSettled blocks until the schedule finishes provisioning. -func awaitScheduleSettled( - t *testing.T, - client *eval_api.EvalClient, - name string, -) *eval_api.Schedule { - t.Helper() - ctx := context.Background() - - deadline := time.Now().Add(2 * time.Minute) - for { - current, err := client.GetSchedule(ctx, name, ProjectEndpointAPIVersion) - require.NoError(t, err, "reading schedule %q back", name) - if current.Settled() { - return current - } - if time.Now().After(deadline) { - t.Fatalf("schedule %q stuck in %q", name, current.ProvisioningStatus) - } - time.Sleep(3 * time.Second) - } -} - -func liveScheduleName(t *testing.T, suffix string) string { - t.Helper() - return fmt.Sprintf("azdsched-%s-%d", suffix, time.Now().UnixNano()) -} - -// TestLiveScheduleLifecycle walks create, read, list and delete. -func TestLiveScheduleLifecycle(t *testing.T) { - client, judge := liveEvalClient(t) - ctx := context.Background() - - evalID := liveScheduleEval(t, client, judge) - name := liveScheduleName(t, "life") - - trigger, err := buildTrigger(triggerFlags{every: "daily", atHours: []int{9}}) - require.NoError(t, err) - - saved, err := putLiveSchedule(t, client, name, evalID, trigger) - skipIfScheduleRoleMissing(t, err) - require.NoError(t, err, "the service rejected a trigger the CLI can produce") - require.NotEmpty(t, saved.ID) - - settled := awaitScheduleSettled(t, client, name) - require.True(t, settled.Enabled, "a schedule created without --disabled must be enabled") - require.NotNil(t, settled.Task, "the schedule must carry the task it runs") - require.Equal(t, evalID, settled.Task.EvalID, - "the schedule must point at the eval it was given") - require.NotNil(t, settled.Trigger) - require.Equal(t, eval_api.TriggerRecurrence, settled.Trigger.Type) - - list, err := client.ListSchedules(ctx, ProjectEndpointAPIVersion) - require.NoError(t, err) - found := false - for i := range list.Value { - if list.Value[i].ID == saved.ID || list.Value[i].DisplayName == name { - found = true - break - } - } - require.True(t, found, "a created schedule must appear in the listing") - - removeLiveSchedule(t, client, name) - - _, err = client.GetSchedule(ctx, name, ProjectEndpointAPIVersion) - require.Error(t, err, "a deleted schedule must not read back") - require.True(t, eval_api.IsNotFound(err), - "deleting should leave a not-found, got %v", err) -} - -// TestLiveScheduleAcceptsEveryTriggerShape sends one schedule per trigger the -// CLI can build and requires the service to accept each. -// -// The unit tests assert the shape of what buildTrigger returns. They cannot -// say whether the service agrees, and a trigger the service rejects is a -// trigger the CLI should never have offered. -func TestLiveScheduleAcceptsEveryTriggerShape(t *testing.T) { - client, judge := liveEvalClient(t) - evalID := liveScheduleEval(t, client, judge) - - // Probe once up front. Skipping inside the subtests instead would leave the - // parent reporting PASS with nothing proven, which is worse than a failure - // because it looks like coverage. - probe := liveScheduleName(t, "probe") - daily, err := buildTrigger(triggerFlags{every: "daily", atHours: []int{4}}) - require.NoError(t, err) - if _, probeErr := putLiveSchedule(t, client, probe, evalID, daily); probeErr != nil { - skipIfScheduleRoleMissing(t, probeErr) - require.NoError(t, probeErr, "could not create the probe schedule") - } - removeLiveSchedule(t, client, probe) - - cases := []struct { - label string - flags triggerFlags - want string - }{ - {"cron", triggerFlags{cron: "0 9 * * *"}, eval_api.TriggerCron}, - {"hourly", triggerFlags{every: "hourly"}, eval_api.TriggerRecurrence}, - {"daily", triggerFlags{every: "daily", atHours: []int{9, 17}}, eval_api.TriggerRecurrence}, - {"weekly", triggerFlags{every: "weekly", onDays: []string{"Monday"}}, eval_api.TriggerRecurrence}, - {"monthly", triggerFlags{every: "monthly", onDaysOfMon: []int{1}}, eval_api.TriggerRecurrence}, - {"interval", triggerFlags{every: "daily", interval: 3}, eval_api.TriggerRecurrence}, - { - "onetime", - triggerFlags{atTime: time.Now().UTC().Add(24 * time.Hour).Format(time.RFC3339)}, - eval_api.TriggerOneTime, - }, - } - - for _, tc := range cases { - t.Run(tc.label, func(t *testing.T) { - trigger, err := buildTrigger(tc.flags) - require.NoError(t, err, "the CLI could not build a %s trigger", tc.label) - require.Equal(t, tc.want, trigger.Type) - - name := liveScheduleName(t, tc.label) - saved, err := putLiveSchedule(t, client, name, evalID, trigger) - skipIfScheduleRoleMissing(t, err) - require.NoError(t, err, "the service rejected the %s trigger", tc.label) - require.NotEmpty(t, saved.ID) - - settled := awaitScheduleSettled(t, client, name) - require.Equal(t, tc.want, settled.Trigger.Type, - "the trigger type must survive the round trip") - removeLiveSchedule(t, client, name) - }) - } -} - -// TestLiveScheduleDisabledStaysDisabled covers --disabled, which is the one -// flag whose whole purpose is a field the service could quietly ignore. -func TestLiveScheduleDisabledStaysDisabled(t *testing.T) { - client, judge := liveEvalClient(t) - evalID := liveScheduleEval(t, client, judge) - name := liveScheduleName(t, "disabled") - - trigger, err := buildTrigger(triggerFlags{every: "daily", atHours: []int{3}}) - require.NoError(t, err) - - saved, err := client.PutSchedule(context.Background(), name, &eval_api.Schedule{ - DisplayName: name, - Enabled: false, - Trigger: trigger, - Task: &eval_api.ScheduleTask{ - Type: eval_api.ScheduleTaskEvaluation, - EvalID: evalID, - EvalRun: &eval_api.CreateOpenAIEvalRunRequest{ - Name: name, - DataSource: datasetOnlyRows([]map[string]any{{"query": "hello"}}), - }, - }, - }, ProjectEndpointAPIVersion) - skipIfScheduleRoleMissing(t, err) - require.NoError(t, err) - t.Cleanup(func() { removeLiveSchedule(t, client, name) }) - require.NotEmpty(t, saved.ID) - - settled := awaitScheduleSettled(t, client, name) - require.False(t, settled.Enabled, - "a schedule created disabled must not come back enabled") -} - -// TestLiveScheduleEditIsRefusedByTheCLI pins the reason `schedule set` refuses -// to reuse a name. -// -// The service takes a PUT on an existing schedule and does not apply it, and a -// replacement can stick in Creating where it can no longer be deleted. The CLI -// therefore refuses before sending. This test records the service behaviour -// the guard exists for, so a change in the service is visible here rather than -// as a stuck schedule in someone's project. -func TestLiveScheduleEditIsRefusedByTheCLI(t *testing.T) { - client, judge := liveEvalClient(t) - evalID := liveScheduleEval(t, client, judge) - name := liveScheduleName(t, "edit") - - daily, err := buildTrigger(triggerFlags{every: "daily", atHours: []int{9}}) - require.NoError(t, err) - _, err = putLiveSchedule(t, client, name, evalID, daily) - skipIfScheduleRoleMissing(t, err) - require.NoError(t, err) - awaitScheduleSettled(t, client, name) - - // The guard in `schedule set` is a GetSchedule that must find this. - existing, err := client.GetSchedule(context.Background(), name, ProjectEndpointAPIVersion) - require.NoError(t, err) - require.NotEmpty(t, existing.ID, - "the CLI decides a name is taken by reading it back, so this must be non-empty") - - weekly, err := buildTrigger(triggerFlags{every: "weekly", onDays: []string{"Friday"}}) - require.NoError(t, err) - _, putErr := client.PutSchedule(context.Background(), name, &eval_api.Schedule{ - DisplayName: name, - Enabled: true, - Trigger: weekly, - Task: &eval_api.ScheduleTask{ - Type: eval_api.ScheduleTaskEvaluation, - EvalID: evalID, - EvalRun: &eval_api.CreateOpenAIEvalRunRequest{ - Name: name, - DataSource: datasetOnlyRows([]map[string]any{{"query": "hello"}}), - }, - }, - }, ProjectEndpointAPIVersion) - - if putErr != nil { - t.Logf("the service refused the edit outright: %v", putErr) - return - } - - after := awaitScheduleSettled(t, client, name) - t.Logf("after editing daily -> weekly the schedule reads back as %q", after.Summary()) - require.Equal(t, eval_api.TriggerRecurrence, after.Trigger.Type) -} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/schedule_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/schedule_test.go deleted file mode 100644 index f518397d843..00000000000 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/schedule_test.go +++ /dev/null @@ -1,168 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package cmd - -import ( - "errors" - "testing" - - "azureaieval/internal/pkg/eval_api" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestBuildTrigger_Cron(t *testing.T) { - got, err := buildTrigger(triggerFlags{cron: "0 9 * * *"}) - require.NoError(t, err) - assert.Equal(t, eval_api.TriggerCron, got.Type) - assert.Equal(t, "0 9 * * *", got.Expression) - assert.Equal(t, "UTC", got.Timezone, "UTC unless the caller says otherwise") -} - -// A schedule repeats the group's most recent run. Scheduling a group whose last -// run came from --from-traces therefore creates a trace evaluation, and the -// service allows only an hourly trigger for those. Confirmed live: a daily -// trigger was accepted after an agent run, refused after a traces run on the -// same group, and hourly was accepted for that same traces run. -func TestIsTracesHourlyOnly(t *testing.T) { - assert.True(t, isTracesHourlyOnly( - errors.New(`{"message": "Scheduled trace evaluations only support hourly recurrence triggers. is invalid"}`))) - assert.False(t, isTracesHourlyOnly(errors.New("some other 400"))) - assert.False(t, isTracesHourlyOnly(nil)) -} - -func TestBuildTrigger_OneTime(t *testing.T) { - got, err := buildTrigger(triggerFlags{atTime: "2026-08-01T09:00:00Z", timezone: "Europe/Dublin"}) - require.NoError(t, err) - assert.Equal(t, eval_api.TriggerOneTime, got.Type) - assert.Equal(t, "2026-08-01T09:00:00Z", got.ScheduledTime) - assert.Equal(t, "Europe/Dublin", got.Timezone) - - _, err = buildTrigger(triggerFlags{atTime: "next tuesday"}) - require.ErrorContains(t, err, "RFC3339") -} - -func TestBuildTrigger_Recurrence(t *testing.T) { - cases := []struct { - name string - flags triggerFlags - wantType string - assert func(*testing.T, *eval_api.RecurrencePattern) - }{ - { - name: "hourly", - flags: triggerFlags{every: "hourly", interval: 6}, - wantType: eval_api.RecurrenceHourly, - }, - { - name: "daily with hours", - flags: triggerFlags{every: "Daily", atHours: []int{9, 17}}, - wantType: eval_api.RecurrenceDaily, - assert: func(t *testing.T, p *eval_api.RecurrencePattern) { - assert.Equal(t, []int{9, 17}, p.Hours) - }, - }, - { - name: "weekly normalizes day casing", - flags: triggerFlags{every: "weekly", onDays: []string{"monday", "THURSDAY"}}, - wantType: eval_api.RecurrenceWeekly, - assert: func(t *testing.T, p *eval_api.RecurrencePattern) { - assert.Equal(t, []string{"Monday", "Thursday"}, p.DaysOfWeek) - }, - }, - { - name: "monthly", - flags: triggerFlags{every: "monthly", onDaysOfMon: []int{1, 15}}, - wantType: eval_api.RecurrenceMonthly, - assert: func(t *testing.T, p *eval_api.RecurrencePattern) { - assert.Equal(t, []int{1, 15}, p.DaysOfMonth) - }, - }, - } - - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - got, err := buildTrigger(tc.flags) - require.NoError(t, err) - assert.Equal(t, eval_api.TriggerRecurrence, got.Type) - require.NotNil(t, got.Schedule) - assert.Equal(t, tc.wantType, got.Schedule.Type) - if tc.assert != nil { - tc.assert(t, got.Schedule) - } - }) - } -} - -// An interval is always sent, so the service never has to infer one. -func TestBuildTrigger_IntervalDefaultsToOne(t *testing.T) { - got, err := buildTrigger(triggerFlags{every: "daily"}) - require.NoError(t, err) - assert.Equal(t, 1, got.Interval) - - got, err = buildTrigger(triggerFlags{every: "daily", interval: 3}) - require.NoError(t, err) - assert.Equal(t, 3, got.Interval) -} - -// Each period reads only its own qualifier. Accepting one that does not apply -// would drop it silently, which is the failure mode the trace fields already -// taught us to avoid. -func TestBuildTrigger_RejectsQualifiersFromAnotherPeriod(t *testing.T) { - cases := []struct { - name string - flags triggerFlags - want string - }{ - {"hours on weekly", triggerFlags{every: "weekly", atHours: []int{9}}, "--at applies to --every daily"}, - {"days on daily", triggerFlags{every: "daily", onDays: []string{"Monday"}}, "--on applies to --every weekly"}, - {"month days on hourly", triggerFlags{every: "hourly", onDaysOfMon: []int{1}}, "--on-day applies to --every monthly"}, - {"hours on monthly", triggerFlags{every: "monthly", atHours: []int{9}}, "--at applies to --every daily"}, - } - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - _, err := buildTrigger(tc.flags) - require.ErrorContains(t, err, tc.want) - }) - } -} - -func TestBuildTrigger_RejectsOutOfRangeValues(t *testing.T) { - _, err := buildTrigger(triggerFlags{every: "daily", atHours: []int{24}}) - require.ErrorContains(t, err, "hour of the day") - - _, err = buildTrigger(triggerFlags{every: "monthly", onDaysOfMon: []int{0}}) - require.ErrorContains(t, err, "day of the month") - - _, err = buildTrigger(triggerFlags{every: "weekly", onDays: []string{"Caturday"}}) - require.ErrorContains(t, err, "not a day of the week") - - _, err = buildTrigger(triggerFlags{every: "fortnightly"}) - require.ErrorContains(t, err, "hourly, daily, weekly or monthly") -} - -func TestBuildTrigger_NeedsATrigger(t *testing.T) { - _, err := buildTrigger(triggerFlags{}) - require.ErrorContains(t, err, "--cron, --every or --at-time") -} - -func TestScheduleSummary(t *testing.T) { - cron := &eval_api.Schedule{Trigger: &eval_api.ScheduleTrigger{ - Type: eval_api.TriggerCron, Expression: "0 9 * * *"}} - assert.Equal(t, "cron 0 9 * * *", cron.Summary()) - - weekly := &eval_api.Schedule{Trigger: &eval_api.ScheduleTrigger{ - Type: eval_api.TriggerRecurrence, - Schedule: &eval_api.RecurrencePattern{Type: eval_api.RecurrenceWeekly}}} - assert.Equal(t, "every Weekly", weekly.Summary()) - - once := &eval_api.Schedule{Trigger: &eval_api.ScheduleTrigger{ - Type: eval_api.TriggerOneTime, ScheduledTime: "2026-08-01T09:00:00Z"}} - assert.Equal(t, "once at 2026-08-01T09:00:00Z", once.Summary()) - - var nilSchedule *eval_api.Schedule - assert.Empty(t, nilSchedule.Summary()) - assert.Empty(t, (&eval_api.Schedule{}).Summary()) -} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/errors.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/errors.go new file mode 100644 index 00000000000..3b4b93d7c7e --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/errors.go @@ -0,0 +1,29 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package eval_api + +import ( + "errors" + "net/http" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore" +) + +// IsConflict reports whether the service refused because the resource is busy. +func IsConflict(err error) bool { + var respErr *azcore.ResponseError + if !errors.As(err, &respErr) { + return false + } + return respErr.StatusCode == http.StatusConflict +} + +// IsNotFound reports whether the service answered 404. +func IsNotFound(err error) bool { + var respErr *azcore.ResponseError + if !errors.As(err, &respErr) { + return false + } + return respErr.StatusCode == http.StatusNotFound +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/schedules.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/schedules.go deleted file mode 100644 index a6ab65746a4..00000000000 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/schedules.go +++ /dev/null @@ -1,184 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package eval_api - -import ( - "context" - "errors" - "net/http" - "net/url" - - "github.com/Azure/azure-sdk-for-go/sdk/azcore" -) - -const pathSchedules = "/schedules" - -// Trigger types accepted by the schedules API. -const ( - TriggerCron = "Cron" - TriggerRecurrence = "Recurrence" - TriggerOneTime = "OneTime" -) - -// Recurrence patterns accepted under a Recurrence trigger. -const ( - RecurrenceHourly = "Hourly" - RecurrenceDaily = "Daily" - RecurrenceWeekly = "Weekly" - RecurrenceMonthly = "Monthly" -) - -// ScheduleTaskEvaluation is the only task type the service accepts today; an -// Insight task is rejected on validation. -const ScheduleTaskEvaluation = "Evaluation" - -// RecurrencePattern is the shape of a Recurrence trigger's repeat rule. Which -// fields apply depends on Type: Daily reads Hours, Weekly reads DaysOfWeek, -// Monthly reads DaysOfMonth, and Hourly reads neither. -type RecurrencePattern struct { - Type string `json:"type"` - Hours []int `json:"hours,omitempty"` - DaysOfWeek []string `json:"daysOfWeek,omitempty"` - DaysOfMonth []int `json:"daysOfMonth,omitempty"` -} - -// ScheduleTrigger says when the task runs. The discriminator is Type; the -// other fields are per-type and only one set is ever populated. -type ScheduleTrigger struct { - Type string `json:"type"` - - // Cron - Expression string `json:"expression,omitempty"` - - // Recurrence - Schedule *RecurrencePattern `json:"schedule,omitempty"` - Interval int `json:"interval,omitempty"` - - // OneTime - ScheduledTime string `json:"scheduledTime,omitempty"` - - // Cron and Recurrence - StartTime string `json:"startTime,omitempty"` - EndTime string `json:"endTime,omitempty"` - - Timezone string `json:"timezone,omitempty"` -} - -// ScheduleTask is what the trigger fires. An evaluation task needs both the -// group and the run to repeat: the group holds only its testing criteria, so -// the target and dataset travel with the run. -type ScheduleTask struct { - Type string `json:"type"` - EvalID string `json:"evalId,omitempty"` - EvalRun *CreateOpenAIEvalRunRequest `json:"evalRun,omitempty"` -} - -// Schedule is a named, project-scoped recurring evaluation. -type Schedule struct { - ID string `json:"id,omitempty"` - DisplayName string `json:"displayName,omitempty"` - Description string `json:"description,omitempty"` - Enabled bool `json:"enabled"` - ProvisioningStatus string `json:"provisioningStatus,omitempty"` - Trigger *ScheduleTrigger `json:"trigger,omitempty"` - Task *ScheduleTask `json:"task,omitempty"` - Tags map[string]string `json:"tags,omitempty"` - Properties map[string]string `json:"properties,omitempty"` - Error *JobError `json:"error,omitempty"` -} - -// Summary renders the trigger as a single line for listings. -func (s *Schedule) Summary() string { - if s == nil || s.Trigger == nil { - return "" - } - switch s.Trigger.Type { - case TriggerCron: - return "cron " + s.Trigger.Expression - case TriggerOneTime: - return "once at " + s.Trigger.ScheduledTime - case TriggerRecurrence: - if s.Trigger.Schedule == nil { - return "recurrence" - } - return "every " + s.Trigger.Schedule.Type - } - return s.Trigger.Type -} - -// ScheduleList is the response for ListSchedules. -type ScheduleList struct { - Value []Schedule `json:"value"` -} - -// Settled reports whether the schedule has finished provisioning. -// -// A schedule that is still being created refuses a delete, and does it two -// different ways: 409 while it is busy, or 404 because the trigger behind it -// does not exist yet. Waiting for it to settle avoids both. -func (s *Schedule) Settled() bool { - if s == nil { - return true - } - switch s.ProvisioningStatus { - case "Creating", "Updating", "Deleting": - return false - } - return true -} - -// PutSchedule creates or replaces a schedule. The route is keyed by name, and -// the same call updates an existing schedule in place. -func (c *EvalClient) PutSchedule( - ctx context.Context, - name string, - schedule *Schedule, - apiVersion string, -) (*Schedule, error) { - path := pathSchedules + "/" + url.PathEscape(name) - return doRequestTyped[Schedule](c, ctx, http.MethodPut, path, nil, schedule, apiVersion) -} - -// GetSchedule reads one schedule by name. -func (c *EvalClient) GetSchedule( - ctx context.Context, - name string, - apiVersion string, -) (*Schedule, error) { - path := pathSchedules + "/" + url.PathEscape(name) - return doRequestTyped[Schedule](c, ctx, http.MethodGet, path, nil, nil, apiVersion) -} - -// ListSchedules returns the project's schedules. -func (c *EvalClient) ListSchedules(ctx context.Context, apiVersion string) (*ScheduleList, error) { - return doRequestTyped[ScheduleList](c, ctx, http.MethodGet, pathSchedules, nil, nil, apiVersion) -} - -// DeleteSchedule removes a schedule by name. -func (c *EvalClient) DeleteSchedule(ctx context.Context, name string, apiVersion string) error { - path := pathSchedules + "/" + url.PathEscape(name) - _, err := c.doRequest(ctx, http.MethodDelete, path, nil, nil, apiVersion) - return err -} - -// IsConflict reports whether the service refused because the resource is busy. -// -// A schedule that is still provisioning answers 409 to a delete, which is worth -// waiting out rather than reporting. -func IsConflict(err error) bool { - var respErr *azcore.ResponseError - if !errors.As(err, &respErr) { - return false - } - return respErr.StatusCode == http.StatusConflict -} - -// IsNotFound reports whether the service answered 404. -func IsNotFound(err error) bool { - var respErr *azcore.ResponseError - if !errors.As(err, &respErr) { - return false - } - return respErr.StatusCode == http.StatusNotFound -} From e8c1f0cc7b9f19fb4eb0b45d90c03e29125d066a Mon Sep 17 00:00:00 2001 From: mohessie Date: Sun, 2 Aug 2026 22:11:31 +0300 Subject: [PATCH 066/320] Correct two stale claims in the command reference The default metric name was listed as invented. It is not: the SDK's own code-based evaluator sample declares `result`, so the name matches. What was left after that read like an open question about the type and bounds, and that was a misreading. A metric describes what the author's grade() returns, so its type and range are theirs to state - continuous, ordinal or boolean over any range - which is what --metrics is for. The sample's ordinal 0.0-1.0 described that sample's grader, not a convention. Probing shows the service does not validate scores against declared bounds: a grader returning 14 and 33 scores identically whether the metric declares continuous-unbounded or ordinal 0.0-1.0. So a wrong bound is not an error, only misleading range metadata - which is the argument for defaulting to unbounded, since it asserts nothing about a range we cannot know. Also correct the azd up entry. It claimed a configuration gap without saying what it costs. Measured: an evaluator published the way azd up publishes one, carrying no data_schema, scores exactly as a fully configured one does - completed, 2 of 2 passed, both. The real cost is narrower and worth stating precisely: image_tag, metrics and init_parameters cannot be declared, so an evaluator that needs a dependency or a non-default metric range can only be published through evaluator create. --- .../extensions/azure.ai.evaluations/COMMANDS.md | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/cli/azd/extensions/azure.ai.evaluations/COMMANDS.md b/cli/azd/extensions/azure.ai.evaluations/COMMANDS.md index 5f58e75f030..7d731281c14 100644 --- a/cli/azd/extensions/azure.ai.evaluations/COMMANDS.md +++ b/cli/azd/extensions/azure.ai.evaluations/COMMANDS.md @@ -189,5 +189,16 @@ What the live tests on that branch establish: | `generate` has no live coverage | The most complex composite command | | `TestLiveRun` skips | Agent-target runs, `--from-traces`, `--response-id` all unverified | | `--image-tag` never used with a real image | The only supported way to give a code evaluator dependencies | -| `azd up` cannot configure a code evaluator | Reconciler passes empty options; no config fields for `data_schema`, `metrics`, `init_params`, `image_tag` | -| Default metric name `result` is invented | Real evaluators use semantic names (`groundedness`, `relevance`) | +| `azd up` cannot configure a code evaluator | Not a correctness break — a schema-less evaluator scores identically to a configured one. But `image_tag`, `metrics` and `init_parameters` cannot be declared, so an evaluator needing a dependency or a non-default metric range can only be published with `evaluator create` | + +### Not a gap: the default metric + +`evaluator create` defaults to `{"result": {"type": "continuous", "desirable_direction": "increase"}}` +when `--metrics` is omitted. The name matches the SDK's code-based evaluator sample. + +The type and bounds are deliberately unbounded. A metric describes what the author's +own `grade()` returns, so it can be continuous, ordinal or boolean over any range — +that is what `--metrics` is for. The service does not validate scores against declared +bounds (a grader returning 14 and 33 scores fine under a declared `ordinal 0.0–1.0`), +so a wrong bound is not an error, only bad range metadata. Defaulting to unbounded +asserts nothing, which is the correct claim to make about a range we cannot know. From b0879e4d4bc847061b617248058016b948188a48 Mon Sep 17 00:00:00 2001 From: mohessie Date: Sun, 2 Aug 2026 22:33:45 +0300 Subject: [PATCH 067/320] Drive the CLI as a subprocess, and fix what that found Every live test so far called the client layer directly. That proves the API paths work and says nothing about the surface on top of them: flag parsing, mutual exclusion, exit codes, the rendered tables, and whether -o json emits something a script can consume. Those are the parts a user touches. tests/cli builds the extension and runs it as a process, the way a user does. Eight tests cover listing built-ins in both renderings, the bare array envelope, the code evaluator lifecycle end to end, source mutual exclusion, local grader validation, the not-found message, and --no-prompt failing instead of hanging. Running them found three things. `evaluator show` on a name that does not exist printed 1327 characters of service JSON. The same was true of `evaluator delete`, `dataset show` and `dataset delete`. Those four now answer briefly and say where to look instead, which run list already did. `init` created evals/datasets and evals/evaluators before asking azd whether there was a project to attach to, so refusing left a half-scaffolded tree behind. The project is the one thing init cannot supply for itself, so it is now confirmed before anything is written. `init` cannot be exercised by this harness at all: it resolves the project over azd's gRPC channel, so it needs azd hosting the extension. What the harness can assert is the refusal a user gets outside a project, and that it names `azd init` rather than surfacing a transport error. The scaffolding itself stays covered by the unit tests, which can supply a fake client. The harness retries once on "AzureDeveloperCLICredential: exit status 1". Each invocation is a fresh process shelling out to azd for a token, and azd intermittently fails under rapid sequential calls. The retry is logged rather than silent, and a persistent failure still fails the test with a message pointing at azd auth login. --- .../internal/cmd/dataset.go | 10 + .../internal/cmd/evaluator.go | 9 + .../azure.ai.evaluations/internal/cmd/init.go | 26 +++ .../tests/cli/evaluator_test.go | 187 +++++++++++++++ .../tests/cli/harness_test.go | 216 ++++++++++++++++++ 5 files changed, 448 insertions(+) create mode 100644 cli/azd/extensions/azure.ai.evaluations/tests/cli/evaluator_test.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/tests/cli/harness_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset.go index fb5b5865e79..9ea7c5b3a48 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset.go @@ -11,6 +11,7 @@ import ( "strings" "azureaieval/internal/pkg/dataset_api" + "azureaieval/internal/pkg/eval_api" "github.com/spf13/cobra" ) @@ -192,6 +193,11 @@ func newDatasetShowCommand() *cobra.Command { ds, err := ec.datasetClient.GetDataset(ctx, name, version, ProjectEndpointAPIVersion) if err != nil { + if eval_api.IsNotFound(err) { + return fmt.Errorf( + "no dataset %q at version %q in this project; "+ + "`azd ai eval dataset list` shows the ones there are", name, version) + } return fmt.Errorf("reading dataset %q version %q: %w", name, version, err) } @@ -239,6 +245,10 @@ func newDatasetDeleteCommand() *cobra.Command { if err := ec.datasetClient.DeleteDatasetVersion( ctx, name, version, ProjectEndpointAPIVersion, ); err != nil { + if eval_api.IsNotFound(err) { + return fmt.Errorf( + "no dataset %q at version %q in this project", name, version) + } return fmt.Errorf("deleting dataset %q version %q: %w", name, version, err) } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator.go index ee601bb16de..151bc14bf42 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator.go @@ -420,6 +420,11 @@ func newEvaluatorShowCommand() *cobra.Command { raw, err := ec.evalClient.GetEvaluatorRaw(ctx, name, version, ProjectEndpointAPIVersion) if err != nil { + if eval_api.IsNotFound(err) { + return fmt.Errorf( + "no evaluator %q in this project; "+ + "`azd ai eval evaluator list` shows the ones there are", name) + } return fmt.Errorf("reading evaluator %q: %w", name, err) } @@ -466,6 +471,10 @@ func newEvaluatorDeleteCommand() *cobra.Command { if err := ec.evalClient.DeleteEvaluatorVersion( ctx, name, version, ProjectEndpointAPIVersion, ); err != nil { + if eval_api.IsNotFound(err) { + return fmt.Errorf( + "no evaluator %q at version %q in this project", name, version) + } return fmt.Errorf("deleting evaluator %q version %q: %w", name, version, err) } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go index 85359d184ae..fba0d533682 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go @@ -53,6 +53,13 @@ func newInitCommand() *cobra.Command { } } + // Asked before anything is written: the project is the one thing init + // cannot supply for itself, and failing after creating directories + // leaves a half-scaffolded tree behind for the user to clean up. + if err := confirmAzdProject(cmd.Context()); err != nil { + return err + } + if err := os.MkdirAll(filepath.Join(outDir, project.DefaultDatasetsDir), 0o750); err != nil { return fmt.Errorf("creating the datasets directory: %w", err) } @@ -134,6 +141,25 @@ const ( wiringPresent = "present" // an eval service was already declared ) +// noAzdProject is what init reports when there is nothing to attach to. +const noAzdProject = "no azd project found in this directory. Run `azd init` first, " + + "or run this from the root of an existing one; the eval service is " + +// confirmAzdProject reports whether a project exists, without changing it. +func confirmAzdProject(ctx context.Context) error { + azdClient, err := azdext.NewAzdClient() + if err != nil { + return fmt.Errorf("%sadded to its azure.yaml", noAzdProject) + } + defer azdClient.Close() + + resp, err := azdClient.Project().Get(ctx, &azdext.EmptyRequest{}) + if err != nil || resp.GetProject() == nil { + return fmt.Errorf("%sadded to its azure.yaml", noAzdProject) + } + return nil +} + // ensureRootEvalService declares the eval service in azd's project file. // // azd acts on nothing until the service exists, so the reference is made rather diff --git a/cli/azd/extensions/azure.ai.evaluations/tests/cli/evaluator_test.go b/cli/azd/extensions/azure.ai.evaluations/tests/cli/evaluator_test.go new file mode 100644 index 00000000000..79eabdd69eb --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/tests/cli/evaluator_test.go @@ -0,0 +1,187 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +//go:build live + +package cli + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +// TestCLIEvaluatorListBuiltin is the cheapest proof the binary can reach the +// service on its own: no azd project, no config, just a flag. +func TestCLIEvaluatorListBuiltin(t *testing.T) { + r := requireSuccess(t, run(t, "evaluator", "list", "--builtin", "-o", "json")) + + var builtins []struct { + Name string `json:"name"` + EvaluatorType string `json:"evaluator_type"` + } + r.JSON(t, &builtins) + require.NotEmpty(t, builtins, "the project must expose built-in evaluators") + + for _, b := range builtins { + require.True(t, strings.HasPrefix(b.Name, "builtin."), + "--builtin must return only built-ins, got %q", b.Name) + } + + // The default rendering is a table, not JSON. A script reading stdout + // without -o json would otherwise silently parse a header row. + table := requireSuccess(t, run(t, "evaluator", "list", "--builtin")) + require.Contains(t, table.Stdout, "NAME") + require.Contains(t, table.Stdout, "VERSION") +} + +// TestCLIJSONListsAreBareArrays pins the envelope. +// +// The service wraps listings in {"value":[...]} or {"data":[...]} depending on +// the route. Leaking either would make every consumer special-case the +// command it came from, so the CLI unwraps them, and this is what says so. +func TestCLIJSONListsAreBareArrays(t *testing.T) { + for _, args := range [][]string{ + {"evaluator", "list", "--builtin", "-o", "json"}, + {"dataset", "list", "-o", "json"}, + } { + t.Run(strings.Join(args[:2], " "), func(t *testing.T) { + r := requireSuccess(t, run(t, args...)) + trimmed := strings.TrimSpace(r.Stdout) + require.True(t, strings.HasPrefix(trimmed, "["), + "a list must be a bare array, not an envelope; got:\n%s", firstLine(trimmed)) + + var out []any + r.JSON(t, &out) + }) + } +} + +// TestCLICodeEvaluatorRoundTrip drives the whole custom evaluator lifecycle +// through the command surface: create from a script, read it back, list its +// versions, then delete it. +func TestCLICodeEvaluatorRoundTrip(t *testing.T) { + name := uniqueName("azdcli_code") + script := writeGrader(t, lengthGrader) + + requireSuccess(t, run(t, "evaluator", "create", "--name", name, "--file", script)) + t.Cleanup(func() { + run(t, "evaluator", "delete", "--name", name, "--version", "1") + }) + + shown := requireSuccess(t, run(t, "evaluator", "show", "--name", name, "-o", "json")) + var def struct { + Name string `json:"name"` + Definition struct { + Type string `json:"type"` + CodeText string `json:"code_text"` + } `json:"definition"` + } + shown.JSON(t, &def) + require.Equal(t, "code", def.Definition.Type, + "a script must register as a code definition") + require.Contains(t, def.Definition.CodeText, "def grade", + "the script's source must round-trip in code_text") + + listed := requireSuccess(t, run(t, "evaluator", "list", "--name", name, "-o", "json")) + var versions []struct { + Name string `json:"name"` + Version string `json:"version"` + } + listed.JSON(t, &versions) + require.NotEmpty(t, versions, "the evaluator must list its own versions") +} + +// TestCLIEvaluatorSourcesAreMutuallyExclusive covers the validation a user is +// most likely to trip, and asserts it costs nothing to find out — no version +// is published on the way to the error. +func TestCLIEvaluatorSourcesAreMutuallyExclusive(t *testing.T) { + script := writeGrader(t, lengthGrader) + rubric := filepath.Join(t.TempDir(), "rubric.json") + require.NoError(t, os.WriteFile(rubric, + []byte(`{"dimensions":[{"id":"tone","description":"polite","weight":5}]}`), 0o600)) + + both := requireFailure(t, run(t, "evaluator", "create", + "--name", uniqueName("azdcli_both"), "--file", script, "--rubric", rubric)) + require.Contains(t, strings.ToLower(both.Combined()), "rubric", + "the error must name the flags in conflict") + + neither := requireFailure(t, run(t, "evaluator", "create", + "--name", uniqueName("azdcli_neither"))) + require.NotEmpty(t, strings.TrimSpace(neither.Combined()), + "refusing without a source must explain itself") +} + +// TestCLIGraderIsValidatedBeforePublishing asserts the check that saves a user +// from a late failure: a script with no top-level grade() is refused locally, +// because the executor would otherwise accept the publish and fail the run. +func TestCLIGraderIsValidatedBeforePublishing(t *testing.T) { + script := writeGrader(t, `class AnswerLengthEvaluator: + def __call__(self, **kwargs): + return {"result": 1.0} +`) + + r := requireFailure(t, run(t, "evaluator", "create", + "--name", uniqueName("azdcli_noglade"), "--file", script)) + require.Contains(t, strings.ToLower(r.Combined()), "grade", + "the refusal must name the function the executor looks for") +} + +// TestCLIUnknownEvaluatorIsBrief covers the failure a user hits by typo. +// +// The service answers with a long JSON body. Printing it verbatim buries the +// one useful sentence, so the CLI shortens it, and a regression here is the +// kind that only shows up in someone's terminal. +func TestCLIUnknownEvaluatorIsBrief(t *testing.T) { + r := requireFailure(t, run(t, "evaluator", "show", "--name", "azdcli-does-not-exist-9999")) + require.Less(t, len(r.Combined()), 600, + "a not-found must stay short, not dump the service body:\n%s", r.Combined()) +} + +// TestCLIInitNeedsAnAzdProject covers the whole of what `init` can be asked +// through this harness. +// +// init resolves the project over azd's gRPC channel, so it only works when azd +// is hosting the extension. Running the binary directly there is no host, and +// that is exactly the case a user hits by running the command outside a +// project — so what is asserted is the refusal: it must name `azd init` rather +// than surface a transport error. The scaffolding itself is covered by the +// unit tests, which can supply a fake azd client. +func TestCLIInitNeedsAnAzdProject(t *testing.T) { + dir := t.TempDir() + + r := requireFailure(t, runIn(t, dir, "init", + "--target", "probe-agent", + "--judge-model", "gpt-4o-mini", + "--no-prompt")) + + require.Contains(t, r.Combined(), "azd init", + "the refusal must name the command that makes a project") + require.NotContains(t, strings.ToLower(r.Combined()), "grpc", + "a missing project must not surface as a transport error") + + entries, err := os.ReadDir(dir) + require.NoError(t, err) + require.Empty(t, entries, + "a refused init must leave nothing behind") +} + +// TestCLINoPromptFailsInsteadOfHanging is what makes the CLI usable in CI: a +// missing required value must end the process, not wait on a terminal nobody +// is watching. +func TestCLINoPromptFailsInsteadOfHanging(t *testing.T) { + dir := t.TempDir() + r := requireFailure(t, runIn(t, dir, "init", "--no-prompt")) + require.NotEmpty(t, strings.TrimSpace(r.Combined()), + "--no-prompt must say what it could not resolve") +} + +func firstLine(s string) string { + if i := strings.IndexByte(s, '\n'); i >= 0 { + return s[:i] + } + return s +} diff --git a/cli/azd/extensions/azure.ai.evaluations/tests/cli/harness_test.go b/cli/azd/extensions/azure.ai.evaluations/tests/cli/harness_test.go new file mode 100644 index 00000000000..e4e86181a1d --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/tests/cli/harness_test.go @@ -0,0 +1,216 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +//go:build live + +// Package cli drives the built binary as a subprocess. +// +// The other live tests call the client layer directly, which proves the API +// paths work but says nothing about the command surface on top of them: flag +// parsing, mutual exclusion, prompting, --no-prompt, exit codes, the rendered +// tables, and whether -o json emits what a script can actually consume. Those +// are the parts a user touches, and until now nothing exercised them against a +// real service. +// +// go test -tags live -v ./tests/cli/... +// +// Required: +// +// AZURE_AI_EVAL_E2E_LIVE=1 +// FOUNDRY_PROJECT_ENDPOINT=https://.services.ai.azure.com/api/projects/ +package cli + +import ( + "encoding/json" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +var ( + binaryPath string + endpoint string +) + +// TestMain builds the extension once so every test runs the same binary a user +// would, rather than an in-process command tree that skips main's wiring. +func TestMain(m *testing.M) { + if os.Getenv("AZURE_AI_EVAL_E2E_LIVE") != "1" { + fmt.Fprintln(os.Stderr, "set AZURE_AI_EVAL_E2E_LIVE=1 to run the CLI tests") + os.Exit(0) + } + + endpoint = strings.TrimSuffix(os.Getenv("FOUNDRY_PROJECT_ENDPOINT"), "/") + if endpoint == "" { + fmt.Fprintln(os.Stderr, "FOUNDRY_PROJECT_ENDPOINT is required") + os.Exit(1) + } + + dir, err := os.MkdirTemp("", "azdeval-cli") + if err != nil { + fmt.Fprintf(os.Stderr, "creating a temp dir: %v\n", err) + os.Exit(1) + } + defer os.RemoveAll(dir) + + binaryPath = filepath.Join(dir, "azdeval"+exeSuffix()) + build := exec.Command("go", "build", "-o", binaryPath, ".") + build.Dir = "../.." + if out, err := build.CombinedOutput(); err != nil { + fmt.Fprintf(os.Stderr, "building the extension: %v\n%s\n", err, out) + os.Exit(1) + } + + code := m.Run() + os.RemoveAll(dir) + os.Exit(code) +} + +func exeSuffix() string { + if os.PathSeparator == '\\' { + return ".exe" + } + return "" +} + +// result is one invocation of the binary. +type result struct { + Args []string + Stdout string + Stderr string + ExitCode int +} + +// Combined is stdout and stderr together, for assertions that do not care +// which stream carried the message. +func (r result) Combined() string { return r.Stdout + r.Stderr } + +// JSON decodes stdout, failing the test when the command did not emit +// something a script could consume. +func (r result) JSON(t *testing.T, into any) { + t.Helper() + require.NoError(t, json.Unmarshal([]byte(r.Stdout), into), + "-o json must emit parseable JSON on stdout; got:\n%s", r.Stdout) +} + +// run invokes the binary with the project endpoint already supplied. +func run(t *testing.T, args ...string) result { + t.Helper() + return runIn(t, "", args...) +} + +// credentialFlake is azd's token helper failing under rapid sequential calls. +// +// Every invocation here is a fresh process, so each one shells out to azd for +// a token, and azd intermittently exits non-zero doing it. Retrying is safe +// because no request was made, and the alternative is a suite that fails on a +// different test each run for a reason that has nothing to do with the code. +const credentialFlake = "AzureDeveloperCLICredential: exit status 1" + +// runIn invokes the binary with a working directory, for commands that write +// files. +func runIn(t *testing.T, dir string, args ...string) result { + t.Helper() + + res := invoke(t, dir, args...) + for attempt := 0; attempt < 2 && strings.Contains(res.Combined(), credentialFlake); attempt++ { + t.Logf("azd credential flaked; retrying `%s`", strings.Join(args, " ")) + time.Sleep(2 * time.Second) + res = invoke(t, dir, args...) + } + require.NotContains(t, res.Combined(), credentialFlake, + "azd could not produce a token after retries; run `azd auth login` and try again") + return res +} + +func invoke(t *testing.T, dir string, args ...string) result { + t.Helper() + + full := append([]string{}, args...) + if !hasFlag(args, "--project-endpoint") && needsEndpoint(args) { + full = append(full, "--project-endpoint", endpoint) + } + + cmd := exec.Command(binaryPath, full...) + if dir != "" { + cmd.Dir = dir + } + var stdout, stderr strings.Builder + cmd.Stdout = &stdout + cmd.Stderr = &stderr + + err := cmd.Run() + code := 0 + if exitErr, ok := err.(*exec.ExitError); ok { + code = exitErr.ExitCode() + } else if err != nil { + t.Fatalf("could not run %v: %v", full, err) + } + + res := result{Args: full, Stdout: stdout.String(), Stderr: stderr.String(), ExitCode: code} + t.Logf("$ azd ai eval %s -> exit %d", strings.Join(args, " "), res.ExitCode) + return res +} + +func hasFlag(args []string, flag string) bool { + for _, a := range args { + if a == flag { + return true + } + } + return false +} + +// needsEndpoint keeps --project-endpoint off the commands that reject it. +func needsEndpoint(args []string) bool { + for _, a := range args { + switch a { + case "init", "--help", "-h": + return false + } + } + return true +} + +// requireSuccess fails with the command's own output, which is what a user +// would have seen. +func requireSuccess(t *testing.T, r result) result { + t.Helper() + require.Equalf(t, 0, r.ExitCode, + "expected `%s` to succeed\nstdout:\n%s\nstderr:\n%s", + strings.Join(r.Args, " "), r.Stdout, r.Stderr) + return r +} + +// requireFailure asserts a non-zero exit, so a command that silently succeeds +// where it should refuse is caught. +func requireFailure(t *testing.T, r result) result { + t.Helper() + require.NotEqualf(t, 0, r.ExitCode, + "expected `%s` to fail\nstdout:\n%s\nstderr:\n%s", + strings.Join(r.Args, " "), r.Stdout, r.Stderr) + return r +} + +func uniqueName(prefix string) string { + return fmt.Sprintf("%s_%d", prefix, time.Now().UnixNano()) +} + +// writeGrader lays down a code evaluator script and returns its path. +func writeGrader(t *testing.T, body string) string { + t.Helper() + dir := t.TempDir() + path := filepath.Join(dir, "grader.py") + require.NoError(t, os.WriteFile(path, []byte(body), 0o600)) + return path +} + +const lengthGrader = `def grade(sample, item) -> float: + return float(len((item or {}).get("response", ""))) +` From 45a1cd1480ad2315448b984a66d4b7f1961de77a Mon Sep 17 00:00:00 2001 From: mohessie Date: Sun, 2 Aug 2026 22:36:24 +0300 Subject: [PATCH 068/320] Record the azd up gap for code evaluators as a TODO A code evaluator deployed by `azd up` cannot be configured: the flags exist on `evaluator create` but the config has nowhere to put them, so the reconciler passes an empty options struct. Scoring is unaffected - an evaluator published with no schema scores identically to a configured one - so this is a capability gap rather than a defect. image_tag is the one that bites: an evaluator needing any dependency cannot be deployed declaratively at all, only published by hand. metrics and init_parameters are the same story for anything that is not a 0-to-1 increasing score. The note sits in both places the fix touches: the call site that passes nothing, and the config type that has nowhere to read it from. --- .../azure.ai.evaluations/internal/cmd/reconciler.go | 8 ++++++++ .../azure.ai.evaluations/internal/project/eval_config.go | 4 ++++ 2 files changed, 12 insertions(+) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go index 687d1936baf..c89c89c6ada 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go @@ -324,6 +324,14 @@ func (r *evalReconciler) ensureCodeEvaluator( return recordedVersion, false, nil } + // TODO: a code evaluator deployed by `azd up` cannot be configured. The + // flags exist on `evaluator create` but the config has nowhere to put them, + // so this passes none. Scoring still works, but image_tag is the one that + // bites: an evaluator needing any dependency cannot be deployed this way at + // all, only published by hand. metrics and init_parameters are the same + // story for anything that is not a 0-to-1 increasing score. The fix is + // fields on project.EvaluatorDecl - image_tag, metrics, data_schema, + // init_parameters - read here instead of this empty struct. opts, err := codeEvaluatorOptions(codeEvaluatorFlags{}) if err != nil { return "", false, err diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go index 336c9b17bdf..93fd22b7bde 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go @@ -39,6 +39,10 @@ type DatasetDecl struct { // `.json` file holds a rubric. A code evaluator cannot name a folder — it runs // as a python grader, which is handed one script's source and cannot import a // helper module beside it. +// TODO: carry the code evaluator settings `evaluator create` accepts - +// image_tag, metrics, data_schema, init_parameters. Without them a code +// evaluator can be declared but not configured, so one needing a dependency +// has to be published by hand. See EnsureEvaluator in internal/cmd/reconciler.go. type EvaluatorDecl struct { Name string `yaml:"name" json:"name"` Source string `yaml:"source,omitempty" json:"source,omitempty"` From 488317bcb6844275ccb104d52dccab61a8cfe617 Mon Sep 17 00:00:00 2001 From: mohessie Date: Sun, 2 Aug 2026 23:12:39 +0300 Subject: [PATCH 069/320] Let azd up configure a code evaluator, and stop TestLiveRun skipping Three changes that turned out to be related, because two of them were hidden by tests that were not running what they claimed. EvaluatorDecl gains image_tag, metrics, data_schema and init_parameters, so a code evaluator can be configured where it is declared rather than only by hand. The three schema fields name JSON files beside the script and resolve like source, because they are edited as files. No service change was needed: the API already accepted all four, only the config had nowhere to put them. Change detection had to move with it. The digest hashed the script alone, so editing metrics or the image tag would have left the edit undeployed with azd up reporting nothing to do. codeEvaluatorDigest now covers the script, the tag, and each settings file, with a unit test for each way it can change. TestLiveRun no longer skips. It defaults to an agent discovered from the project and fails when there is none, because skipping by default is how the agent-target path stayed unverified while the suite reported green. Running it immediately showed why that mattered: discovery first read /assistants, the run failed with "resources not found", and the reason is that an eval target resolves against /agents. This project has eight assistants and no agents, so the path genuinely cannot be verified here - which the suite now says out loud instead of passing. Finally, the credential. Both live packages built a new AzureDeveloperCLICredential per test, and azidentity caches tokens per instance, so every test shelled out to azd again. A refresh that overruns the SDK's ten-second budget for that subprocess surfaces as "AzureDeveloperCLICredential: exit status 1" - which reads like a broken login rather than a timeout, and lands on whichever test happened to follow a slow one. Both packages now share one credential. Fifteen rapid azd token calls succeed on their own, so the trigger was the repeated cold cache, not call frequency. --- .../internal/cmd/build_live_test.go | 24 ++++- .../internal/cmd/reconciler.go | 54 +++++++++-- .../internal/cmd/reconciler_digest_test.go | 81 ++++++++++++++++ .../internal/project/eval_config.go | 12 ++- .../internal/project/service_target_eval.go | 3 + .../tests/live/live_test.go | 97 +++++++++++++++++-- 6 files changed, 245 insertions(+), 26 deletions(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler_digest_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build_live_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build_live_test.go index b7c9c32500c..1018e9cb4e4 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build_live_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build_live_test.go @@ -19,6 +19,7 @@ import ( "fmt" "os" "strings" + "sync" "testing" "time" @@ -30,6 +31,26 @@ import ( "github.com/stretchr/testify/require" ) +// One credential for the whole package, because azidentity caches tokens per +// instance. Building one per test made every test shell out to azd again, and +// a refresh that overruns the SDK's ten-second budget for that subprocess +// surfaces as "AzureDeveloperCLICredential: exit status 1" — which reads like +// a broken login rather than a timeout, and lands on whichever test happened +// to run after a slow one. +var ( + sharedCredOnce sync.Once + sharedCred *azidentity.AzureDeveloperCLICredential + sharedCredErr error +) + +func liveCredential() (*azidentity.AzureDeveloperCLICredential, error) { + sharedCredOnce.Do(func() { + sharedCred, sharedCredErr = azidentity.NewAzureDeveloperCLICredential( + &azidentity.AzureDeveloperCLICredentialOptions{}) + }) + return sharedCred, sharedCredErr +} + func liveEvalClient(t *testing.T) (*eval_api.EvalClient, string) { t.Helper() if os.Getenv("AZURE_AI_EVAL_E2E_LIVE") != "1" { @@ -39,8 +60,7 @@ func liveEvalClient(t *testing.T) (*eval_api.EvalClient, string) { if endpoint == "" { t.Fatal("FOUNDRY_PROJECT_ENDPOINT is required") } - cred, err := azidentity.NewAzureDeveloperCLICredential( - &azidentity.AzureDeveloperCLICredentialOptions{}) + cred, err := liveCredential() require.NoError(t, err) judge := os.Getenv("AZURE_AI_EVAL_MODEL") diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go index c89c89c6ada..8a1e5eb8138 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go @@ -6,6 +6,8 @@ package cmd import ( "bufio" "context" + "crypto/sha256" + "encoding/hex" "encoding/json" "fmt" "os" @@ -287,6 +289,41 @@ func (r *evalReconciler) EnsureEvaluator( return created.Version, true, nil } +// codeEvaluatorDigest fingerprints everything a published version depends on. +// +// The script alone is not enough: changing only metrics or the image tag +// changes what gets published, and hashing just the source would leave that +// edit undeployed with `azd up` reporting no change. +func codeEvaluatorDigest(decl project.EvaluatorDecl, path string) (string, error) { + sum := sha256.New() + + script, err := project.Fingerprint(path) + if err != nil { + return "", err + } + fmt.Fprintf(sum, "script:%s\nimage:%s\n", script, decl.ImageTag) + + for _, settings := range []struct { + label string + path string + }{ + {"metrics", decl.Metrics}, + {"data_schema", decl.DataSchema}, + {"init_parameters", decl.InitParameters}, + } { + if settings.path == "" { + continue + } + digest, err := project.Fingerprint(settings.path) + if err != nil { + return "", fmt.Errorf("evaluator %q %s: %w", decl.Name, settings.label, err) + } + fmt.Fprintf(sum, "%s:%s\n", settings.label, digest) + } + + return hex.EncodeToString(sum.Sum(nil)), nil +} + // ensureCodeEvaluator publishes a Python script only when its content changed // since the last deploy. // @@ -306,7 +343,7 @@ func (r *evalReconciler) ensureCodeEvaluator( return "", false, err } - digest, err := project.Fingerprint(path) + digest, err := codeEvaluatorDigest(decl, path) if err != nil { return "", false, err } @@ -324,15 +361,12 @@ func (r *evalReconciler) ensureCodeEvaluator( return recordedVersion, false, nil } - // TODO: a code evaluator deployed by `azd up` cannot be configured. The - // flags exist on `evaluator create` but the config has nowhere to put them, - // so this passes none. Scoring still works, but image_tag is the one that - // bites: an evaluator needing any dependency cannot be deployed this way at - // all, only published by hand. metrics and init_parameters are the same - // story for anything that is not a 0-to-1 increasing score. The fix is - // fields on project.EvaluatorDecl - image_tag, metrics, data_schema, - // init_parameters - read here instead of this empty struct. - opts, err := codeEvaluatorOptions(codeEvaluatorFlags{}) + opts, err := codeEvaluatorOptions(codeEvaluatorFlags{ + imageTag: decl.ImageTag, + metrics: decl.Metrics, + dataSchema: decl.DataSchema, + initParams: decl.InitParameters, + }) if err != nil { return "", false, err } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler_digest_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler_digest_test.go new file mode 100644 index 00000000000..5fd6d7521c7 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler_digest_test.go @@ -0,0 +1,81 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "os" + "path/filepath" + "testing" + + "azureaieval/internal/project" + + "github.com/stretchr/testify/require" +) + +// A code evaluator's published version depends on more than its script, so the +// digest that decides whether to republish has to cover the rest of it. +// Hashing only the source would leave a changed metric or image tag sitting in +// the config while `azd up` reported nothing to do. +func TestCodeEvaluatorDigest(t *testing.T) { + dir := t.TempDir() + + script := filepath.Join(dir, "grader.py") + require.NoError(t, os.WriteFile(script, + []byte("def grade(sample, item) -> float:\n return 1.0\n"), 0o600)) + + metrics := filepath.Join(dir, "metrics.json") + require.NoError(t, os.WriteFile(metrics, + []byte(`{"result":{"type":"continuous"}}`), 0o600)) + + base := project.EvaluatorDecl{Name: "answer_length"} + baseline, err := codeEvaluatorDigest(base, script) + require.NoError(t, err) + + t.Run("stable across calls", func(t *testing.T) { + again, err := codeEvaluatorDigest(base, script) + require.NoError(t, err) + require.Equal(t, baseline, again, + "an unchanged evaluator must not republish on every deploy") + }) + + t.Run("notices the script", func(t *testing.T) { + require.NoError(t, os.WriteFile(script, + []byte("def grade(sample, item) -> float:\n return 2.0\n"), 0o600)) + changed, err := codeEvaluatorDigest(base, script) + require.NoError(t, err) + require.NotEqual(t, baseline, changed) + }) + + t.Run("notices the image tag", func(t *testing.T) { + withImage := base + withImage.ImageTag = "python:3.12-slim" + changed, err := codeEvaluatorDigest(withImage, script) + require.NoError(t, err) + require.NotEqual(t, baseline, changed, + "changing the image changes what runs, so it must republish") + }) + + t.Run("notices the metrics file", func(t *testing.T) { + withMetrics := base + withMetrics.Metrics = metrics + before, err := codeEvaluatorDigest(withMetrics, script) + require.NoError(t, err) + + require.NoError(t, os.WriteFile(metrics, + []byte(`{"result":{"type":"ordinal","min_value":0,"max_value":1}}`), 0o600)) + after, err := codeEvaluatorDigest(withMetrics, script) + require.NoError(t, err) + require.NotEqual(t, before, after, + "editing metrics alone must republish, or the edit never deploys") + }) + + t.Run("reports a missing settings file", func(t *testing.T) { + missing := base + missing.DataSchema = filepath.Join(dir, "nope.json") + _, err := codeEvaluatorDigest(missing, script) + require.Error(t, err) + require.Contains(t, err.Error(), "data_schema", + "the error must name which setting could not be read") + }) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go index 93fd22b7bde..76b0d4bf7cb 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go @@ -39,14 +39,18 @@ type DatasetDecl struct { // `.json` file holds a rubric. A code evaluator cannot name a folder — it runs // as a python grader, which is handed one script's source and cannot import a // helper module beside it. -// TODO: carry the code evaluator settings `evaluator create` accepts - -// image_tag, metrics, data_schema, init_parameters. Without them a code -// evaluator can be declared but not configured, so one needing a dependency -// has to be published by hand. See EnsureEvaluator in internal/cmd/reconciler.go. type EvaluatorDecl struct { Name string `yaml:"name" json:"name"` Source string `yaml:"source,omitempty" json:"source,omitempty"` Version string `yaml:"version,omitempty" json:"version,omitempty"` + + // Code evaluators only. The three schema fields name JSON files beside the + // script, resolved like Source, because they are edited as files rather + // than written inline in YAML. + ImageTag string `yaml:"image_tag,omitempty" json:"image_tag,omitempty"` + Metrics string `yaml:"metrics,omitempty" json:"metrics,omitempty"` + DataSchema string `yaml:"data_schema,omitempty" json:"data_schema,omitempty"` + InitParameters string `yaml:"init_parameters,omitempty" json:"init_parameters,omitempty"` } // Eval is a run definition: evaluators plus options, bound to a dataset. diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go index f4902f236a3..5f6786e954e 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go @@ -162,6 +162,9 @@ func (p *EvalServiceTargetProvider) Deploy( for _, decl := range cfg.Evaluators { report(progress, fmt.Sprintf("Reconciling evaluator %s", decl.Name)) localPath := resolveSource(baseDir, decl.Source) + decl.Metrics = resolveSource(baseDir, decl.Metrics) + decl.DataSchema = resolveSource(baseDir, decl.DataSchema) + decl.InitParameters = resolveSource(baseDir, decl.InitParameters) version, changed, err := reconciler.EnsureEvaluator(ctx, decl, localPath) if err != nil { return nil, fmt.Errorf("evaluator %q: %w", decl.Name, err) diff --git a/cli/azd/extensions/azure.ai.evaluations/tests/live/live_test.go b/cli/azd/extensions/azure.ai.evaluations/tests/live/live_test.go index 4b3c8765728..32dda623b17 100644 --- a/cli/azd/extensions/azure.ai.evaluations/tests/live/live_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/tests/live/live_test.go @@ -24,15 +24,18 @@ import ( "context" "encoding/json" "fmt" + "net/http" "os" "path/filepath" "strings" + "sync" "testing" "time" "azureaieval/internal/pkg/dataset_api" "azureaieval/internal/pkg/eval_api" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" "github.com/Azure/azure-sdk-for-go/sdk/azidentity" "github.com/stretchr/testify/require" ) @@ -54,6 +57,29 @@ type liveEnv struct { datasetClient *dataset_api.DatasetClient } +// One credential for the whole package, because azidentity caches tokens per +// instance. Building one per test made every test shell out to azd again, and +// a refresh that overruns the SDK's ten-second budget for that subprocess +// surfaces as "AzureDeveloperCLICredential: exit status 1" — which reads like +// a broken login rather than a timeout, and lands on whichever test happened +// to run after a slow one. +var ( + sharedCredOnce sync.Once + sharedCred *azidentity.AzureDeveloperCLICredential + sharedCredErr error +) + +func liveCredential() (*azidentity.AzureDeveloperCLICredential, error) { + sharedCredOnce.Do(func() { + // Works non-interactively when azd already holds a refresh token, + // which is what makes an unattended run possible. + sharedCred, sharedCredErr = azidentity.NewAzureDeveloperCLICredential( + &azidentity.AzureDeveloperCLICredentialOptions{}, + ) + }) + return sharedCred, sharedCredErr +} + func setup(t *testing.T) *liveEnv { t.Helper() @@ -65,11 +91,7 @@ func setup(t *testing.T) *liveEnv { t.Fatal("FOUNDRY_PROJECT_ENDPOINT is required") } - // The azd developer CLI credential works non-interactively when azd already - // holds a refresh token, which is what makes an unattended run possible. - cred, err := azidentity.NewAzureDeveloperCLICredential( - &azidentity.AzureDeveloperCLICredentialOptions{}, - ) + cred, err := liveCredential() require.NoError(t, err, "acquiring an azd credential") judge := os.Getenv("AZURE_AI_EVAL_MODEL") @@ -243,12 +265,67 @@ func TestLiveEvalLifecycle(t *testing.T) { require.Equal(t, group.ID, fetched.ID) } -// TestLiveRun invokes a real agent, so it only runs when one is named. +// resolveAgent names the agent the run phase evaluates. +// +// AZURE_AI_EVAL_AGENT wins when set. Otherwise one is discovered, and failing +// to find one is a failure rather than a skip: skipping by default is how the +// agent-target path went unverified for weeks while the suite reported green. +// +// The listing is /agents, not /assistants. They are different collections and +// a project can have plenty of the latter and none of the former — an eval +// target resolves against /agents, so an assistant name is accepted by the +// request and then fails the run with "resources not found". +func resolveAgent(t *testing.T, env *liveEnv) string { + t.Helper() + + if env.agentName != "" { + return env.agentName + } + + cred, err := liveCredential() + require.NoError(t, err) + token, err := cred.GetToken(context.Background(), policy.TokenRequestOptions{ + Scopes: []string{"https://ai.azure.com/.default"}, + }) + require.NoError(t, err, "acquiring a token to list agents") + + req, err := http.NewRequest(http.MethodGet, env.endpoint+"/agents?api-version="+projectAPIVersion, nil) + require.NoError(t, err) + req.Header.Set("Authorization", "Bearer "+token.Token) + + resp, err := http.DefaultClient.Do(req) + require.NoError(t, err, "listing the project's agents") + defer resp.Body.Close() + require.Equal(t, http.StatusOK, resp.StatusCode, + "could not list agents; set AZURE_AI_EVAL_AGENT to name one directly") + + var listing struct { + Data []struct { + Name string `json:"name"` + } `json:"data"` + } + require.NoError(t, json.NewDecoder(resp.Body).Decode(&listing)) + + for _, a := range listing.Data { + if a.Name != "" { + t.Logf("no AZURE_AI_EVAL_AGENT set; evaluating %q", a.Name) + return a.Name + } + } + + t.Fatal("this project has no agent in /agents, so the agent-target run path " + + "cannot be verified here. Assistants do not count: an eval target " + + "resolves against /agents, and naming an assistant fails the run with " + + "\"resources not found\". Deploy an agent, or set AZURE_AI_EVAL_AGENT " + + "to one in another project") + return "" +} + +// TestLiveRun invokes a real agent, which is the only cover the agent-target +// run path has. func TestLiveRun(t *testing.T) { env := setup(t) - if env.agentName == "" { - t.Skip("set AZURE_AI_EVAL_AGENT to a deployed agent to exercise the run phase") - } + agentName := resolveAgent(t, env) ctx := context.Background() builtins, err := env.evalClient.ListEvaluators( @@ -285,7 +362,7 @@ func TestLiveRun(t *testing.T) { }) require.NoError(t, err, "creating the eval for the run") - ds := eval_api.NewAgentTargetDataSource(env.agentName, nil) + ds := eval_api.NewAgentTargetDataSource(agentName, nil) ds.SetFileContent([]map[string]any{ {"query": "How do I reset my password?"}, }) From f9bb2b098de7cc396a381610c7852ec10377f51d Mon Sep 17 00:00:00 2001 From: mohessie Date: Mon, 3 Aug 2026 02:28:45 +0300 Subject: [PATCH 070/320] Cancel runs through the OpenAI-spec route The service documents POST on the run itself as the cancel route, matching the OpenAI SDK, and keeps the /cancel suffix only as a back-compat alias. Move to the documented one. The body now carries meaning: empty cancels the run, non-empty updates its status and counters for trusted callers. Both are POST on the same path and both return 200, so a stray body would overwrite a run instead of stopping it and nothing offline would catch it. TestLiveRunCancel starts a run and requires it to reach canceled. --- .../azd-eval-extension-implementation.md | 2 +- .../internal/pkg/eval_api/evaluators.go | 5 +- .../tests/live/run_cancel_test.go | 104 ++++++++++++++++++ 3 files changed, 109 insertions(+), 2 deletions(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/tests/live/run_cancel_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/azd-eval-extension-implementation.md b/cli/azd/extensions/azure.ai.evaluations/azd-eval-extension-implementation.md index 44fb4ae170b..5fda30c6e18 100644 --- a/cli/azd/extensions/azure.ai.evaluations/azd-eval-extension-implementation.md +++ b/cli/azd/extensions/azure.ai.evaluations/azd-eval-extension-implementation.md @@ -162,7 +162,7 @@ type OpenAIEval struct { ID, Name string } // ID is canonical; Name is NOT | get / list | `GET /openai/v1/evals/{id}` · `GET /openai/v1/evals?limit=` | | start run | `POST /openai/v1/evals/{evalId}/runs` | | poll / list runs | `GET /openai/v1/evals/{evalId}/runs/{runId}` · `GET …/runs` | -| cancel | `POST /openai/v1/evals/{evalId}/runs/{runId}/cancel` | +| cancel | `POST /openai/v1/evals/{evalId}/runs/{runId}` with an empty body | | results | `GET …/runs/{runId}` → `result_counts` + `per_testing_criteria_results` | **The group carries evaluators, not the dataset.** The dataset goes on the **run**. `evaluation_level` is `turn` | `conversation`, service default **`turn`**. diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/evaluators.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/evaluators.go index 2d9a85ffa65..84ad2ac811f 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/evaluators.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/evaluators.go @@ -174,13 +174,16 @@ func (c *EvalClient) DeleteEvaluatorVersion( } // CancelOpenAIEvalRun stops an in-flight run. +// +// The body must stay nil: this route cancels only when the body is empty, and +// updates the run's status and counters when it is not. func (c *EvalClient) CancelOpenAIEvalRun( ctx context.Context, evalID string, runID string, ) (*OpenAIEvalRun, error) { path := fmt.Sprintf( - "%s/%s/runs/%s/cancel", + "%s/%s/runs/%s", pathOpenAIEvals, url.PathEscape(evalID), url.PathEscape(runID), ) return doRequestTyped[OpenAIEvalRun](c, ctx, http.MethodPost, path, nil, nil, "") diff --git a/cli/azd/extensions/azure.ai.evaluations/tests/live/run_cancel_test.go b/cli/azd/extensions/azure.ai.evaluations/tests/live/run_cancel_test.go new file mode 100644 index 00000000000..a6b01729a5e --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/tests/live/run_cancel_test.go @@ -0,0 +1,104 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +//go:build live + +package live + +import ( + "context" + "strings" + "testing" + "time" + + "azureaieval/internal/pkg/eval_api" + + "github.com/stretchr/testify/require" +) + +// TestLiveRunCancel covers the one route whose meaning depends on the request +// body. POST on the run cancels it when the body is empty and updates its +// status and counters when it is not, so a stray body here would silently +// overwrite a run instead of stopping it. Only a live call can tell the two +// apart: both are the same method on the same path, and both return 200. +func TestLiveRunCancel(t *testing.T) { + env := setup(t) + agentName := resolveAgent(t, env) + ctx := context.Background() + + builtins, err := env.evalClient.ListEvaluators( + ctx, eval_api.EvaluatorTypeBuiltin, projectAPIVersion) + require.NoError(t, err) + require.NotEmpty(t, builtins.Value) + evaluatorName := pickQualityEvaluator(t, builtins.Value) + + group, err := env.evalClient.CreateOpenAIEval(ctx, &eval_api.CreateOpenAIEvalRequest{ + Name: uniqueName("azd-eval-e2e-cancel"), + DataSourceConfig: &eval_api.DataSourceConfig{ + Type: "custom", + IncludeSampleSchema: true, + ItemSchema: map[string]any{ + "type": "object", + "properties": map[string]any{"query": map[string]any{"type": "string"}}, + }, + }, + TestingCriteria: []eval_api.TestingCriterion{{ + Type: "azure_ai_evaluator", + Name: strings.TrimPrefix(evaluatorName, "builtin."), + EvaluatorName: evaluatorName, + DataMapping: map[string]string{ + "query": "{{item.query}}", + "response": "{{sample.output_items}}", + "tool_calls": "{{sample.tool_calls}}", + "tool_definitions": "{{sample.tool_definitions}}", + }, + InitializationParameters: map[string]any{ + "model": env.judgeModel, + "deployment_name": env.judgeModel, + }, + }}, + }) + require.NoError(t, err, "creating the eval to cancel a run from") + + t.Cleanup(func() { + _ = env.evalClient.DeleteOpenAIEval(context.Background(), group.ID) + }) + + ds := eval_api.NewAgentTargetDataSource(agentName, nil) + ds.SetFileContent([]map[string]any{ + {"query": "How do I reset my password?"}, + }) + + run, err := env.evalClient.CreateOpenAIEvalRun(ctx, group.ID, &eval_api.CreateOpenAIEvalRunRequest{ + Name: uniqueName("cancel"), + DataSource: ds, + }) + require.NoError(t, err, "starting the run to cancel") + require.NotEmpty(t, run.ID) + t.Logf("started run %s (status %s)", run.ID, run.Status) + + canceled, err := env.evalClient.CancelOpenAIEvalRun(ctx, group.ID, run.ID) + require.NoError(t, err, "cancelling the run") + require.NotNil(t, canceled) + t.Logf("cancel returned status %s", canceled.Status) + + // A sample takes roughly 40 seconds, so a run cancelled immediately after + // it starts should never reach completed. + deadline := time.Now().Add(5 * time.Minute) + var status string + for { + current, err := env.evalClient.GetOpenAIEvalRun(ctx, group.ID, run.ID) + require.NoError(t, err, "polling the cancelled run") + status = strings.ToLower(current.Status) + if status == "canceled" || status == "cancelled" { + break + } + require.NotEqual(t, "completed", status, + "the run completed instead of cancelling, so the empty-body POST did not cancel it") + require.False(t, time.Now().After(deadline), + "the run never reached a cancelled state; last status was %s", status) + time.Sleep(5 * time.Second) + } + + t.Logf("run reached %s", status) +} From 042ae636a77f9599bcc2584b5fea4009a5a22fd6 Mon Sep 17 00:00:00 2001 From: mohessie Date: Mon, 3 Aug 2026 02:48:30 +0300 Subject: [PATCH 071/320] Cover the remaining commands with end-to-end CLI tests The harness drives the built binary as a subprocess against a live project, so these exercise the same path a user takes rather than the client packages underneath. dataset, generate, results, rubric, and run ops each gain a test, and a shared fixture publishes the eval they all read. One product bug surfaced: dataset delete answers 204, which the client rejected, so every successful delete reported an error. The eval client already allowed 204; the dataset client now does too. The fixture outlives any single test, so it is released from TestMain rather than t.Cleanup, which would take it from the tests that follow. --- .../internal/cmd/generate_plan_test.go | 253 +++++++++++ .../internal/pkg/dataset_api/operations.go | 6 +- .../tests/cli/dataset_test.go | 241 +++++++++++ .../tests/cli/fixture_test.go | 404 ++++++++++++++++++ .../tests/cli/generate_test.go | 158 +++++++ .../tests/cli/harness_test.go | 3 + .../tests/cli/results_test.go | 317 ++++++++++++++ .../tests/cli/rubric_test.go | 203 +++++++++ .../tests/cli/run_ops_test.go | 208 +++++++++ 9 files changed, 1792 insertions(+), 1 deletion(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_plan_test.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/tests/cli/dataset_test.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/tests/cli/fixture_test.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/tests/cli/generate_test.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/tests/cli/results_test.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/tests/cli/rubric_test.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/tests/cli/run_ops_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_plan_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_plan_test.go new file mode 100644 index 00000000000..584a19e7547 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_plan_test.go @@ -0,0 +1,253 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "azureaieval/internal/project" + + "github.com/stretchr/testify/require" +) + +// `generate` decides what to submit before it touches the network, so the plan +// it builds — which artifacts, from what instruction, at what sample size — is +// checkable without paying for a generation job. These are the parts that +// cannot be observed afterwards: once the jobs are submitted, a wrong default +// is indistinguishable from an intended one. + +func writeConfig(t *testing.T, body string) string { + t.Helper() + path := filepath.Join(t.TempDir(), "eval_generate.yaml") + require.NoError(t, os.WriteFile(path, []byte(body), 0o600)) + return path +} + +// A spec is optional, so the defaults are what most callers actually run with. +func TestResolveGenerateConfigDefaultsFromFlagsAlone(t *testing.T) { + absent := filepath.Join(t.TempDir(), "absent.yaml") + + cfg, err := resolveGenerateConfig(absent, "shop-agent", "gpt-4o-mini", "", 0, 0) + require.NoError(t, err) + require.NoError(t, cfg.Validate()) + + require.Equal(t, "shop-agent", cfg.Agent.Name) + require.NotNil(t, cfg.Generate.Rubric) + require.Equal(t, "shop-agent-quality", cfg.Generate.Rubric.Name) + require.Equal(t, "gpt-4o-mini", cfg.Generate.Rubric.Model) + require.Equal(t, "./"+project.DefaultEvaluatorsDir, cfg.Generate.Rubric.LocalDir) + + require.NotNil(t, cfg.Generate.Dataset) + require.Equal(t, "shop-agent-golden", cfg.Generate.Dataset.Name) + require.Equal(t, project.StrategySynthetic, cfg.Generate.Dataset.Strategy) + require.Equal(t, project.DefaultSampleSize, cfg.Generate.Dataset.SampleSize) + require.Equal(t, "./"+project.DefaultDatasetsDir, cfg.Generate.Dataset.LocalDir) +} + +// Without a target there is nothing to generate from, and the refusal has to +// name the flag rather than a config field the caller may not have. +func TestResolveGenerateConfigRequiresATarget(t *testing.T) { + _, err := resolveGenerateConfig( + filepath.Join(t.TempDir(), "absent.yaml"), "", "gpt-4o-mini", "", 0, 0) + require.Error(t, err) + require.Contains(t, err.Error(), "--target") +} + +func TestResolveGenerateConfigReadsTheSpec(t *testing.T) { + path := writeConfig(t, ` +agent: + name: from-spec + context: + instructions: ./instructions.md + traces: + window: 7d + source: ignored-today + sample: 5 +generate: + rubric: + name: spec-rubric + model: gpt-4o + local_dir: ./custom-evaluators + dataset: + name: spec-dataset + strategy: synthetic + sampleSize: 200 + local_dir: ./custom-datasets +`) + + cfg, err := resolveGenerateConfig(path, "", "", "", 0, 0) + require.NoError(t, err) + require.NoError(t, cfg.Validate()) + + require.Equal(t, "from-spec", cfg.Agent.Name) + require.Equal(t, "./instructions.md", cfg.Agent.Context.Instructions) + require.Equal(t, "spec-rubric", cfg.Generate.Rubric.Name) + require.Equal(t, "gpt-4o", cfg.Generate.Rubric.Model) + require.Equal(t, "./custom-evaluators", cfg.Generate.Rubric.LocalDir) + require.Equal(t, "spec-dataset", cfg.Generate.Dataset.Name) + require.Equal(t, 200, cfg.Generate.Dataset.SampleSize) + + require.NotNil(t, cfg.Agent.Context.Traces) + require.Equal(t, "7d", cfg.Agent.Context.Traces.Window) + require.Equal(t, 5, cfg.Agent.Context.Traces.Sample) +} + +// Flags win over the spec, which is what makes a one-off run possible without +// editing a file that is checked in. +func TestResolveGenerateConfigLayersFlagsOverTheSpec(t *testing.T) { + path := writeConfig(t, ` +agent: + name: from-spec +generate: + rubric: + name: spec-rubric + model: gpt-4o + dataset: + name: spec-dataset + sampleSize: 200 +`) + + cfg, err := resolveGenerateConfig(path, "from-flag", "gpt-4o-mini", "", 500, 14) + require.NoError(t, err) + require.NoError(t, cfg.Validate()) + + require.Equal(t, "from-flag", cfg.Agent.Name) + require.Equal(t, "gpt-4o-mini", cfg.Generate.Rubric.Model) + require.Equal(t, 500, cfg.Generate.Dataset.SampleSize) + require.Equal(t, "14d", cfg.Agent.Context.Traces.Window, + "--trace-days must reach the spec as a window, since that is the only "+ + "trace field the generation API takes") + + // The rubric name is not derived when the spec named one, so a --target + // override must not silently rename an artifact the spec author declared. + require.Equal(t, "spec-rubric", cfg.Generate.Rubric.Name) +} + +// A spec that declares a dataset without a size still has to submit a legal +// job, so the default is applied rather than left at zero. +func TestResolveGenerateConfigFillsAMissingSampleSize(t *testing.T) { + path := writeConfig(t, ` +agent: + name: sized +generate: + dataset: + name: no-size +`) + + cfg, err := resolveGenerateConfig(path, "", "", "", 0, 0) + require.NoError(t, err) + require.Equal(t, project.DefaultSampleSize, cfg.Generate.Dataset.SampleSize) + require.NoError(t, cfg.Validate()) +} + +// The bounds are the service's, and the boundaries themselves have to be +// accepted: a check that rejected 15 or 1000 would be indistinguishable from +// one that is simply too strict. +func TestGenerateSampleSizeBounds(t *testing.T) { + for _, tc := range []struct { + size int + allowed bool + }{ + {project.MinSampleSize - 1, false}, + {project.MinSampleSize, true}, + {project.DefaultSampleSize, true}, + {project.MaxSampleSize, true}, + {project.MaxSampleSize + 1, false}, + } { + cfg, err := resolveGenerateConfig( + filepath.Join(t.TempDir(), "absent.yaml"), + "bounded", "gpt-4o-mini", "", tc.size, 0) + require.NoError(t, err) + require.Equal(t, tc.size, cfg.Generate.Dataset.SampleSize) + + err = cfg.Validate() + if tc.allowed { + require.NoErrorf(t, err, "%d is inside the service's range", tc.size) + continue + } + require.Errorf(t, err, "%d is outside the service's range", tc.size) + require.Contains(t, err.Error(), "sampleSize") + } +} + +// --dataset and --evaluator both mean "use this one". Only the dataset side is +// resolved here; the evaluator side is decided in the command body, so it is +// covered by the CLI test that watches for the skip message. +func TestResolveGenerateConfigSkipsTheDatasetWhenOneIsSupplied(t *testing.T) { + cfg, err := resolveGenerateConfig( + filepath.Join(t.TempDir(), "absent.yaml"), + "supplied", "gpt-4o-mini", "prod-sample", 0, 0) + require.NoError(t, err) + require.Nil(t, cfg.Generate.Dataset, "a supplied dataset must not be generated") + require.NotNil(t, cfg.Generate.Rubric) + require.NoError(t, cfg.Validate()) +} + +// Both jobs bill against one deployment, so a spec with no rubric has no model +// to run either of them. +func TestGenerationModelComesFromTheRubricSpec(t *testing.T) { + require.Equal(t, "", generationModel(&project.GenerateConfig{})) + + cfg := &project.GenerateConfig{} + cfg.Generate.Rubric = &project.RubricSpec{Name: "r", Model: "gpt-4o"} + require.Equal(t, "gpt-4o", generationModel(cfg)) +} + +func TestResolveInstruction(t *testing.T) { + dir := t.TempDir() + filled := filepath.Join(dir, "instruction.md") + require.NoError(t, os.WriteFile(filled, []byte(" test refunds and returns\n\n"), 0o600)) + blank := filepath.Join(dir, "blank.md") + require.NoError(t, os.WriteFile(blank, []byte(" \n"), 0o600)) + + t.Run("inline is returned as given", func(t *testing.T) { + got, err := resolveInstruction("inline text", "") + require.NoError(t, err) + require.Equal(t, "inline text", got) + }) + + t.Run("a file is read and trimmed", func(t *testing.T) { + got, err := resolveInstruction("", filled) + require.NoError(t, err) + require.Equal(t, "test refunds and returns", got) + }) + + // A whitespace-only file would otherwise generate from nothing, which + // produces a rubric with no relation to the agent. + t.Run("an empty file is refused", func(t *testing.T) { + _, err := resolveInstruction("", blank) + require.Error(t, err) + require.Contains(t, err.Error(), "is empty") + }) + + t.Run("a missing file names the flag", func(t *testing.T) { + _, err := resolveInstruction("", filepath.Join(dir, "absent.md")) + require.Error(t, err) + require.Contains(t, err.Error(), "--agent-instruction-file") + }) +} + +// The generation API takes a day window and nothing else, so the two fields it +// drops are reported rather than silently discarded. +func TestWarnIgnoredTraceFields(t *testing.T) { + cfg := &project.GenerateConfig{} + cfg.Agent.Context.Traces = &project.TraceSpec{Window: "7d", Source: "some-source", Sample: 5} + + var out strings.Builder + warnIgnoredTraceFields(cfg, &out) + require.Contains(t, out.String(), "agent.context.traces.source") + require.Contains(t, out.String(), "agent.context.traces.sample") + require.NotContains(t, out.String(), "agent.context.traces.window", + "the window is the one trace field the API takes, so it is not a no-op") + + // A window on its own is fully supported and must not produce a warning. + quiet := &project.GenerateConfig{} + quiet.Agent.Context.Traces = &project.TraceSpec{Window: "7d"} + out.Reset() + warnIgnoredTraceFields(quiet, &out) + require.Empty(t, out.String()) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/operations.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/operations.go index c285c19a450..74d1752a4c5 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/operations.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/operations.go @@ -558,7 +558,11 @@ func (c *DatasetClient) doRequest( log.Printf("[dataset_api] response status: %d", resp.StatusCode) - if !runtime.HasStatusCode(resp, http.StatusOK, http.StatusCreated, http.StatusAccepted) { + // 204 belongs here for the same reason it does in eval_api: a delete that + // removed the version answers No Content, and rejecting that reports every + // successful delete as an error. + if !runtime.HasStatusCode(resp, + http.StatusOK, http.StatusCreated, http.StatusAccepted, http.StatusNoContent) { resp.Body = io.NopCloser(bytes.NewReader(respBody)) return nil, runtime.NewResponseError(resp) } diff --git a/cli/azd/extensions/azure.ai.evaluations/tests/cli/dataset_test.go b/cli/azd/extensions/azure.ai.evaluations/tests/cli/dataset_test.go new file mode 100644 index 00000000000..3b0dbeb1856 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/tests/cli/dataset_test.go @@ -0,0 +1,241 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +//go:build live + +package cli + +import ( + "os" + "path/filepath" + "sync" + "testing" + + "github.com/stretchr/testify/require" +) + +type datasetSummary struct { + Name string `json:"name"` + Version string `json:"version"` + Format string `json:"format"` +} + +const datasetRows = `{"query":"How do I reset my password?"} +{"query":"What is the refund window?"} +` + +// registeredDataset is a dataset with more than one version, which is what +// makes --version on show and --name on list worth asserting. +type registeredDataset struct { + Name string + // Versions are read back from each registration rather than assumed to + // start at 1: the server assigns them, and a test that hardcoded the + // numbering would be asserting its own guess. + Versions []string +} + +var ( + readOnlyDatasetOnce sync.Once + readOnlyDataset *registeredDataset +) + +// sharedDataset is registered once for the tests that only read it. Each +// registration uploads a blob, so redoing it per test buys nothing. +func sharedDataset(t *testing.T) *registeredDataset { + t.Helper() + readOnlyDatasetOnce.Do(func() { + readOnlyDataset = registerDataset(t, 2) + }) + require.NotNil(t, readOnlyDataset, "the shared dataset could not be registered") + return readOnlyDataset +} + +// registerDataset publishes a dataset and removes every version it created. +func registerDataset(t *testing.T, versions int) *registeredDataset { + t.Helper() + require.Positive(t, versions) + + path := filepath.Join(t.TempDir(), "golden.jsonl") + require.NoError(t, os.WriteFile(path, []byte(datasetRows), 0o600)) + + ds := ®isteredDataset{Name: uniqueName("azdcli_ds")} + for range versions { + r := requireSuccess(t, run(t, "dataset", "create", + "--name", ds.Name, "--file", path, "-o", "json")) + + var created datasetSummary + r.JSON(t, &created) + require.NotEmpty(t, created.Version, "the service assigns the version") + ds.Versions = append(ds.Versions, created.Version) + + version := created.Version + deferTeardown(func() { + runQuietly("dataset", "delete", "--name", ds.Name, "--version", version) + }) + } + require.Len(t, ds.Versions, versions) + require.NotEqual(t, ds.Versions[0], ds.Versions[len(ds.Versions)-1], + "re-registering must advance the version rather than overwrite") + return ds +} + +func TestCLIDatasetList(t *testing.T) { + ds := sharedDataset(t) + + t.Run("table", func(t *testing.T) { + r := requireSuccess(t, run(t, "dataset", "list", "--name", ds.Name)) + for _, header := range []string{"NAME", "VERSION", "FORMAT"} { + require.Containsf(t, r.Stdout, header, "the listing lost its %s column", header) + } + require.Contains(t, r.Stdout, ds.Name) + }) + + // --name is what makes the listing usable once a project holds more than a + // screenful: it narrows to one dataset's versions. + t.Run("name scopes the listing to one dataset's versions", func(t *testing.T) { + r := requireSuccess(t, run(t, "dataset", "list", "--name", ds.Name, "-o", "json")) + var listed []datasetSummary + r.JSON(t, &listed) + require.NotEmpty(t, listed) + + seen := map[string]bool{} + for _, v := range listed { + require.Equalf(t, ds.Name, v.Name, + "--name must return only that dataset's versions; got %q", v.Name) + seen[v.Version] = true + } + for _, want := range ds.Versions { + require.Truef(t, seen[want], "version %s is missing from the listing", want) + } + }) + + // Unscoped, the listing is every dataset rather than every version, so the + // one just registered has to be in it. + t.Run("unscoped lists the project's datasets", func(t *testing.T) { + r := requireSuccess(t, run(t, "dataset", "list", "-o", "json")) + var all []datasetSummary + r.JSON(t, &all) + require.NotEmpty(t, all) + + found := false + for _, d := range all { + if d.Name == ds.Name { + found = true + } + } + require.True(t, found, "a registered dataset must appear in the unscoped listing") + }) + + t.Run("an unknown name lists nothing rather than failing", func(t *testing.T) { + r := requireSuccess(t, run(t, "dataset", "list", + "--name", "azdcli-no-such-dataset", "-o", "json")) + var listed []datasetSummary + r.JSON(t, &listed) + require.Empty(t, listed) + }) +} + +func TestCLIDatasetShow(t *testing.T) { + ds := sharedDataset(t) + latest := ds.Versions[len(ds.Versions)-1] + + // Omitting the version means the latest, which is the only sensible + // default for a name that gains a version on every registration. + t.Run("defaults to the latest version", func(t *testing.T) { + r := requireSuccess(t, run(t, "dataset", "show", "--name", ds.Name, "-o", "json")) + var shown datasetSummary + r.JSON(t, &shown) + require.Equal(t, ds.Name, shown.Name) + require.Equal(t, latest, shown.Version) + }) + + t.Run("version pins an earlier one", func(t *testing.T) { + r := requireSuccess(t, run(t, "dataset", "show", + "--name", ds.Name, "--version", ds.Versions[0], "-o", "json")) + var shown datasetSummary + r.JSON(t, &shown) + require.Equal(t, ds.Versions[0], shown.Version) + require.NotEqual(t, latest, shown.Version) + }) + + t.Run("table", func(t *testing.T) { + r := requireSuccess(t, run(t, "dataset", "show", "--name", ds.Name)) + for _, header := range []string{"NAME", "VERSION", "FORMAT", "URI"} { + require.Containsf(t, r.Stdout, header, "the table lost its %s column", header) + } + require.Contains(t, r.Stdout, ds.Name) + }) + + t.Run("the name is required", func(t *testing.T) { + r := requireFailure(t, run(t, "dataset", "show")) + require.Contains(t, r.Combined(), "--name is required") + }) + + t.Run("an unknown dataset is brief", func(t *testing.T) { + r := requireFailure(t, run(t, "dataset", "show", "--name", "azdcli-no-such-dataset")) + require.Less(t, len(r.Combined()), 600, + "a not-found must stay short, not dump the service body:\n%s", r.Combined()) + require.Contains(t, r.Combined(), "azdcli-no-such-dataset") + }) + + t.Run("an unknown version of a real dataset is refused", func(t *testing.T) { + r := requireFailure(t, run(t, "dataset", "show", + "--name", ds.Name, "--version", "9999")) + require.Contains(t, r.Combined(), "9999") + require.Less(t, len(r.Combined()), 600, r.Combined()) + }) +} + +func TestCLIDatasetDelete(t *testing.T) { + t.Run("both flags are required", func(t *testing.T) { + require.Contains(t, + requireFailure(t, run(t, "dataset", "delete", "--version", "1")).Combined(), + "--name is required") + require.Contains(t, + requireFailure(t, run(t, "dataset", "delete", "--name", "whatever")).Combined(), + "--version is required") + }) + + // Deleting something that was never registered succeeds. The service + // treats DELETE as idempotent and answers 204 whatever the name, so the + // command reports a removal it did not perform — and the not-found branch + // in `dataset delete` cannot be reached this way. Asserted rather than + // wished away, because a caller scripting against the exit code is + // entitled to know it means "gone", not "was there and is now gone". + t.Run("deleting an unregistered dataset is idempotent, not an error", func(t *testing.T) { + r := requireSuccess(t, run(t, "dataset", "delete", + "--name", "azdcli-no-such-dataset", "--version", "1")) + require.Contains(t, r.Stdout, "Deleted dataset") + + listed := requireSuccess(t, run(t, "dataset", "list", + "--name", "azdcli-no-such-dataset", "-o", "json")) + var remaining []datasetSummary + listed.JSON(t, &remaining) + require.Empty(t, remaining, "nothing was there to delete in the first place") + }) + + // A successful delete answers 204 No Content, so asserting the exit code + // is what catches a client that reads an empty body as a failure and + // reports a removal it just performed as an error. + t.Run("one version is removed and the other survives", func(t *testing.T) { + ds := registerDataset(t, 2) + gone, kept := ds.Versions[0], ds.Versions[1] + + r := requireSuccess(t, run(t, "dataset", "delete", + "--name", ds.Name, "--version", gone)) + require.Contains(t, r.Stdout, "Deleted dataset") + require.Contains(t, r.Stdout, ds.Name) + + listed := requireSuccess(t, run(t, "dataset", "list", + "--name", ds.Name, "-o", "json")) + var remaining []datasetSummary + listed.JSON(t, &remaining) + + versions := map[string]bool{} + for _, v := range remaining { + versions[v.Version] = true + } + require.False(t, versions[gone], "the deleted version must leave the listing") + require.True(t, versions[kept], "deleting one version must not remove the others") + }) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/tests/cli/fixture_test.go b/cli/azd/extensions/azure.ai.evaluations/tests/cli/fixture_test.go new file mode 100644 index 00000000000..af745cae250 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/tests/cli/fixture_test.go @@ -0,0 +1,404 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +//go:build live + +package cli + +import ( + "context" + "encoding/json" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "azureaieval/internal/pkg/eval_api" + "azureaieval/internal/pkg/evalcore" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" + "github.com/Azure/azure-sdk-for-go/sdk/azidentity" +) + +// The command tests need an eval that has already been run, and building one +// through the CLI is not possible: there is no command that creates an eval +// from flags, only `run`, which needs a config file and a deployed target. +// So the fixture is built with the client and every assertion is made against +// the binary. What is under test is the command surface; the eval is scenery. +// +// It is built once for the whole package because two completed runs cost +// minutes, and torn down in TestMain rather than t.Cleanup so that whichever +// test happened to trigger the build does not take the fixture away from the +// rest. + +const fixtureAPIVersion = "2025-11-15-preview" + +// scoringGrader splits the rows deterministically. A grader that scores every +// row the same way makes --failed-only and a comparison indistinguishable from +// a no-op, so "good" is the difference between a pass and a failure. +const scoringGrader = `def grade(sample, item) -> float: + response = (item or {}).get("response", "") + return 1.0 if "good" in response else 0.0 +` + +// evalFixture is one eval with two completed runs of the same criterion. +type evalFixture struct { + EvaluatorName string + EvalID string + + // Baseline scores worse than Treatment, so a comparison between them has + // a delta to report rather than zero. + BaselineRunID string + TreatmentRunID string +} + +var ( + fixtureOnce sync.Once + fixture *evalFixture + fixtureErr error + + // teardown runs after the last test, in reverse order. + teardownMu sync.Mutex + teardown []func() +) + +func deferTeardown(fn func()) { + teardownMu.Lock() + defer teardownMu.Unlock() + teardown = append(teardown, fn) +} + +func runTeardown() { + teardownMu.Lock() + defer teardownMu.Unlock() + for i := len(teardown) - 1; i >= 0; i-- { + teardown[i]() + } + teardown = nil +} + +// runQuietly invokes the binary without a *testing.T. +// +// Teardown runs after the last test has reported, and logging or asserting +// against a finished test panics, so nothing here may touch one. +func runQuietly(args ...string) { + full := append(append([]string{}, args...), "--project-endpoint", endpoint) + _ = exec.Command(binaryPath, full...).Run() +} + +var ( + credOnce sync.Once + cred *azidentity.AzureDeveloperCLICredential + credErr error +) + +// liveClient builds the client the fixture is assembled with. One credential +// for the package, because azidentity caches tokens per instance and a fresh +// one per call makes every call shell out to azd again. +// +// The first token is fetched here rather than lazily on the first request: +// that call is the one that flakes, and paying for it up front means the rest +// of the fixture runs against a cached token. +func liveClient() (*eval_api.EvalClient, error) { + credOnce.Do(func() { + cred, credErr = azidentity.NewAzureDeveloperCLICredential( + &azidentity.AzureDeveloperCLICredentialOptions{}) + if credErr != nil { + return + } + credErr = retryCredentialFlake(func() error { + _, err := cred.GetToken(context.Background(), policy.TokenRequestOptions{ + Scopes: []string{"https://ai.azure.com/.default"}, + }) + return err + }) + }) + if credErr != nil { + return nil, credErr + } + return eval_api.NewEvalClient(endpoint, cred), nil +} + +// retryCredentialFlake reruns a request that failed only because azd's token +// helper exited non-zero. +// +// It is the same failure the harness retries around the binary, for the same +// reason: nothing was sent, and the alternative is a suite that fails on a +// different test each run for a reason unrelated to the code. Any other error +// is returned immediately. +func retryCredentialFlake(fn func() error) error { + var err error + for attempt := range 4 { + if attempt > 0 { + time.Sleep(time.Duration(attempt) * 2 * time.Second) + } + if err = fn(); err == nil || !strings.Contains(err.Error(), credentialFlake) { + return err + } + } + return err +} + +// sharedEval returns the fixture, building it on first use. +// +// A failure here fails the calling test rather than skipping it: every test +// that asks for the fixture is testing something that cannot be exercised +// without one, and a suite that goes green because its subject was missing is +// worse than one that goes red. +func sharedEval(t *testing.T) *evalFixture { + t.Helper() + fixtureOnce.Do(func() { + start := time.Now() + fixture, fixtureErr = buildFixture(t.Logf) + t.Logf("fixture ready in %s", time.Since(start).Round(time.Second)) + }) + if fixtureErr != nil { + t.Fatalf("building the shared eval the command tests run against: %v", fixtureErr) + } + return fixture +} + +func buildFixture(logf func(string, ...any)) (*evalFixture, error) { + ctx := context.Background() + + client, err := liveClient() + if err != nil { + return nil, fmt.Errorf("acquiring an azd credential: %w", err) + } + + name := strings.ReplaceAll(uniqueName("azdclifx"), "-", "_") + script, err := publishScoringEvaluator(ctx, client, name) + if err != nil { + return nil, err + } + logf("published code evaluator %s version %s", name, script.Version) + + if err := awaitEvaluatorListed(ctx, client, name, script.Version); err != nil { + return nil, err + } + + evalID, err := createFixtureEval(ctx, client, name) + if err != nil { + return nil, err + } + logf("created eval %s", evalID) + + // Different pass rates so the comparison has something to measure. + baseline, err := startFixtureRun(ctx, client, evalID, "baseline", + []string{"a bad answer", "another bad answer", "a good answer"}) + if err != nil { + return nil, err + } + treatment, err := startFixtureRun(ctx, client, evalID, "treatment", + []string{"a good answer", "another good answer", "a third good answer"}) + if err != nil { + return nil, err + } + logf("started runs %s and %s", baseline, treatment) + + // Polled together: they are independent, and serialising them doubles the + // slowest part of the suite for nothing. + errs := make(chan error, 2) + for _, runID := range []string{baseline, treatment} { + go func(id string) { errs <- awaitCompleted(ctx, client, evalID, id, logf) }(runID) + } + for range 2 { + if err := <-errs; err != nil { + return nil, err + } + } + + return &evalFixture{ + EvaluatorName: name, + EvalID: evalID, + BaselineRunID: baseline, + TreatmentRunID: treatment, + }, nil +} + +func publishScoringEvaluator( + ctx context.Context, + client *eval_api.EvalClient, + name string, +) (*eval_api.EvaluatorVersion, error) { + dir, err := os.MkdirTemp("", "azdcli-grader") + if err != nil { + return nil, err + } + defer os.RemoveAll(dir) + + path := filepath.Join(dir, name+".py") + if err := os.WriteFile(path, []byte(scoringGrader), 0o600); err != nil { + return nil, err + } + script, err := evalcore.LoadCodeEvaluator(name, path) + if err != nil { + return nil, fmt.Errorf("loading the grader: %w", err) + } + + // Without a data_schema the criteria builder has no mapping to derive, so + // the schema is what makes the evaluator usable rather than merely + // publishable. + opts := eval_api.CodeEvaluatorOptions{ + DataSchema: json.RawMessage( + `{"type":"object","properties":{"response":{"type":"string"}},"required":["response"]}`), + Metrics: json.RawMessage( + `{"result":{"type":"continuous","desirable_direction":"increase","is_primary":true}}`), + } + var version *eval_api.EvaluatorVersion + if err := retryCredentialFlake(func() error { + var err error + version, err = client.CreateCodeEvaluatorVersion(ctx, script, opts, fixtureAPIVersion) + return err + }); err != nil { + return nil, fmt.Errorf("publishing the code evaluator: %w", err) + } + deferTeardown(func() { + _ = client.DeleteEvaluatorVersion( + context.Background(), name, version.Version, fixtureAPIVersion) + }) + return version, nil +} + +// awaitEvaluatorListed waits for the version listing to catch up, which is the +// view eval creation resolves against. The direct read goes consistent first, +// so waiting on that alone still leaves the create failing with "was not +// found". +func awaitEvaluatorListed( + ctx context.Context, + client *eval_api.EvalClient, + name, version string, +) error { + deadline := time.Now().Add(2 * time.Minute) + for { + list, err := client.ListEvaluatorVersions(ctx, name, fixtureAPIVersion) + if err == nil && list != nil { + for _, entry := range list.Value { + if entry.Version == version { + return nil + } + } + } + if time.Now().After(deadline) { + return fmt.Errorf("evaluator %s version %s never appeared in the listing", name, version) + } + time.Sleep(500 * time.Millisecond) + } +} + +func createFixtureEval( + ctx context.Context, + client *eval_api.EvalClient, + evaluatorName string, +) (string, error) { + // Hand-written rather than built with buildEvalRequest, which is + // unexported. That is safe only because the evaluator is one published + // here whose schema is a single `response` column; a built-in would need + // the shipping builder, since their input contracts differ per evaluator. + var group *eval_api.OpenAIEval + if err := retryCredentialFlake(func() error { + var err error + group, err = client.CreateOpenAIEval(ctx, &eval_api.CreateOpenAIEvalRequest{ + Name: uniqueName("azdcli-fixture"), + DataSourceConfig: &eval_api.DataSourceConfig{ + Type: "custom", + ItemSchema: map[string]any{ + "type": "object", + "properties": map[string]any{"response": map[string]any{"type": "string"}}, + }, + }, + TestingCriteria: []eval_api.TestingCriterion{{ + Type: "azure_ai_evaluator", + Name: evaluatorName, + EvaluatorName: evaluatorName, + DataMapping: map[string]string{"response": "{{item.response}}"}, + }}, + }) + return err + }); err != nil { + return "", fmt.Errorf("creating the fixture eval: %w", err) + } + deferTeardown(func() { + _ = client.DeleteOpenAIEval(context.Background(), group.ID) + }) + return group.ID, nil +} + +func startFixtureRun( + ctx context.Context, + client *eval_api.EvalClient, + evalID, label string, + responses []string, +) (string, error) { + rows := make([]map[string]any, 0, len(responses)) + for _, r := range responses { + rows = append(rows, map[string]any{"response": r}) + } + + ds := eval_api.NewDatasetOnlyDataSource() + ds.SetFileContent(rows) + + var run *eval_api.OpenAIEvalRun + if err := retryCredentialFlake(func() error { + var err error + run, err = client.CreateOpenAIEvalRun(ctx, evalID, &eval_api.CreateOpenAIEvalRunRequest{ + Name: uniqueName("azdcli-" + label), + DataSource: ds, + }) + return err + }); err != nil { + return "", fmt.Errorf("starting the %s run: %w", label, err) + } + return run.ID, nil +} + +var terminalRunStatus = map[string]bool{ + "completed": true, "failed": true, "canceled": true, "cancelled": true, "error": true, +} + +// awaitCompleted requires the run to have scored something. +// +// A run whose every sample errors still reports completed, so the status alone +// would let the whole suite run against an eval that measured nothing. +func awaitCompleted( + ctx context.Context, + client *eval_api.EvalClient, + evalID, runID string, + logf func(string, ...any), +) error { + deadline := time.Now().Add(15 * time.Minute) + for { + var run *eval_api.OpenAIEvalRun + if err := retryCredentialFlake(func() error { + var err error + run, err = client.GetOpenAIEvalRun(ctx, evalID, runID) + return err + }); err != nil { + return fmt.Errorf("polling run %s: %w", runID, err) + } + if terminalRunStatus[strings.ToLower(run.Status)] { + if strings.ToLower(run.Status) != "completed" { + return fmt.Errorf("run %s finished as %q: %s", runID, run.Status, run.Failure()) + } + if run.ResultCounts == nil { + return fmt.Errorf("run %s completed without reporting counts", runID) + } + if run.ResultCounts.Passed+run.ResultCounts.Failed == 0 { + return fmt.Errorf( + "run %s completed without scoring any row (errored=%d); the fixture "+ + "would prove nothing", runID, run.ResultCounts.Errored) + } + logf("run %s completed: passed=%d failed=%d errored=%d", + runID, run.ResultCounts.Passed, run.ResultCounts.Failed, run.ResultCounts.Errored) + return nil + } + if time.Now().After(deadline) { + return fmt.Errorf("run %s did not finish in time (last status %q)", runID, run.Status) + } + time.Sleep(10 * time.Second) + } +} diff --git a/cli/azd/extensions/azure.ai.evaluations/tests/cli/generate_test.go b/cli/azd/extensions/azure.ai.evaluations/tests/cli/generate_test.go new file mode 100644 index 00000000000..ff3f37b0a31 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/tests/cli/generate_test.go @@ -0,0 +1,158 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +//go:build live + +package cli + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +// `generate` submits two jobs that cost model time and take minutes, so what +// is exercised here is everything up to that point: the flag combinations it +// refuses, the spec it parses, and the two flags that mean "I already have +// this one, do not make another". None of these tests submits a job — the last +// one reaches the service and deliberately generates nothing. + +// TestCLIGenerateRefusesBadFlagCombinations covers the mistakes that must cost +// nothing to make. Each of these is decided locally, so a user finds out +// before a job is billed. +func TestCLIGenerateRefusesBadFlagCombinations(t *testing.T) { + dir := t.TempDir() + instruction := filepath.Join(dir, "instruction.md") + require.NoError(t, os.WriteFile(instruction, []byte("test refunds"), 0o600)) + + cases := []struct { + name string + args []string + want string + }{{ + name: "the two instruction sources are mutually exclusive", + args: []string{"--target", "a", "--agent-instruction", "inline", + "--agent-instruction-file", instruction}, + want: "agent-instruction-file", + }, { + name: "below the minimum sample size", + args: []string{"--target", "a", "--max-samples", "14"}, + want: "between 15 and 1000", + }, { + name: "above the maximum sample size", + args: []string{"--target", "a", "--max-samples", "1001"}, + want: "between 15 and 1000", + }, { + name: "a missing instruction file names the flag", + args: []string{"--target", "a", "--agent-instruction-file", + filepath.Join(dir, "absent.md")}, + want: "--agent-instruction-file", + }, { + name: "generating needs a model deployment", + args: []string{"--target", "a", "--agent-instruction", "inline"}, + want: "--generation-model", + }} + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + r := requireFailure(t, runIn(t, dir, append([]string{"generate"}, tc.args...)...)) + require.Contains(t, r.Combined(), tc.want) + }) + } +} + +// TestCLIGenerateNoPromptNamesWhatIsMissing is the CI case: with no target and +// nothing to prompt with, the process has to end saying which flag to pass. +func TestCLIGenerateNoPromptNamesWhatIsMissing(t *testing.T) { + r := requireFailure(t, runIn(t, t.TempDir(), "generate", "--no-prompt")) + require.Contains(t, r.Combined(), "--target is required") + require.Contains(t, r.Combined(), "--no-prompt", + "the message must say why it could not be resolved") +} + +// TestCLIGenerateReadsTheSpec proves the config file is loaded and validated +// rather than only the flags. +// +// The strategy is the clearest evidence: `from-traces` is a value the spec +// accepts syntactically and the generation API cannot honour, so the refusal +// can only come from having parsed the file. +func TestCLIGenerateReadsTheSpec(t *testing.T) { + dir := t.TempDir() + spec := filepath.Join(dir, "gen.yaml") + require.NoError(t, os.WriteFile(spec, []byte(` +agent: + name: from-spec +generate: + dataset: + name: spec-dataset + strategy: from-traces +`), 0o600)) + + r := requireFailure(t, runIn(t, dir, "generate", "--config", spec)) + require.Contains(t, r.Combined(), "from-traces") + require.Contains(t, r.Combined(), "agent.context.traces.window", + "the refusal must point at the field that does seed generation from traces") +} + +// TestCLIGenerateSkipsWhatWasSupplied is the one generate test that reaches the +// service, and it is here because the skip decision is made in the command +// body rather than in the config resolver. +// +// With both artifacts supplied there is nothing left to generate, so the whole +// command runs without submitting a job — which is what makes it affordable to +// assert on. A regression that stopped honouring either flag would show up as +// a generation job starting instead of this returning. +func TestCLIGenerateSkipsWhatWasSupplied(t *testing.T) { + dir := t.TempDir() + + r := requireSuccess(t, runIn(t, dir, "generate", + "--target", "azd-eval-probe-agent", + "--agent-instruction", "answer questions about orders", + "--evaluator", "already-published", + "--dataset", "already-registered")) + + require.Contains(t, r.Stdout, "skipping rubric generation") + require.Contains(t, r.Stdout, "skipping data generation") + require.Contains(t, r.Stdout, "Nothing was generated.") + + // The deployment spec is only rewritten when something was produced, and + // writing an empty reference into it would be worse than not writing. + entries, err := os.ReadDir(dir) + require.NoError(t, err) + require.Empty(t, entries, "a generate that produced nothing must write nothing") +} + +// TestCLIGenerateSuppressionIsPerArtifact pins the two flags apart: neither +// may suppress the artifact it does not name. +// +// Each case supplies one artifact and leaves the other to be generated, and is +// stopped at the model check that precedes submission. Reaching that error is +// the proof: it is only raised when something is still going to be generated, +// so it says the unsupplied artifact survived the other flag. +func TestCLIGenerateSuppressionIsPerArtifact(t *testing.T) { + cases := []struct { + name string + supplied []string + survives string + }{ + {"a supplied evaluator leaves the dataset", []string{"--evaluator", "already-published"}, "dataset"}, + {"a supplied dataset leaves the rubric", []string{"--dataset", "already-registered"}, "rubric"}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + args := append([]string{"generate", + "--target", "azd-eval-probe-agent", + "--agent-instruction", "answer questions about orders"}, tc.supplied...) + + r := requireFailure(t, runIn(t, t.TempDir(), args...)) + require.Contains(t, r.Combined(), "--generation-model", + "the %s was suppressed by a flag that does not name it", tc.survives) + require.NotContains(t, strings.ToLower(r.Combined()), "generating ", + "the run must stop at the model check, before any job is submitted") + }) + } +} diff --git a/cli/azd/extensions/azure.ai.evaluations/tests/cli/harness_test.go b/cli/azd/extensions/azure.ai.evaluations/tests/cli/harness_test.go index e4e86181a1d..dec143ce139 100644 --- a/cli/azd/extensions/azure.ai.evaluations/tests/cli/harness_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/tests/cli/harness_test.go @@ -68,6 +68,9 @@ func TestMain(m *testing.M) { } code := m.Run() + // The shared eval outlives any single test, so it cannot be released with + // t.Cleanup without taking it away from the tests that run after. + runTeardown() os.RemoveAll(dir) os.Exit(code) } diff --git a/cli/azd/extensions/azure.ai.evaluations/tests/cli/results_test.go b/cli/azd/extensions/azure.ai.evaluations/tests/cli/results_test.go new file mode 100644 index 00000000000..8660b5eda26 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/tests/cli/results_test.go @@ -0,0 +1,317 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +//go:build live + +package cli + +import ( + "encoding/csv" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +// resultsPayload is what `results show -o json` emits: the run and the rows. +type resultsPayload struct { + Run struct { + ID string `json:"id"` + Status string `json:"status"` + ResultCounts struct { + Total int `json:"total"` + Passed int `json:"passed"` + Failed int `json:"failed"` + Errored int `json:"errored"` + } `json:"result_counts"` + PerTestingCriteria []struct { + TestingCriteria string `json:"testing_criteria"` + Passed int `json:"passed"` + Failed int `json:"failed"` + } `json:"per_testing_criteria_results"` + } `json:"run"` + OutputItems []struct { + ID string `json:"id"` + Status string `json:"status"` + DataSourceItem map[string]any `json:"datasource_item"` + Results []struct { + Name string `json:"name"` + Score *float64 `json:"score"` + Passed bool `json:"passed"` + } `json:"results"` + } `json:"output_items"` +} + +// TestCLIResultsShowRendersTheRows is the difference between `results show` and +// `run show`: the totals say how many failed, these say which. +// +// The fixture's baseline run scores one row and fails two, so the rendering is +// checked against known verdicts rather than against whatever came back. +func TestCLIResultsShowRendersTheRows(t *testing.T) { + f := sharedEval(t) + + r := requireSuccess(t, run(t, "results", "show", f.EvalID, "--run-id", f.BaselineRunID)) + + require.Contains(t, r.Stdout, f.BaselineRunID) + require.Contains(t, r.Stdout, "Totals: 1 passed, 2 failed, 0 errored") + require.Contains(t, r.Stdout, "CRITERION") + require.Contains(t, r.Stdout, "ITEM") + require.Contains(t, r.Stdout, "EVALUATOR") + require.Contains(t, r.Stdout, "SCORE") + require.Contains(t, r.Stdout, f.EvaluatorName) + + // Both verdicts, and the row's own input: a table that showed only the + // counts would satisfy every assertion above. + require.Contains(t, r.Stdout, "FAIL") + require.Contains(t, r.Stdout, "pass") + require.Contains(t, r.Stdout, "response=a good answer") + require.Contains(t, r.Stdout, "response=a bad answer") + require.Contains(t, r.Stdout, "Report:") +} + +func TestCLIResultsShowJSON(t *testing.T) { + f := sharedEval(t) + + r := requireSuccess(t, run(t, "results", "show", f.EvalID, + "--run-id", f.BaselineRunID, "-o", "json")) + + var payload resultsPayload + r.JSON(t, &payload) + + require.Equal(t, f.BaselineRunID, payload.Run.ID) + require.Equal(t, "completed", payload.Run.Status) + require.Equal(t, 3, payload.Run.ResultCounts.Total) + require.Equal(t, 1, payload.Run.ResultCounts.Passed) + require.Equal(t, 2, payload.Run.ResultCounts.Failed) + require.Zero(t, payload.Run.ResultCounts.Errored) + + require.Len(t, payload.Run.PerTestingCriteria, 1) + require.Equal(t, f.EvaluatorName, payload.Run.PerTestingCriteria[0].TestingCriteria) + + // The rows are the reason this command exists, and a run reporting counts + // while returning none would still satisfy everything above. + require.Len(t, payload.OutputItems, 3, "every dataset row must come back as an item") + + passed := 0 + for _, item := range payload.OutputItems { + require.NotEmpty(t, item.DataSourceItem["response"], + "each row must carry the column it was evaluated on") + require.Len(t, item.Results, 1) + require.Equal(t, f.EvaluatorName, item.Results[0].Name) + require.NotNil(t, item.Results[0].Score, "a scored row must report its score") + if item.Results[0].Passed { + passed++ + require.Equal(t, 1.0, *item.Results[0].Score) + } else { + require.Equal(t, 0.0, *item.Results[0].Score) + } + } + require.Equal(t, 1, passed, "the per-row verdicts must agree with the totals") +} + +// TestCLIResultsShowFailedOnly asserts the filter removes rows rather than +// merely relabelling them. +func TestCLIResultsShowFailedOnly(t *testing.T) { + f := sharedEval(t) + + r := requireSuccess(t, run(t, "results", "show", f.EvalID, + "--run-id", f.BaselineRunID, "--failed-only")) + + require.Contains(t, r.Stdout, "response=a bad answer") + require.NotContains(t, r.Stdout, "response=a good answer", + "--failed-only must drop the rows that passed") + + // The passing run has nothing to show, and saying so is not the same as + // printing an empty table. + empty := requireSuccess(t, run(t, "results", "show", f.EvalID, + "--run-id", f.TreatmentRunID, "--failed-only")) + require.Contains(t, empty.Stdout, "No failing rows.") +} + +func TestCLIResultsExport(t *testing.T) { + f := sharedEval(t) + + t.Run("json to stdout", func(t *testing.T) { + r := requireSuccess(t, run(t, "results", "export", f.EvalID, + "--run-id", f.BaselineRunID, "--format", "json")) + + var exported struct { + ID string `json:"id"` + Status string `json:"status"` + ResultCounts struct { + Passed int `json:"passed"` + Failed int `json:"failed"` + } `json:"result_counts"` + } + r.JSON(t, &exported) + require.Equal(t, f.BaselineRunID, exported.ID) + require.Equal(t, "completed", exported.Status) + require.Equal(t, 1, exported.ResultCounts.Passed) + require.Equal(t, 2, exported.ResultCounts.Failed) + }) + + t.Run("csv to stdout", func(t *testing.T) { + r := requireSuccess(t, run(t, "results", "export", f.EvalID, + "--run-id", f.BaselineRunID, "--format", "csv")) + + rows, err := csv.NewReader(strings.NewReader(r.Stdout)).ReadAll() + require.NoError(t, err, "--format csv must emit parseable CSV:\n%s", r.Stdout) + require.Len(t, rows, 2, "a header and one row per criterion") + require.Equal(t, + []string{"run_id", "status", "criterion", "passed", "failed"}, rows[0]) + require.Equal(t, + []string{f.BaselineRunID, "completed", f.EvaluatorName, "1", "2"}, rows[1]) + }) + + t.Run("out-file writes the path instead of stdout", func(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "results.csv") + + r := requireSuccess(t, runIn(t, dir, "results", "export", f.EvalID, + "--run-id", f.BaselineRunID, "--format", "csv", "-O", path)) + require.Empty(t, strings.TrimSpace(r.Stdout), + "-O redirects the payload; leaving it on stdout too would double it") + + body, err := os.ReadFile(path) + require.NoError(t, err) + require.Contains(t, string(body), "run_id,status,criterion,passed,failed") + require.Contains(t, string(body), f.BaselineRunID) + }) + + t.Run("an unknown format is refused", func(t *testing.T) { + r := requireFailure(t, run(t, "results", "export", f.EvalID, + "--run-id", f.BaselineRunID, "--format", "xml")) + require.Contains(t, r.Combined(), "json or csv") + }) +} + +// comparison is the shape `results compare -o json` emits. +type comparison struct { + State string `json:"state"` + Request struct { + EvalID string `json:"evalId"` + BaselineRunID string `json:"baselineRunId"` + TreatmentRunIDs []string `json:"treatmentRunIds"` + } `json:"request"` + Result struct { + Method string `json:"method"` + Comparisons []struct { + TestingCriteria string `json:"testingCriteria"` + Metric string `json:"metric"` + BaselineRunSummary struct { + RunID string `json:"runId"` + SampleCount int `json:"sampleCount"` + Average float64 `json:"average"` + } `json:"baselineRunSummary"` + CompareItems []struct { + TreatmentRunSummary struct { + RunID string `json:"runId"` + SampleCount int `json:"sampleCount"` + Average float64 `json:"average"` + } `json:"treatmentRunSummary"` + DeltaEstimate float64 `json:"deltaEstimate"` + TreatmentEffect string `json:"treatmentEffect"` + } `json:"compareItems"` + } `json:"comparisons"` + } `json:"result"` +} + +// TestCLIResultsCompare needs two completed runs of the same eval that scored +// differently, which is why the fixture seeds one run to fail two of three +// rows: comparing two identical runs reports a zero delta, and a comparison +// that computed nothing would look the same. +func TestCLIResultsCompare(t *testing.T) { + f := sharedEval(t) + + t.Run("rendered columns", func(t *testing.T) { + r := requireSuccess(t, run(t, "results", "compare", f.EvalID, + "--baseline", f.BaselineRunID, "--treatment", f.TreatmentRunID)) + + for _, header := range []string{ + "METRIC", "TREATMENT RUN", "BASELINE", "TREATMENT", "DELTA", "P-VALUE", "EFFECT", + } { + require.Containsf(t, r.Stdout, header, "the comparison table lost its %s column", header) + } + require.Contains(t, r.Stdout, "Method:") + require.Contains(t, r.Stdout, f.TreatmentRunID) + require.Contains(t, r.Stdout, f.EvaluatorName) + + // One in three against three in three. The delta is signed, which is + // the whole point of naming a baseline. + require.Contains(t, r.Stdout, "0.333") + require.Contains(t, r.Stdout, "1.000") + require.Contains(t, r.Stdout, "+0.667") + }) + + t.Run("json shape", func(t *testing.T) { + r := requireSuccess(t, run(t, "results", "compare", f.EvalID, + "--baseline", f.BaselineRunID, "--treatment", f.TreatmentRunID, "-o", "json")) + + var got comparison + r.JSON(t, &got) + + require.Equal(t, "Succeeded", got.State) + require.Equal(t, f.EvalID, got.Request.EvalID) + require.Equal(t, f.BaselineRunID, got.Request.BaselineRunID) + require.Equal(t, []string{f.TreatmentRunID}, got.Request.TreatmentRunIDs) + + require.NotEmpty(t, got.Result.Method) + require.Len(t, got.Result.Comparisons, 1) + c := got.Result.Comparisons[0] + require.Equal(t, f.EvaluatorName, c.TestingCriteria) + require.Equal(t, f.BaselineRunID, c.BaselineRunSummary.RunID) + require.Equal(t, 3, c.BaselineRunSummary.SampleCount) + require.InDelta(t, 1.0/3.0, c.BaselineRunSummary.Average, 0.001) + + require.Len(t, c.CompareItems, 1) + item := c.CompareItems[0] + require.Equal(t, f.TreatmentRunID, item.TreatmentRunSummary.RunID) + require.Equal(t, 1.0, item.TreatmentRunSummary.Average) + require.InDelta(t, 2.0/3.0, item.DeltaEstimate, 0.001) + require.NotEmpty(t, item.TreatmentEffect, + "the effect classifies the result, including when there are too few samples") + }) + + // Naming neither run is the common case — "did my last change help?" — so + // the defaults are asserted against the listing rather than against the + // fixture's own ids, which is what the command itself resolves from. + t.Run("defaults to the two most recent completed runs", func(t *testing.T) { + listed := requireSuccess(t, run(t, "run", "list", f.EvalID, "-o", "json")) + var runs []struct { + ID string `json:"id"` + Status string `json:"status"` + } + listed.JSON(t, &runs) + + completed := make([]string, 0, len(runs)) + for _, r := range runs { + if r.Status == "completed" { + completed = append(completed, r.ID) + } + } + require.GreaterOrEqual(t, len(completed), 2, + "comparing needs two completed runs of the same eval") + + r := requireSuccess(t, run(t, "results", "compare", f.EvalID, "-o", "json")) + var got comparison + r.JSON(t, &got) + + require.Equal(t, completed[0], got.Request.TreatmentRunIDs[0], + "the treatment defaults to the most recent completed run") + require.Equal(t, completed[1], got.Request.BaselineRunID, + "the baseline defaults to the one before it") + require.Equal(t, "Succeeded", got.State) + }) +} + +// TestCLIResultsUnknownEvalIsBrief covers the failure a user hits by typo. The +// service answers with a long JSON body; printing it verbatim buries the one +// useful sentence. +func TestCLIResultsUnknownEvalIsBrief(t *testing.T) { + r := requireFailure(t, run(t, "results", "show", "eval_azdcli_does_not_exist")) + require.Less(t, len(r.Combined()), 600, + "a not-found must stay short, not dump the service body:\n%s", r.Combined()) + require.Contains(t, r.Combined(), "eval_azdcli_does_not_exist") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/tests/cli/rubric_test.go b/cli/azd/extensions/azure.ai.evaluations/tests/cli/rubric_test.go new file mode 100644 index 00000000000..ebe5b49bd24 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/tests/cli/rubric_test.go @@ -0,0 +1,203 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +//go:build live + +package cli + +import ( + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +// A rubric is the other kind of evaluator: a JSON file of weighted dimensions, +// graded by a judge model rather than by code. It shares nothing with the code +// path on the wire beyond the route, so publishing one had never been +// exercised against a real project. + +// writeRubric lays down a rubric file and returns its path. +func writeRubric(t *testing.T, dimensions string) string { + t.Helper() + path := filepath.Join(t.TempDir(), "rubric.json") + require.NoError(t, os.WriteFile(path, + []byte(`{"dimensions":`+dimensions+`}`), 0o600)) + return path +} + +// evaluatorDocument is what `evaluator show` prints. +type evaluatorDocument struct { + Name string `json:"name"` + Version string `json:"version"` + EvaluatorType string `json:"evaluator_type"` + Definition struct { + Type string `json:"type"` + Dimensions []struct { + ID string `json:"id"` + Description string `json:"description"` + Weight int `json:"weight"` + } `json:"dimensions"` + DataSchema map[string]any `json:"data_schema"` + InitParameters map[string]any `json:"init_parameters"` + Metrics map[string]any `json:"metrics"` + } `json:"definition"` + SupportedEvaluationLevels []string `json:"supported_evaluation_levels"` +} + +// TestCLIRubricRoundTrip publishes a rubric, reads it back, and republishes it. +func TestCLIRubricRoundTrip(t *testing.T) { + name := uniqueName("azdcli_rubric") + rubric := writeRubric(t, `[ + {"id":"tone","description":"Is the answer polite?","weight":5}, + {"id":"accuracy","description":"Is the answer correct?","weight":10} + ]`) + + created := requireSuccess(t, run(t, "evaluator", "create", "--name", name, "--rubric", rubric)) + require.Contains(t, created.Stdout, "version 1") + t.Cleanup(func() { + run(t, "evaluator", "delete", "--name", name, "--version", "1") + }) + + shown := requireSuccess(t, run(t, "evaluator", "show", "--name", name)) + var doc evaluatorDocument + shown.JSON(t, &doc) + + require.Equal(t, name, doc.Name) + require.Equal(t, "1", doc.Version) + require.Equal(t, "custom", doc.EvaluatorType) + require.Equal(t, "rubric", doc.Definition.Type, + "the discriminator is what tells the service which definition kind it holds") + + require.Len(t, doc.Definition.Dimensions, 2) + byID := map[string]int{} + for _, d := range doc.Definition.Dimensions { + byID[d.ID] = d.Weight + require.NotEmpty(t, d.Description, "a dimension's description is what the judge grades against") + } + require.Equal(t, 5, byID["tone"]) + require.Equal(t, 10, byID["accuracy"]) + + // The rubric named only dimensions. Everything else is filled in by the + // service, and a caller reading the definition back gets those defaults + // rather than what was sent — including the judge model the evaluator will + // require at run time. + require.NotEmpty(t, doc.Definition.DataSchema, + "the service supplies a rubric's data schema; the author never writes one") + require.NotEmpty(t, doc.Definition.InitParameters) + require.NotEmpty(t, doc.Definition.Metrics) + require.NotEmpty(t, doc.SupportedEvaluationLevels) + + // Every registration publishes a new immutable version, which is why there + // is no separate update command. + republished := requireSuccess(t, run(t, "evaluator", "create", "--name", name, "--rubric", rubric)) + require.Contains(t, republished.Stdout, "version 2", + "re-registering must advance the version rather than overwrite") + t.Cleanup(func() { + run(t, "evaluator", "delete", "--name", name, "--version", "2") + }) + + // The earlier version stays reachable, which is what makes a published + // version safe to reference from a config. + pinned := requireSuccess(t, run(t, "evaluator", "show", "--name", name, "--version", "1")) + var first evaluatorDocument + pinned.JSON(t, &first) + require.Equal(t, "1", first.Version) +} + +// TestCLIRubricWeightMustBeAnIntegerFromOneToTen covers the validation a +// hand-authored rubric is most likely to trip. +// +// The service runs two separate checks and they answer differently: a +// fractional weight is rejected for not being an integer, an out-of-range one +// for being out of range. Both are asserted because a caller only ever sees +// one of them, and both have to say what a legal weight is. +func TestCLIRubricWeightMustBeAnIntegerFromOneToTen(t *testing.T) { + cases := []struct { + name string + weight string + }{ + {"fractional", "2.5"}, + {"zero", "0"}, + {"above ten", "11"}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + rubric := writeRubric(t, + `[{"id":"tone","description":"Is the answer polite?","weight":`+tc.weight+`}]`) + + r := requireFailure(t, run(t, "evaluator", "create", + "--name", uniqueName("azdcli_badweight"), "--rubric", rubric)) + require.Contains(t, r.Combined(), "between 1 and 10", + "the refusal must say what a legal weight is") + }) + } + + // A weight the service accepts, so the cases above are failing on the + // weight rather than on the rubric shape they share. + name := uniqueName("azdcli_goodweight") + ok := writeRubric(t, `[{"id":"tone","description":"Is the answer polite?","weight":1}]`) + requireSuccess(t, run(t, "evaluator", "create", "--name", name, "--rubric", ok)) + t.Cleanup(func() { + run(t, "evaluator", "delete", "--name", name, "--version", "1") + }) +} + +// TestCLIRubricNeedsDimensions covers the local check, which costs nothing and +// names the field the service would not. +func TestCLIRubricNeedsDimensions(t *testing.T) { + path := filepath.Join(t.TempDir(), "rubric.json") + require.NoError(t, os.WriteFile(path, []byte(`{"criteria":[]}`), 0o600)) + + r := requireFailure(t, run(t, "evaluator", "create", + "--name", uniqueName("azdcli_nodims"), "--rubric", path)) + require.Contains(t, r.Combined(), "dimensions") +} + +// TestCLIRubricRejectsCodeOnlyFlags asserts the settings that would be +// accepted and then dropped are refused instead. A rubric runs no code and its +// schemas come from the service, so carrying them would be a silent no-op. +func TestCLIRubricRejectsCodeOnlyFlags(t *testing.T) { + rubric := writeRubric(t, `[{"id":"tone","description":"polite","weight":5}]`) + schema := filepath.Join(t.TempDir(), "schema.json") + require.NoError(t, os.WriteFile(schema, []byte(`{"type":"object"}`), 0o600)) + + r := requireFailure(t, run(t, "evaluator", "create", + "--name", uniqueName("azdcli_rubricflags"), "--rubric", rubric, + "--data-schema", schema)) + require.Contains(t, r.Combined(), "--data-schema") + require.Contains(t, r.Combined(), "--file") +} + +// TestCLIEvaluatorShowAcceptsAFullDocument proves `evaluator show` emits JSON a +// script can consume, whatever the definition kind. It renders the service's +// body rather than a typed struct, so nothing else pins that it stays parseable. +func TestCLIEvaluatorShowAcceptsAFullDocument(t *testing.T) { + name := uniqueName("azdcli_rubricdoc") + + // The wrapped form: a whole evaluator document rather than a bare + // definition. Both are accepted, and generated rubrics arrive wrapped. + path := filepath.Join(t.TempDir(), "rubric.json") + require.NoError(t, os.WriteFile(path, []byte( + `{"name":"ignored","definition":{"dimensions":[{"id":"tone","description":"polite","weight":3}]}}`, + ), 0o600)) + + requireSuccess(t, run(t, "evaluator", "create", "--name", name, "--rubric", path)) + t.Cleanup(func() { + run(t, "evaluator", "delete", "--name", name, "--version", "1") + }) + + shown := requireSuccess(t, run(t, "evaluator", "show", "--name", name)) + var raw map[string]any + require.NoError(t, json.Unmarshal([]byte(shown.Stdout), &raw), + "evaluator show must emit parseable JSON:\n%s", shown.Stdout) + + // The flag names the evaluator, so a name inside the file must not win. + require.Equal(t, name, raw["name"], + "--name must decide the evaluator's name, not the document's own field") + require.NotContains(t, strings.ToLower(shown.Stdout), `"name": "ignored"`) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/tests/cli/run_ops_test.go b/cli/azd/extensions/azure.ai.evaluations/tests/cli/run_ops_test.go new file mode 100644 index 00000000000..77628b90270 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/tests/cli/run_ops_test.go @@ -0,0 +1,208 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +//go:build live + +package cli + +import ( + "context" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +type runSummary struct { + ID string `json:"id"` + Name string `json:"name"` + Status string `json:"status"` + ResultCounts *struct { + Passed int `json:"passed"` + Failed int `json:"failed"` + Errored int `json:"errored"` + } `json:"result_counts"` +} + +func TestCLIRunList(t *testing.T) { + f := sharedEval(t) + + t.Run("table", func(t *testing.T) { + r := requireSuccess(t, run(t, "run", "list", f.EvalID)) + for _, header := range []string{"RUN ID", "NAME", "STATUS", "RESULTS"} { + require.Containsf(t, r.Stdout, header, "the listing lost its %s column", header) + } + require.Contains(t, r.Stdout, f.BaselineRunID) + require.Contains(t, r.Stdout, f.TreatmentRunID) + require.Contains(t, r.Stdout, "1 passed, 2 failed, 0 errored", + "the listing must summarise each run's counts, not just its status") + }) + + t.Run("json", func(t *testing.T) { + r := requireSuccess(t, run(t, "run", "list", f.EvalID, "-o", "json")) + require.True(t, strings.HasPrefix(strings.TrimSpace(r.Stdout), "["), + "a list must be a bare array, not the service's envelope") + + var runs []runSummary + r.JSON(t, &runs) + require.GreaterOrEqual(t, len(runs), 2) + + byID := map[string]runSummary{} + for _, entry := range runs { + byID[entry.ID] = entry + } + baseline, ok := byID[f.BaselineRunID] + require.True(t, ok, "the eval's own run is missing from its listing") + require.Equal(t, "completed", baseline.Status) + require.NotNil(t, baseline.ResultCounts) + require.Equal(t, 2, baseline.ResultCounts.Failed) + }) + + // The client has always taken a limit; until recently the command did not + // expose one, so a service-side truncation would have passed unnoticed. + t.Run("limit", func(t *testing.T) { + r := requireSuccess(t, run(t, "run", "list", f.EvalID, "--limit", "1", "-o", "json")) + var runs []runSummary + r.JSON(t, &runs) + require.Len(t, runs, 1, "--limit must reach the service") + }) + + t.Run("unknown eval is brief", func(t *testing.T) { + r := requireFailure(t, run(t, "run", "list", "eval_azdcli_no_such_eval")) + require.Less(t, len(r.Combined()), 600, + "a not-found must stay short, not dump the service body:\n%s", r.Combined()) + require.Contains(t, r.Combined(), "eval_azdcli_no_such_eval") + }) +} + +func TestCLIRunShow(t *testing.T) { + f := sharedEval(t) + + t.Run("by run id", func(t *testing.T) { + r := requireSuccess(t, run(t, "run", "show", f.EvalID, "--run-id", f.BaselineRunID)) + require.Contains(t, r.Stdout, f.BaselineRunID) + require.Contains(t, r.Stdout, "status") + require.Contains(t, r.Stdout, "completed") + require.Contains(t, r.Stdout, "1 passed, 2 failed, 0 errored") + require.Contains(t, r.Stdout, "report") + }) + + // Without --run-id the command has to pick one, and outside an azd + // environment there is no remembered id to fall back on, so what is + // exercised is the listing path. + t.Run("defaults to the most recent run", func(t *testing.T) { + listed := requireSuccess(t, run(t, "run", "list", f.EvalID, "--limit", "1", "-o", "json")) + var newest []runSummary + listed.JSON(t, &newest) + require.Len(t, newest, 1) + + r := requireSuccess(t, run(t, "run", "show", f.EvalID, "-o", "json")) + var shown runSummary + r.JSON(t, &shown) + require.Equal(t, newest[0].ID, shown.ID, + "the default must be the run the listing puts first") + }) + + // A remembered run that no longer resolves falls through to the eval's + // latest, but one named explicitly must not: silently showing a different + // run than the one asked for is worse than saying it is gone. + // + // Only the substitution is asserted. Unlike `run list` and `run delete`, + // this path does not shorten the service's body, so the message runs to + // about 1700 characters of raw JSON — recorded in the report rather than + // pinned here, since pinning it would make the length a requirement. + t.Run("an unknown run id is reported, not silently replaced", func(t *testing.T) { + r := requireFailure(t, run(t, "run", "show", f.EvalID, "--run-id", "evalrun_azdcli_nope")) + require.Contains(t, r.Combined(), "evalrun_azdcli_nope", + "the failure must name the run that was asked for") + require.NotContains(t, r.Combined(), f.BaselineRunID, + "an explicit --run-id must not fall back to another run") + }) +} + +// TestCLIRunCancelAndDelete covers both halves of cancel, and the delete that +// follows it, against a single in-flight run: each run costs a minute of +// service time, so the two happy paths share one. +// +// The service answers a cancel on a finished run with success, so without the +// guard the command would tell a user it had stopped something it had not. +func TestCLIRunCancelAndDelete(t *testing.T) { + f := sharedEval(t) + + t.Run("a finished run is refused", func(t *testing.T) { + r := requireFailure(t, run(t, "run", "cancel", f.EvalID, "--run-id", f.BaselineRunID)) + require.Contains(t, r.Combined(), "already finished") + require.Contains(t, r.Combined(), "completed") + }) + + // Delete is covered as far as the service honours it. + // + // The removal itself is not asserted, because it does not happen: the + // service accepts the DELETE and the run is still readable by id and still + // in the listing minutes later. What is asserted instead is that the + // command reaches the right resource — a real run is accepted, an unknown + // one is refused — which is the part that would break if the route or the + // id handling regressed. + t.Run("an in-flight run is cancelled, and the delete is accepted", func(t *testing.T) { + runID := startCancellableRun(t, f) + + cancelled := requireSuccess(t, run(t, "run", "cancel", f.EvalID, "--run-id", runID)) + require.Contains(t, cancelled.Stdout, runID) + require.Contains(t, cancelled.Stdout, "is now") + + shown := requireSuccess(t, run(t, "run", "show", f.EvalID, "--run-id", runID, "-o", "json")) + var after runSummary + shown.JSON(t, &after) + require.NotEqual(t, "completed", after.Status, + "a cancelled run must not go on to complete") + + deleted := requireSuccess(t, run(t, "run", "delete", f.EvalID, "--run-id", runID)) + require.Contains(t, deleted.Stdout, "Deleted run") + require.Contains(t, deleted.Stdout, runID) + + still := requireSuccess(t, run(t, "run", "show", f.EvalID, "--run-id", runID, "-o", "json")) + var survivor runSummary + still.JSON(t, &survivor) + t.Logf("the run is still readable after a successful delete (status %q); "+ + "the service accepts the request without removing anything", survivor.Status) + }) + + // Deleting is not undoable, so the id is required rather than defaulted to + // whichever run happens to be newest. + t.Run("delete requires the run id", func(t *testing.T) { + r := requireFailure(t, run(t, "run", "delete", f.EvalID)) + require.Contains(t, r.Combined(), "--run-id is required") + }) + + t.Run("deleting an unknown run is reported briefly", func(t *testing.T) { + r := requireFailure(t, run(t, "run", "delete", f.EvalID, "--run-id", "evalrun_azdcli_nope")) + require.Contains(t, r.Combined(), "evalrun_azdcli_nope") + require.Less(t, len(r.Combined()), 600, + "a not-found must stay short, not dump the service body:\n%s", r.Combined()) + }) +} + +// startCancellableRun adds a run to the fixture's eval and returns it before it +// can finish. +// +// The rows are padded so the run cannot complete inside the second it takes to +// issue the cancel; a run that finished first would turn the cancel test into +// an assertion about the guard it is not testing. +func startCancellableRun(t *testing.T, f *evalFixture) string { + t.Helper() + + client, err := liveClient() + require.NoError(t, err) + + responses := make([]string, 0, 40) + for i := range 40 { + responses = append(responses, strings.Repeat("a good answer ", i%5+1)) + } + + runID, err := startFixtureRun(context.Background(), client, f.EvalID, "cancelme", responses) + require.NoError(t, err, "starting a run to cancel") + t.Cleanup(func() { + _ = client.DeleteOpenAIEvalRun(context.Background(), f.EvalID, runID) + }) + return runID +} From d98d5c5106e19d88cb923fb1d03339f4801fb453 Mon Sep 17 00:00:00 2001 From: mohessie Date: Mon, 3 Aug 2026 03:12:40 +0300 Subject: [PATCH 072/320] Record the publish-to-reference consistency lag Eval creation resolves an evaluator through the version listing, which lags the direct read, so reconciliation has to gate on both or it fails just after a successful publish. --- .../azd-eval-extension-implementation.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/cli/azd/extensions/azure.ai.evaluations/azd-eval-extension-implementation.md b/cli/azd/extensions/azure.ai.evaluations/azd-eval-extension-implementation.md index 5fda30c6e18..b5373ffc545 100644 --- a/cli/azd/extensions/azure.ai.evaluations/azd-eval-extension-implementation.md +++ b/cli/azd/extensions/azure.ai.evaluations/azd-eval-extension-implementation.md @@ -248,6 +248,15 @@ func configureExtensionHost(host *azdext.ExtensionHost) { 4. **Eval groups** — `POST /openai/v1/evals` with `testing_criteria` from the resolved evaluator versions. Groups are immutable, so only recreate when the resolved versions or options actually changed. 5. Persist resolved ids, versions, and fingerprints to the azd env. +**Publish → reference is eventually consistent.** After an evaluator version is +published, the direct read `GET /evaluators/{name}/versions/{version}` goes +consistent in roughly 330 ms, but the version *listing* lags 650 ms–1.4 s — and +eval creation resolves the reference through the listing, not the direct read. +Step 4 has to gate on both, or it fails with *"The evaluator X was not found"* +just after successfully publishing that evaluator. It reads like a flake, +because retrying a second later succeeds, so the tempting fix is a blanket retry +rather than the wait. + **How azd reaches us:** `azd up` runs one DAG; per service it calls `GetServiceTarget()`, which does `serviceLocator.ResolveNamed(host, &target)`. If our extension is not installed, azd fails that service with *"install an extension that provides this host."* We implement **no sequencing or rollback across services** — `uses:` and the DAG handle that. --- From 5d6663aca59962ea9cd36c1a8b4b97259d567027 Mon Sep 17 00:00:00 2001 From: mohessie Date: Mon, 3 Aug 2026 04:06:57 +0300 Subject: [PATCH 073/320] Take code evaluators and comparison out of M1 Both are deferred, so they leave the branch rather than sit unused: code evaluator publishing, the grader loader and its validation, and results compare. LatestEvaluatorVersionNumber and the score formatter move rather than go, being the parts of those files M1 still uses. Evaluator create and update now take the name positionally and read the definition from --from-file, matching the sibling Foundry extensions. They are one request with one difference: create refuses a name already in use and update refuses one that is free, which the service does not check because it assigns the version either way. The CLI fixture is rebuilt on what M1 can actually run - a built-in evaluator against an agent - because a deterministic code grader over a target-less dataset was both of the deferred features at once. Pass and fail are now a judge's call, so the tests assert what holds regardless: every row comes back scored, and --failed-only returns a subset that agrees with the totals. --- .../internal/cmd/compare.go | 219 ----------- .../internal/cmd/compare_test.go | 83 ----- .../internal/cmd/evaluator.go | 330 ++++++----------- .../internal/cmd/evaluator_code_live_test.go | 246 ------------- .../internal/cmd/evaluator_code_test.go | 202 ---------- .../internal/cmd/model_target_test.go | 98 ----- .../internal/cmd/reconciler.go | 102 ----- .../internal/cmd/reconciler_digest_test.go | 81 ---- .../internal/cmd/results.go | 12 +- .../internal/cmd/run_ops_test.go | 2 +- .../internal/cmd/traces_test.go | 119 ------ .../internal/pkg/eval_api/code_evaluator.go | 178 --------- .../internal/pkg/eval_api/evaluators.go | 18 + .../internal/pkg/evalcore/codescript.go | 113 ------ .../internal/pkg/evalcore/codescript_test.go | 128 ------- .../tests/cli/evaluator_test.go | 73 +--- .../tests/cli/fixture_test.go | 251 +++++++------ .../tests/cli/harness_test.go | 13 - .../tests/cli/results_test.go | 235 ++++-------- .../tests/cli/rubric_test.go | 47 +-- .../tests/cli/run_ops_test.go | 40 +- .../tests/live/code_evaluator_run_test.go | 347 ------------------ 22 files changed, 365 insertions(+), 2572 deletions(-) delete mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/compare.go delete mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/compare_test.go delete mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator_code_live_test.go delete mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator_code_test.go delete mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/model_target_test.go delete mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler_digest_test.go delete mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/traces_test.go delete mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/code_evaluator.go delete mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/codescript.go delete mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/codescript_test.go delete mode 100644 cli/azd/extensions/azure.ai.evaluations/tests/live/code_evaluator_run_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/compare.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/compare.go deleted file mode 100644 index 3859dc3664f..00000000000 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/compare.go +++ /dev/null @@ -1,219 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package cmd - -import ( - "context" - "fmt" - "time" - - "azureaieval/internal/pkg/eval_api" - - "github.com/spf13/cobra" -) - -// comparePollBudget bounds the wait for a comparison. The probe returned in -// about a second, so this is generous headroom rather than an expected wait. -const ( - comparePollInterval = 3 * time.Second - comparePollAttempts = 100 -) - -func newResultsCompareCommand() *cobra.Command { - var ( - baseline string - treatments []string - displayName string - endpointFlg string - groupName string - ) - - cmd := &cobra.Command{ - Use: "compare [eval-id]", - Short: "Compare runs of an eval against a baseline.", - Args: cobra.MaximumNArgs(1), - RunE: func(cmd *cobra.Command, args []string) error { - ctx := cmd.Context() - out := cmd.OutOrStdout() - - ec, err := newEvalContext(ctx, endpointFlg) - if err != nil { - return err - } - defer ec.Close() - - evalID, err := resolveEvalID(cmd, ec, args, groupName) - if err != nil { - return err - } - - baseline, treatments, err = ec.resolveComparisonRuns(ctx, evalID, baseline, treatments) - if err != nil { - return err - } - - if displayName == "" { - displayName = fmt.Sprintf("compare-%s", time.Now().UTC().Format("20060102-150405")) - } - - insight, err := ec.evalClient.CreateInsight(ctx, &eval_api.CreateInsightRequest{ - DisplayName: displayName, - Request: &eval_api.InsightRequest{ - Type: eval_api.InsightTypeEvaluationComparison, - EvalID: evalID, - BaselineRunID: baseline, - TreatmentRunIDs: treatments, - }, - }, ProjectEndpointAPIVersion) - if err != nil { - return fmt.Errorf("starting the comparison: %w", err) - } - - if !isJSON(cmd) { - fmt.Fprintf(out, "Comparing %d run(s) against %s...\n", len(treatments), baseline) - } - - completed, err := ec.pollInsight(ctx, insight.ID) - if err != nil { - return err - } - if isJSON(cmd) { - return emitJSON(out, completed) - } - return renderComparison(out, completed) - }, - } - - cmd.Flags().StringVar(&baseline, "baseline", "", - "Run to compare against. Defaults to the second most recent completed run.") - cmd.Flags().StringArrayVar(&treatments, "treatment", nil, - "Run to measure, repeatable. Defaults to the most recent completed run.") - cmd.Flags().StringVar(&displayName, "name", "", "Name for this comparison.") - addEvalFlags(cmd, &groupName) - cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") - return cmd -} - -// resolveComparisonRuns fills in whichever runs were not named. -// -// Comparing the two most recent completed runs is what "did my change help?" -// means most of the time, so neither flag is required. -func (ec *evalContext) resolveComparisonRuns( - ctx context.Context, - evalID, baseline string, - treatments []string, -) (string, []string, error) { - if baseline != "" && len(treatments) > 0 { - return baseline, treatments, nil - } - - list, err := ec.evalClient.ListOpenAIEvalRuns(ctx, evalID, 0) - if err != nil { - return "", nil, fmt.Errorf("listing runs of eval %s: %w", evalID, err) - } - - completed := make([]string, 0, 2) - if list == nil { - return "", nil, fmt.Errorf("eval %s has no runs", evalID) - } - for _, run := range list.Data { - if run.Status == "completed" { - completed = append(completed, run.ID) - } - } - - if len(treatments) == 0 { - if len(completed) == 0 { - return "", nil, fmt.Errorf( - "eval %s has no completed runs to compare", evalID) - } - treatments = []string{completed[0]} - } - if baseline == "" { - if len(completed) < 2 { - return "", nil, fmt.Errorf( - "eval %s has only one completed run, so there is nothing to compare it "+ - "against; run it again, or name a baseline with --baseline", - evalID) - } - baseline = completed[1] - } - return baseline, treatments, nil -} - -// pollInsight waits for the comparison to reach a terminal state. -func (ec *evalContext) pollInsight(ctx context.Context, insightID string) (*eval_api.Insight, error) { - for attempt := 0; attempt < comparePollAttempts; attempt++ { - select { - case <-ctx.Done(): - return nil, ctx.Err() - case <-time.After(comparePollInterval): - } - - insight, err := ec.evalClient.GetInsight(ctx, insightID, ProjectEndpointAPIVersion) - if err != nil { - return nil, fmt.Errorf("reading comparison %s: %w", insightID, err) - } - if !insight.Terminal() { - continue - } - if !insight.Succeeded() { - return nil, fmt.Errorf("comparison %s finished with state %q", insightID, insight.State) - } - return insight, nil - } - return nil, fmt.Errorf("comparison %s did not finish in time", insightID) -} - -// renderComparison prints one row per criterion per treatment run. -// formatStat renders a statistic, showing an undefined one as a dash. A -// standard deviation over a single sample has no value, and printing the -// literal "NaN" in a results table reads like a failure rather than the -// arithmetic it is. -func formatStat(verb string, v eval_api.LenientFloat) string { - if !v.Defined() { - return "-" - } - return fmt.Sprintf(verb, float64(v)) -} - -func renderComparison(w interface{ Write([]byte) (int, error) }, insight *eval_api.Insight) error { - if insight.Result == nil || len(insight.Result.Comparisons) == 0 { - fmt.Fprintln(w, "The comparison produced no metrics.") - return nil - } - - if insight.Result.Method != "" { - fmt.Fprintf(w, "Method: %s\n\n", insight.Result.Method) - } - - rows := [][]string{} - for _, c := range insight.Result.Comparisons { - baseAvg := "-" - if c.BaselineRunSummary != nil { - baseAvg = formatStat("%.3f", c.BaselineRunSummary.Average) - } - for _, item := range c.CompareItems { - treatAvg := "-" - runID := "-" - if item.TreatmentRunSummary != nil { - treatAvg = formatStat("%.3f", item.TreatmentRunSummary.Average) - runID = item.TreatmentRunSummary.RunID - } - rows = append(rows, []string{ - c.Metric, - runID, - baseAvg, - treatAvg, - formatStat("%+.3f", item.DeltaEstimate), - formatStat("%.3f", item.PValue), - item.TreatmentEffect, - }) - } - } - - return emitTable(w, - []string{"METRIC", "TREATMENT RUN", "BASELINE", "TREATMENT", "DELTA", "P-VALUE", "EFFECT"}, - rows) -} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/compare_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/compare_test.go deleted file mode 100644 index 5bbf0ad2370..00000000000 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/compare_test.go +++ /dev/null @@ -1,83 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package cmd - -import ( - "bytes" - "testing" - - "azureaieval/internal/pkg/eval_api" - - "github.com/stretchr/testify/require" -) - -func TestInsightTerminalStates(t *testing.T) { - for _, state := range []string{"", "NotStarted", "Running", "InProgress", "Queued"} { - require.False(t, (&eval_api.Insight{State: state}).Terminal(), "%q is not terminal", state) - } - for _, state := range []string{"Succeeded", "Failed", "Cancelled"} { - require.True(t, (&eval_api.Insight{State: state}).Terminal(), "%q is terminal", state) - } - require.True(t, (&eval_api.Insight{State: "Succeeded"}).Succeeded()) - require.False(t, (&eval_api.Insight{State: "Failed"}).Succeeded()) -} - -// The rendered table is how a reader decides whether a change helped, so the -// delta carries its sign and the effect classification is not dropped. -func TestRenderComparisonShowsSignedDeltaAndEffect(t *testing.T) { - insight := &eval_api.Insight{ - State: "Succeeded", - Result: &eval_api.InsightResult{ - Method: "PairedTTest", - Comparisons: []eval_api.MetricComparison{{ - Metric: "task_adherence", - BaselineRunSummary: &eval_api.RunSummary{RunID: "base", Average: 0.75}, - CompareItems: []eval_api.CompareItem{{ - TreatmentRunSummary: &eval_api.RunSummary{RunID: "treat", Average: 0.5}, - DeltaEstimate: -0.25, - PValue: 0.39, - TreatmentEffect: "TooFewSamples", - }}, - }}, - }, - } - - var buf bytes.Buffer - require.NoError(t, renderComparison(&buf, insight)) - out := buf.String() - - require.Contains(t, out, "PairedTTest", "the statistical method qualifies the result") - require.Contains(t, out, "task_adherence") - require.Contains(t, out, "-0.250", "a regression must read as negative") - require.Contains(t, out, "0.390") - require.Contains(t, out, "TooFewSamples", - "an inconclusive result must not look like a finding") -} - -// A positive delta reads as an improvement rather than an unsigned number. -func TestRenderComparisonSignsImprovements(t *testing.T) { - insight := &eval_api.Insight{ - State: "Succeeded", - Result: &eval_api.InsightResult{ - Comparisons: []eval_api.MetricComparison{{ - Metric: "similarity", - BaselineRunSummary: &eval_api.RunSummary{Average: 0.5}, - CompareItems: []eval_api.CompareItem{{ - TreatmentRunSummary: &eval_api.RunSummary{RunID: "t", Average: 0.8}, - DeltaEstimate: 0.3, - }}, - }}, - }, - } - - var buf bytes.Buffer - require.NoError(t, renderComparison(&buf, insight)) - require.Contains(t, buf.String(), "+0.300") -} - -func TestRenderComparisonHandlesEmptyResult(t *testing.T) { - var buf bytes.Buffer - require.NoError(t, renderComparison(&buf, &eval_api.Insight{State: "Succeeded"})) - require.Contains(t, buf.String(), "no metrics") -} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator.go index 151bc14bf42..72f431536ed 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator.go @@ -9,7 +9,6 @@ import ( "os" "azureaieval/internal/pkg/eval_api" - "azureaieval/internal/pkg/evalcore" "github.com/spf13/cobra" ) @@ -21,83 +20,74 @@ func newEvaluatorCommand() *cobra.Command { } cmd.AddCommand( newEvaluatorCreateCommand(), + newEvaluatorUpdateCommand(), newEvaluatorListCommand(), newEvaluatorShowCommand(), newEvaluatorDeleteCommand(), + newEvaluatorVersionsCommand(), ) return cmd } -// newEvaluatorCreateCommand builds `evaluator create`, named to match -// `dataset create`: both register an artifact and both publish a new immutable -// version every time, so there is nothing for a separate `update` to do. -// -// An evaluator is either a rubric — a JSON file of weighted dimensions — or -// code — one self-contained Python script. They are different definition types -// on the wire, so exactly one of the two sources has to be named. +// newEvaluatorCreateCommand builds `evaluator create `, which registers +// an evaluator that does not exist yet. func newEvaluatorCreateCommand() *cobra.Command { + return newEvaluatorWriteCommand("create", "Register an evaluator, publishing its first version.") +} + +// newEvaluatorUpdateCommand builds `evaluator update `, which publishes a +// further version of one that does. +func newEvaluatorUpdateCommand() *cobra.Command { + return newEvaluatorWriteCommand("update", "Publish a new version of an evaluator.") +} + +// newEvaluatorWriteCommand builds create and update, which send the same +// request and differ only in which starting state they accept. The service has +// one route for both and assigns the version either way, so the existence check +// is ours: without it, `create` on a name already in use would silently publish +// a further version of someone else's evaluator. +func newEvaluatorWriteCommand(verb, short string) *cobra.Command { var ( - name string - rubric string - file string - imageTag string - initParams string - dataSchema string - metrics string + fromFile string endpointFlg string ) - use := "create" - short := "Register a rubric or code evaluator, publishing a new version." - long := short + "\n\n" + - "A rubric (--rubric) is a JSON file of weighted dimensions.\n\n" + - "A code evaluator (--file) is a single Python script declaring a top-level\n" + - "grade(sample, item) function that returns a float. It runs as a python\n" + - "grader, which is handed the script's source and nothing else: there is no\n" + - "package and no import path, so a helper module beside the script cannot be\n" + - "imported. Dependencies come from the image named by --image-tag." - cmd := &cobra.Command{ - Use: use, + Use: verb + " ", Short: short, - Long: long, + Long: short + "\n\n" + + "An evaluator is a rubric: a JSON file of weighted scoring dimensions.", + Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { - if name == "" { - return requireFlag("name") - } - flags := codeEvaluatorFlags{ - imageTag: imageTag, - initParams: initParams, - dataSchema: dataSchema, - metrics: metrics, - endpoint: endpointFlg, - } - if err := validateEvaluatorSource(rubric, file, flags); err != nil { - return err + name := args[0] + if fromFile == "" { + return requireFlag("from-file") } - ctx := cmd.Context() - - if file != "" { - return runEvaluatorCreateFromFile(cmd, name, file, flags) - } - - raw, err := os.ReadFile(rubric) + raw, err := os.ReadFile(fromFile) if err != nil { - return fmt.Errorf("reading rubric %q: %w", rubric, err) + return fmt.Errorf("reading evaluator %q: %w", fromFile, err) } body, err := normalizeRubricBody(name, raw) if err != nil { - return fmt.Errorf("rubric %q: %w", rubric, err) + return fmt.Errorf("evaluator %q: %w", fromFile, err) } + ctx := cmd.Context() ec, err := newEvalContext(ctx, endpointFlg) if err != nil { return err } defer ec.Close() + latest := ec.evalClient.LatestEvaluatorVersionNumber( + ctx, name, ProjectEndpointAPIVersion, + ) + if err := checkAssetExistence(verb, "evaluator", name, latest > 0); err != nil { + return err + } + created, err := ec.evalClient.CreateEvaluatorVersion( ctx, name, body, ProjectEndpointAPIVersion, ) @@ -114,165 +104,24 @@ func newEvaluatorCreateCommand() *cobra.Command { }, } - cmd.Flags().StringVar(&name, "name", "", "Name of the evaluator.") - cmd.Flags().StringVar(&rubric, "rubric", "", "Path to the rubric JSON file.") - cmd.Flags().StringVar(&file, "file", "", - "Path to a single Python script declaring a top-level grade(sample, item) function.") - cmd.Flags().StringVar(&imageTag, "image-tag", "", - "Container image the evaluator runs in. Its packages are the only "+ - "dependencies the script can import beyond the standard library.") - cmd.Flags().StringVar(&initParams, "init-params", "", - "Path to a JSON Schema for the evaluator's initialization parameters.") - cmd.Flags().StringVar(&dataSchema, "data-schema", "", - "Path to a JSON Schema for the evaluator's input data.") - cmd.Flags().StringVar(&metrics, "metrics", "", - "Path to a JSON object describing the metrics the evaluator produces.") + cmd.Flags().StringVar(&fromFile, "from-file", "", "Path to the evaluator JSON file.") cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") return cmd } -// codeEvaluatorFlags are the optional settings for a code evaluator. -type codeEvaluatorFlags struct { - imageTag string - initParams string - dataSchema string - metrics string - endpoint string -} - -// validateEvaluatorSource enforces that exactly one source is named, and that -// the code-only settings are only used with the source they apply to. -// -// Deliberately checked here rather than with MarkFlagsMutuallyExclusive: that -// only rejects the "both" case, and its message names a flag group rather than -// saying what the two flags mean. Both mistakes deserve advice, and this is -// testable without driving cobra. -func validateEvaluatorSource(rubric, file string, flags codeEvaluatorFlags) error { +// checkAssetExistence enforces the one difference between create and update. +func checkAssetExistence(verb, kind, name string, exists bool) error { switch { - case rubric == "" && file == "": + case verb == "create" && exists: return fmt.Errorf( - "one of --rubric or --file is required: --rubric takes a JSON file of " + - "weighted dimensions, --file takes a single Python script") - case rubric != "" && file != "": + "%s %q already exists: use `update` to publish a new version", kind, name) + case verb == "update" && !exists: return fmt.Errorf( - "--rubric and --file cannot be used together: an evaluator is either a " + - "rubric or code, not both") - } - - // A rubric's schemas are fixed by the service and a rubric runs no code, so - // these would be accepted and then quietly dropped — the worst kind of - // no-op, because the author believes the evaluator was published carrying - // them. - if file == "" { - for _, named := range []struct { - flag string - value string - }{ - {"image-tag", flags.imageTag}, - {"init-params", flags.initParams}, - {"data-schema", flags.dataSchema}, - {"metrics", flags.metrics}, - } { - if named.value != "" { - return fmt.Errorf( - "--%s applies to a code evaluator and needs --file; "+ - "a rubric runs no code and its schemas are set by the service", - named.flag) - } - } + "%s %q does not exist: use `create` to register it", kind, name) } return nil } -// runEvaluatorCreateFromFile validates the script, then publishes it. -func runEvaluatorCreateFromFile( - cmd *cobra.Command, - name string, - file string, - flags codeEvaluatorFlags, -) error { - script, err := evalcore.LoadCodeEvaluator(name, file) - if err != nil { - return err - } - - opts, err := codeEvaluatorOptions(flags) - if err != nil { - return err - } - - ctx := cmd.Context() - ec, err := newEvalContext(ctx, flags.endpoint) - if err != nil { - return err - } - defer ec.Close() - - created, err := ec.evalClient.CreateCodeEvaluatorVersion( - ctx, script, opts, ProjectEndpointAPIVersion, - ) - if err != nil { - return fmt.Errorf("publishing evaluator %q: %w", name, err) - } - - if isJSON(cmd) { - return emitJSON(cmd.OutOrStdout(), created) - } - fmt.Fprintf(cmd.OutOrStdout(), - "Published evaluator %s version %s from %s\n", - created.Name, created.Version, file) - return nil -} - -// codeEvaluatorOptions resolves the evaluator's schemas from the flags. -// -// They are not read from the script and not read from a descriptor beside it: -// the grader is handed one file of source, so anything the service needs that -// is not Python has to be named on the command line or carried in the eval -// config. -func codeEvaluatorOptions(flags codeEvaluatorFlags) (eval_api.CodeEvaluatorOptions, error) { - opts := eval_api.CodeEvaluatorOptions{ImageTag: flags.imageTag} - - for _, declared := range []struct { - path string - flag string - field *json.RawMessage - }{ - {flags.initParams, "init-params", &opts.InitParameters}, - {flags.dataSchema, "data-schema", &opts.DataSchema}, - {flags.metrics, "metrics", &opts.Metrics}, - } { - if declared.path == "" { - continue - } - raw, err := readJSONObject(declared.path) - if err != nil { - return opts, fmt.Errorf("--%s %q: %w", declared.flag, declared.path, err) - } - *declared.field = raw - } - - return opts, nil -} - -// readJSONObject reads a file that must hold a JSON object. -// -// Parsing here rather than letting the service reject it keeps a typo from -// costing an upload and a published version, and names the file that is wrong. -func readJSONObject(path string) (json.RawMessage, error) { - raw, err := os.ReadFile(path) - if err != nil { - return nil, err - } - var probe map[string]json.RawMessage - if err := json.Unmarshal(raw, &probe); err != nil { - return nil, fmt.Errorf("not a JSON object: %w", err) - } - return json.RawMessage(raw), nil -} - -// normalizeRubricBody accepts either a bare definition ({type, dimensions}) or -// a full evaluator document ({name, definition}) and returns the request body. // rubricDefinitionType is the discriminator the service uses to deserialize a // rubric definition. const rubricDefinitionType = "rubric" @@ -296,6 +145,8 @@ func ensureDefinitionType(definition json.RawMessage) (json.RawMessage, error) { return json.Marshal(doc) } +// normalizeRubricBody accepts either a bare definition ({type, dimensions}) or +// a full evaluator document ({name, definition}) and returns the request body. func normalizeRubricBody(name string, raw []byte) (json.RawMessage, error) { var probe map[string]json.RawMessage if err := json.Unmarshal(raw, &probe); err != nil { @@ -303,7 +154,7 @@ func normalizeRubricBody(name string, raw []byte) (json.RawMessage, error) { } if definition, hasDefinition := probe["definition"]; hasDefinition { - // Already a full document; make sure the name matches the flag. + // Already a full document; make sure the name matches the argument. typed, err := ensureDefinitionType(definition) if err != nil { return nil, err @@ -339,14 +190,14 @@ func normalizeRubricBody(name string, raw []byte) (json.RawMessage, error) { func newEvaluatorListCommand() *cobra.Command { var ( - name string builtin bool endpointFlg string ) cmd := &cobra.Command{ Use: "list", - Short: "List evaluators, the versions of one evaluator, or the built-in evaluators.", + Short: "List the project's evaluators, or the built-in ones.", + Args: cobra.NoArgs, RunE: func(cmd *cobra.Command, args []string) error { ctx := cmd.Context() ec, err := newEvalContext(ctx, endpointFlg) @@ -355,18 +206,13 @@ func newEvaluatorListCommand() *cobra.Command { } defer ec.Close() - var list *eval_api.EvaluatorListResponse - switch { - case name != "": - list, err = ec.evalClient.ListEvaluatorVersions(ctx, name, ProjectEndpointAPIVersion) - case builtin: - // The service filters by type, and asking for nothing returns - // only the project's own evaluators. - list, err = ec.evalClient.ListEvaluators( - ctx, eval_api.EvaluatorTypeBuiltin, ProjectEndpointAPIVersion) - default: - list, err = ec.evalClient.ListEvaluators(ctx, "", ProjectEndpointAPIVersion) + // The service filters by type, and asking for nothing returns only + // the project's own evaluators. + filter := "" + if builtin { + filter = eval_api.EvaluatorTypeBuiltin } + list, err := ec.evalClient.ListEvaluators(ctx, filter, ProjectEndpointAPIVersion) if err != nil { return fmt.Errorf("listing evaluators: %w", err) } @@ -374,9 +220,49 @@ func newEvaluatorListCommand() *cobra.Command { }, } - cmd.Flags().StringVar(&name, "name", "", "Limit the listing to versions of this evaluator.") - cmd.Flags().BoolVar(&builtin, "builtin", false, "List the built-in evaluators instead of the project's own.") - cmd.MarkFlagsMutuallyExclusive("name", "builtin") + cmd.Flags().BoolVar(&builtin, "builtin", false, + "List the built-in evaluators instead of the project's own.") + cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") + return cmd +} + +// newEvaluatorVersionsCommand groups the version listing, so that `list` means +// the same thing for evaluators as it does for datasets: the assets, not their +// history. +func newEvaluatorVersionsCommand() *cobra.Command { + cmd := &cobra.Command{ + Use: "versions", + Short: "Inspect the versions of one evaluator.", + } + cmd.AddCommand(newEvaluatorVersionsListCommand()) + return cmd +} + +func newEvaluatorVersionsListCommand() *cobra.Command { + var endpointFlg string + + cmd := &cobra.Command{ + Use: "list ", + Short: "List the versions of an evaluator.", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + name := args[0] + + ctx := cmd.Context() + ec, err := newEvalContext(ctx, endpointFlg) + if err != nil { + return err + } + defer ec.Close() + + list, err := ec.evalClient.ListEvaluatorVersions(ctx, name, ProjectEndpointAPIVersion) + if err != nil { + return fmt.Errorf("listing versions of evaluator %q: %w", name, err) + } + return renderEvaluators(cmd, list) + }, + } + cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") return cmd } @@ -398,18 +284,16 @@ func renderEvaluators(cmd *cobra.Command, list *eval_api.EvaluatorListResponse) func newEvaluatorShowCommand() *cobra.Command { var ( - name string version string endpointFlg string ) cmd := &cobra.Command{ - Use: "show", + Use: "show ", Short: "Show an evaluator definition.", + Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { - if name == "" { - return requireFlag("name") - } + name := args[0] ctx := cmd.Context() ec, err := newEvalContext(ctx, endpointFlg) @@ -437,7 +321,6 @@ func newEvaluatorShowCommand() *cobra.Command { }, } - cmd.Flags().StringVar(&name, "name", "", "Name of the evaluator.") cmd.Flags().StringVar(&version, "version", "", "Version to show. Omit for the latest.") cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") return cmd @@ -445,18 +328,16 @@ func newEvaluatorShowCommand() *cobra.Command { func newEvaluatorDeleteCommand() *cobra.Command { var ( - name string version string endpointFlg string ) cmd := &cobra.Command{ - Use: "delete", + Use: "delete ", Short: "Delete an evaluator version.", + Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { - if name == "" { - return requireFlag("name") - } + name := args[0] if version == "" { return requireFlag("version") } @@ -488,7 +369,6 @@ func newEvaluatorDeleteCommand() *cobra.Command { }, } - cmd.Flags().StringVar(&name, "name", "", "Name of the evaluator.") cmd.Flags().StringVar(&version, "version", "", "Version to delete.") cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") return cmd diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator_code_live_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator_code_live_test.go deleted file mode 100644 index df32ba7cd8d..00000000000 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator_code_live_test.go +++ /dev/null @@ -1,246 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -//go:build live - -// This file proves the code-evaluator body the extension builds is accepted by -// the real service, and that what comes back is the shape the extension -// expects. -// -// It exists because the wire contract was settled from source rather than from -// a live call: two published documents disagreed on the definition body, and -// only the service can say which one it honours. It asserts the round trip -// field by field so a drift shows up as a named mismatch, not a vague failure. -// -// go test -tags live -v ./internal/cmd/ -run TestLiveCodeEvaluator -// -// Required: AZURE_AI_EVAL_E2E_LIVE=1 and FOUNDRY_PROJECT_ENDPOINT. - -package cmd - -import ( - "context" - "encoding/json" - "fmt" - "os" - "path/filepath" - "testing" - "time" - - "azureaieval/internal/pkg/eval_api" - "azureaieval/internal/pkg/evalcore" - - "github.com/stretchr/testify/require" -) - -// liveCodeEvaluatorName is unique per run so concurrent runs, and reruns after -// a failure that skipped cleanup, do not collide. -func liveCodeEvaluatorName(t *testing.T, suffix string) string { - t.Helper() - return fmt.Sprintf("azdcode_%s_%d", suffix, time.Now().UnixNano()) -} - -// writeLiveEvaluator writes a self-contained evaluator script and returns its -// path. It goes through the production loader afterwards, so the shipping -// validation is exercised rather than bypassed. -func writeLiveEvaluator(t *testing.T, name string) string { - t.Helper() - dir := t.TempDir() - path := filepath.Join(dir, name+".py") - - source := `def grade(sample, item) -> float: - return float(len((item or {}).get("response", ""))) -` - require.NoError(t, os.WriteFile(path, []byte(source), 0o600)) - return path -} - -// codeDefinitionOnService reads the registered version back and returns its -// definition, so the assertions run against what the service persisted rather -// than against what was sent. -func codeDefinitionOnService( - t *testing.T, - client *eval_api.EvalClient, - name, version string, -) map[string]json.RawMessage { - t.Helper() - - raw, err := client.GetEvaluatorRaw( - context.Background(), name, version, ProjectEndpointAPIVersion) - require.NoError(t, err, "reading back evaluator %s version %s", name, version) - - var doc map[string]json.RawMessage - require.NoError(t, json.Unmarshal(raw, &doc)) - require.Contains(t, doc, "definition", - "the registered evaluator carries no definition: %s", string(raw)) - - var definition map[string]json.RawMessage - require.NoError(t, json.Unmarshal(doc["definition"], &definition)) - return definition -} - -func stringField(t *testing.T, definition map[string]json.RawMessage, key string) string { - t.Helper() - raw, ok := definition[key] - if !ok { - return "" - } - var value string - if err := json.Unmarshal(raw, &value); err != nil { - return "" - } - return value -} - -// TestLiveCodeEvaluatorRoundTrip publishes a script and asserts it comes back -// as a code definition carrying the source inline. -// -// code_text is the only source property that reaches the executor: the -// definition is consumed as an OpenAI python grader, whose contract is a -// single Source string. A version registered with blob_uri instead publishes -// cleanly and then fails every run with "top-level grade() function not found -// in source", so what matters here is that the source itself round-trips. -func TestLiveCodeEvaluatorRoundTrip(t *testing.T) { - client, _ := liveEvalClient(t) - ctx := context.Background() - - name := liveCodeEvaluatorName(t, "roundtrip") - path := writeLiveEvaluator(t, name) - - // The shipping loader, not a hand-built script: this test has to fail if - // the production path stops producing a publishable script. - script, err := evalcore.LoadCodeEvaluator(name, path) - require.NoError(t, err) - require.Contains(t, script.Source, "def grade(") - - opts, err := codeEvaluatorOptions(codeEvaluatorFlags{}) - require.NoError(t, err) - - created, err := client.CreateCodeEvaluatorVersion(ctx, script, opts, ProjectEndpointAPIVersion) - require.NoError(t, err, "the service rejected the code evaluator body") - require.NotEmpty(t, created.Version) - t.Cleanup(func() { - _ = client.DeleteEvaluatorVersion( - context.Background(), name, created.Version, ProjectEndpointAPIVersion) - }) - - definition := codeDefinitionOnService(t, client, name, created.Version) - - require.Equal(t, eval_api.CodeDefinitionType, stringField(t, definition, "type"), - "the discriminator must round-trip as the lowercase snake_case value") - require.Contains(t, stringField(t, definition, "code_text"), "def grade(", - "the source must round-trip inline; an empty code_text means the grader "+ - "would be handed nothing to run") - require.Contains(t, definition, "metrics", - "a code definition must carry metrics; the service rejects one without") -} - -// TestLiveCodeEvaluatorCarriesSchemasAndImage proves the settings that only -// reach the service through flags survive the round trip. -// -// They cannot come from anywhere else. The grader is handed one file of -// source, so a descriptor beside the script would never travel with it, and an -// image tag dropped on the way would leave an evaluator whose imports fail at -// run time with no sign of why. -func TestLiveCodeEvaluatorCarriesSchemasAndImage(t *testing.T) { - client, _ := liveEvalClient(t) - ctx := context.Background() - - name := liveCodeEvaluatorName(t, "schemas") - path := writeLiveEvaluator(t, name) - - script, err := evalcore.LoadCodeEvaluator(name, path) - require.NoError(t, err) - - dir := t.TempDir() - schemaPath := filepath.Join(dir, "schema.json") - require.NoError(t, os.WriteFile(schemaPath, []byte( - `{"type":"object","properties":{"response":{"type":"string"}},"required":["response"]}`, - ), 0o600)) - - opts, err := codeEvaluatorOptions(codeEvaluatorFlags{dataSchema: schemaPath}) - require.NoError(t, err) - require.NotEmpty(t, opts.DataSchema) - - created, err := client.CreateCodeEvaluatorVersion(ctx, script, opts, ProjectEndpointAPIVersion) - require.NoError(t, err, "the service rejected a definition carrying a data schema") - t.Cleanup(func() { - _ = client.DeleteEvaluatorVersion( - context.Background(), name, created.Version, ProjectEndpointAPIVersion) - }) - - definition := codeDefinitionOnService(t, client, name, created.Version) - require.Contains(t, definition, "data_schema", - "the declared data schema must round-trip; without it the criteria builder "+ - "derives no data_mapping and the eval cannot be created") - require.Contains(t, string(definition["data_schema"]), "response") -} - -// TestLiveCodeEvaluatorPublishesANewVersion proves a second publish does not -// overwrite the first. -// -// Versions are immutable and evals bind to one, so a publish that replaced the -// previous version would silently change what every existing eval evaluates. -// -// The wait between the two publishes is not padding. The service assigns the -// next version from its own listing, and that listing lags the create by about -// a second: two publishes issued back to back were both answered with version -// 1, the second overwriting the first. The reconciler waits for a published -// version to appear in the listing before it moves on, so this waits the same -// way — the assertion is about publishing twice, not about racing the service. -func TestLiveCodeEvaluatorPublishesANewVersion(t *testing.T) { - client, _ := liveEvalClient(t) - ctx := context.Background() - - name := liveCodeEvaluatorName(t, "versions") - path := writeLiveEvaluator(t, name) - - script, err := evalcore.LoadCodeEvaluator(name, path) - require.NoError(t, err) - opts, err := codeEvaluatorOptions(codeEvaluatorFlags{}) - require.NoError(t, err) - - first, err := client.CreateCodeEvaluatorVersion(ctx, script, opts, ProjectEndpointAPIVersion) - require.NoError(t, err) - t.Cleanup(func() { - _ = client.DeleteEvaluatorVersion( - context.Background(), name, first.Version, ProjectEndpointAPIVersion) - }) - require.NotEmpty(t, first.Version, "the service must assign a version") - - awaitVersionListed(t, client, name, first.Version) - - second, err := client.CreateCodeEvaluatorVersion(ctx, script, opts, ProjectEndpointAPIVersion) - require.NoError(t, err) - t.Cleanup(func() { - _ = client.DeleteEvaluatorVersion( - context.Background(), name, second.Version, ProjectEndpointAPIVersion) - }) - require.NotEqual(t, first.Version, second.Version, - "a second publish must create a new version rather than replace the first") -} - -// awaitVersionListed blocks until a published version shows up in the version -// listing, which is the view the service's own version assignment reads. -func awaitVersionListed(t *testing.T, client *eval_api.EvalClient, name, version string) { - t.Helper() - ctx := context.Background() - - start := time.Now() - for { - list, err := client.ListEvaluatorVersions(ctx, name, ProjectEndpointAPIVersion) - if err == nil && list != nil { - for _, entry := range list.Value { - if entry.Version == version { - t.Logf("version %s listed after %s", - version, time.Since(start).Round(time.Millisecond)) - return - } - } - } - if time.Since(start) > 30*time.Second { - t.Fatalf("version %s of evaluator %s never appeared in the listing", version, name) - } - time.Sleep(250 * time.Millisecond) - } -} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator_code_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator_code_test.go deleted file mode 100644 index 0b1ff9205c2..00000000000 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator_code_test.go +++ /dev/null @@ -1,202 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package cmd - -import ( - "encoding/json" - "io" - "os" - "path/filepath" - "testing" - - "azureaieval/internal/pkg/eval_api" - "azureaieval/internal/pkg/evalcore" - "azureaieval/internal/project" - - "github.com/stretchr/testify/require" -) - -func writeTestFile(t *testing.T, root, rel, content string) string { - t.Helper() - path := filepath.Join(root, filepath.FromSlash(rel)) - require.NoError(t, os.MkdirAll(filepath.Dir(path), 0o755)) - require.NoError(t, os.WriteFile(path, []byte(content), 0o600)) - return path -} - -// toneEvaluatorSource is the shape the grader requires: one top-level -// grade(sample, item) returning a float. -const toneEvaluatorSource = "def grade(sample, item) -> float:\n" + - " return float(len((item or {}).get(\"response\", \"\")))\n" - -// An evaluator is either a rubric or code. Naming both, or neither, is a -// mistake the command has to name precisely — the two flags take different -// kinds of path and produce different definition types. -func TestValidateEvaluatorSource(t *testing.T) { - err := validateEvaluatorSource("", "", codeEvaluatorFlags{}) - require.Error(t, err) - require.Contains(t, err.Error(), "--rubric") - require.Contains(t, err.Error(), "--file") - require.Contains(t, err.Error(), "required") - - err = validateEvaluatorSource("rubric.json", "evaluator.py", codeEvaluatorFlags{}) - require.Error(t, err) - require.Contains(t, err.Error(), "cannot be used together") - - require.NoError(t, validateEvaluatorSource("rubric.json", "", codeEvaluatorFlags{})) - require.NoError(t, validateEvaluatorSource("", "evaluator.py", codeEvaluatorFlags{})) -} - -// The code-only settings describe a python grader. Accepting them beside a -// rubric and dropping them would leave the author believing the evaluator was -// published carrying an image and schemas it never had. -func TestValidateEvaluatorSource_RejectsCodeFlagsOnARubric(t *testing.T) { - for flag, flags := range map[string]codeEvaluatorFlags{ - "image-tag": {imageTag: "python:3.11"}, - "init-params": {initParams: "init.json"}, - "data-schema": {dataSchema: "schema.json"}, - "metrics": {metrics: "metrics.json"}, - } { - err := validateEvaluatorSource("rubric.json", "", flags) - require.Error(t, err, "for --%s", flag) - require.Contains(t, err.Error(), "--"+flag) - require.Contains(t, err.Error(), "--file") - - require.NoError(t, validateEvaluatorSource("", "evaluator.py", flags), - "--%s is valid with --file", flag) - } -} - -// The same check the command runs must be reachable from the command, so a -// future refactor cannot leave the flags declared but unvalidated. -func TestEvaluatorCreateRejectsBothSources(t *testing.T) { - cmd := newEvaluatorCreateCommand() - cmd.SetArgs([]string{"--name", "tone", "--rubric", "r.json", "--file", "tone.py"}) - cmd.SetOut(io.Discard) - cmd.SetErr(io.Discard) - cmd.SilenceUsage = true - - err := cmd.Execute() - require.Error(t, err) - require.Contains(t, err.Error(), "cannot be used together") -} - -func TestEvaluatorCreateRejectsNeitherSource(t *testing.T) { - cmd := newEvaluatorCreateCommand() - cmd.SetArgs([]string{"--name", "tone"}) - cmd.SetOut(io.Discard) - cmd.SetErr(io.Discard) - cmd.SilenceUsage = true - - err := cmd.Execute() - require.Error(t, err) - require.Contains(t, err.Error(), "one of --rubric or --file") -} - -// A script with no top-level grade() must be refused before a version is -// published, and the refusal must come from the command rather than from a run -// that fails minutes later. -func TestEvaluatorCreateRejectsAScriptWithoutGrade(t *testing.T) { - dir := t.TempDir() - path := writeTestFile(t, dir, "tone.py", - "class ToneEvaluator:\n def __call__(self, **kwargs):\n return {\"result\": 1}\n") - - cmd := newEvaluatorCreateCommand() - cmd.SetArgs([]string{"--name", "tone", "--file", path}) - cmd.SetOut(io.Discard) - cmd.SetErr(io.Discard) - cmd.SilenceUsage = true - - err := cmd.Execute() - require.Error(t, err) - require.Contains(t, err.Error(), "grade(sample, item)") -} - -// The flags are the only place a code evaluator's schemas can come from: the -// grader is handed one file of source, so nothing that is not Python can -// travel with it. -func TestCodeEvaluatorOptions_ReadsTheFlags(t *testing.T) { - empty, err := codeEvaluatorOptions(codeEvaluatorFlags{}) - require.NoError(t, err) - require.Empty(t, empty.ImageTag) - require.Empty(t, empty.Metrics) - require.Empty(t, empty.DataSchema) - require.Empty(t, empty.InitParameters) - - dir := t.TempDir() - metricsPath := writeTestFile(t, dir, "metrics.json", - `{"result":{"type":"continuous"}}`) - initPath := writeTestFile(t, dir, "init.json", - `{"type":"object","properties":{"deployment_name":{"type":"string"}}}`) - schemaPath := writeTestFile(t, dir, "schema.json", - `{"type":"object","properties":{"response":{"type":"string"}}}`) - - opts, err := codeEvaluatorOptions(codeEvaluatorFlags{ - imageTag: "mcr.microsoft.com/azureml/evaluator:latest", - metrics: metricsPath, - initParams: initPath, - dataSchema: schemaPath, - }) - require.NoError(t, err) - require.Equal(t, "mcr.microsoft.com/azureml/evaluator:latest", opts.ImageTag) - require.Contains(t, string(opts.Metrics), "continuous") - require.Contains(t, string(opts.InitParameters), "deployment_name") - require.Contains(t, string(opts.DataSchema), "response") -} - -// A typo in a schema file must be reported against the flag that named it, -// not discovered by the service after a version has been published. -func TestCodeEvaluatorOptions_RejectsMalformedInput(t *testing.T) { - dir := t.TempDir() - - bad := writeTestFile(t, dir, "metrics.json", "[1,2,3]") - _, err := codeEvaluatorOptions(codeEvaluatorFlags{metrics: bad}) - require.Error(t, err) - require.Contains(t, err.Error(), "--metrics") - require.Contains(t, err.Error(), "JSON object") - - _, err = codeEvaluatorOptions(codeEvaluatorFlags{ - dataSchema: filepath.Join(dir, "absent.json"), - }) - require.Error(t, err) - require.Contains(t, err.Error(), "--data-schema") -} - -// The service rejects a code definition carrying no metrics, so a script -// published without any still has to publish with one. -func TestDefaultCodeMetricsIsAJSONObject(t *testing.T) { - var metrics map[string]map[string]any - require.NoError(t, json.Unmarshal(eval_api.DefaultCodeMetrics, &metrics)) - require.Len(t, metrics, 1) - require.Contains(t, metrics, "result") - require.Equal(t, "continuous", metrics["result"]["type"]) -} - -// Both kinds of evaluator source are one file, and the reconciler tells them -// apart by extension rather than by stat-ing the path. -func TestEvaluatorSourceClassificationAndFingerprint(t *testing.T) { - root := t.TempDir() - - rubric := writeTestFile(t, root, "rubric.json", `{"dimensions":[]}`) - script := writeTestFile(t, root, "tone.py", toneEvaluatorSource) - - require.False(t, evalcore.IsCodeEvaluatorSource(rubric)) - require.True(t, evalcore.IsCodeEvaluatorSource(script)) - - rubricDigest, err := project.Fingerprint(rubric) - require.NoError(t, err) - scriptDigest, err := project.Fingerprint(script) - require.NoError(t, err) - require.NotEqual(t, rubricDigest, scriptDigest) - - // Editing the script must be noticed, or a deploy would reuse a version - // holding the old source. - writeTestFile(t, root, "tone.py", toneEvaluatorSource+"\n# tweak\n") - changed, err := project.Fingerprint(script) - require.NoError(t, err) - require.NotEqual(t, scriptDigest, changed) - - _, err = project.Fingerprint(filepath.Join(root, "absent.py")) - require.Error(t, err) -} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/model_target_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/model_target_test.go deleted file mode 100644 index c202a87a4c7..00000000000 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/model_target_test.go +++ /dev/null @@ -1,98 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package cmd - -import ( - "encoding/json" - "testing" - - "azureaieval/internal/pkg/eval_api" - "azureaieval/internal/pkg/evalcore" - "azureaieval/internal/project" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -// A model answers as plain text and calls no tools. Binding an agent's richer -// output would leave the evaluator waiting on fields the run never produces, -// which the service reports as a missing input rather than a mapping mistake. -func TestSampleBindingsFor(t *testing.T) { - agent := sampleBindingsFor(project.TargetTypeAgent) - assert.Equal(t, "{{sample.output_items}}", agent["response"]) - assert.Contains(t, agent, "tool_calls") - assert.Contains(t, agent, "tool_definitions") - - model := sampleBindingsFor(project.TargetTypeModel) - assert.Equal(t, "{{sample.output_text}}", model["response"]) - assert.NotContains(t, model, "tool_calls", "a model calls no tools") - assert.NotContains(t, model, "tool_definitions") - - assert.Nil(t, sampleBindingsFor(""), "with nothing invoked, nothing is bound") -} - -// The criteria a group sends depend on what it targets. -func TestBuildEvalRequest_BindsByTargetKind(t *testing.T) { - schemas := map[string]*eval_api.EvaluatorSummary{ - "builtin.coherence": { - Name: "builtin.coherence", - Definition: &eval_api.EvaluatorContract{ - DataSchema: &eval_api.JSONSchema{ - Required: []string{"query", "response"}, - Properties: map[string]any{ - "query": map[string]any{"type": "string"}, - "response": map[string]any{"type": "string"}, - }, - }, - }, - }, - } - - for _, tc := range []struct { - targetType string - want string - }{ - {project.TargetTypeAgent, "{{sample.output_items}}"}, - {project.TargetTypeModel, "{{sample.output_text}}"}, - } { - t.Run(tc.targetType, func(t *testing.T) { - group := &project.Eval{ - Name: "quality", - Evaluators: []evalcore.EvaluatorRef{{Name: "builtin.coherence"}}, - Target: &project.Target{Type: tc.targetType, Name: "thing"}, - } - req, err := buildEvalRequest(group, schemas, map[string]bool{"query": true}) - require.NoError(t, err) - require.Len(t, req.TestingCriteria, 1) - assert.Equal(t, tc.want, req.TestingCriteria[0].DataMapping["response"]) - assert.Equal(t, "{{item.query}}", req.TestingCriteria[0].DataMapping["query"]) - }) - } -} - -// The target the run posts has to match what the group's criteria expect. -func TestNewModelTargetDataSource(t *testing.T) { - ds := eval_api.NewModelTargetDataSource("gpt-4.1-nano") - require.NotNil(t, ds.Target) - assert.Equal(t, "azure_ai_model", ds.Target.Type) - assert.Equal(t, "gpt-4.1-nano", ds.Target.Model) - - raw, err := json.Marshal(ds) - require.NoError(t, err) - body := string(raw) - assert.Contains(t, body, `"model":"gpt-4.1-nano"`) - assert.NotContains(t, body, `"name"`, "a model target is addressed by deployment, not name") - assert.NotContains(t, body, "tool_descriptions", "a model calls no tools") -} - -func TestNewAgentTargetDataSource_StillSendsAgentFields(t *testing.T) { - ds := eval_api.NewAgentTargetDataSource("support-agent", nil) - require.NotNil(t, ds.Target) - assert.Equal(t, "azure_ai_agent", ds.Target.Type) - assert.Equal(t, "support-agent", ds.Target.Name) - - raw, err := json.Marshal(ds) - require.NoError(t, err) - assert.NotContains(t, string(raw), `"model"`) -} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go index 8a1e5eb8138..44df054a40b 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go @@ -6,8 +6,6 @@ package cmd import ( "bufio" "context" - "crypto/sha256" - "encoding/hex" "encoding/json" "fmt" "os" @@ -18,7 +16,6 @@ import ( "time" "azureaieval/internal/pkg/dataset_api" - "azureaieval/internal/pkg/evalcore" "azureaieval/internal/project" ) @@ -256,9 +253,6 @@ func (r *evalReconciler) EnsureEvaluator( if _, err := os.Stat(localPath); err != nil { return "", false, fmt.Errorf("evaluator source %q: %w", localPath, err) } - if evalcore.IsCodeEvaluatorSource(localPath) { - return r.ensureCodeEvaluator(ctx, decl, localPath) - } raw, err := os.ReadFile(localPath) if err != nil { @@ -289,102 +283,6 @@ func (r *evalReconciler) EnsureEvaluator( return created.Version, true, nil } -// codeEvaluatorDigest fingerprints everything a published version depends on. -// -// The script alone is not enough: changing only metrics or the image tag -// changes what gets published, and hashing just the source would leave that -// edit undeployed with `azd up` reporting no change. -func codeEvaluatorDigest(decl project.EvaluatorDecl, path string) (string, error) { - sum := sha256.New() - - script, err := project.Fingerprint(path) - if err != nil { - return "", err - } - fmt.Fprintf(sum, "script:%s\nimage:%s\n", script, decl.ImageTag) - - for _, settings := range []struct { - label string - path string - }{ - {"metrics", decl.Metrics}, - {"data_schema", decl.DataSchema}, - {"init_parameters", decl.InitParameters}, - } { - if settings.path == "" { - continue - } - digest, err := project.Fingerprint(settings.path) - if err != nil { - return "", fmt.Errorf("evaluator %q %s: %w", decl.Name, settings.label, err) - } - fmt.Fprintf(sum, "%s:%s\n", settings.label, digest) - } - - return hex.EncodeToString(sum.Sum(nil)), nil -} - -// ensureCodeEvaluator publishes a Python script only when its content changed -// since the last deploy. -// -// Every publish is a new immutable version, so without this a repeated -// `azd up` would leave a trail of identical versions and force every eval -// bound to the evaluator to be recreated along with them. -func (r *evalReconciler) ensureCodeEvaluator( - ctx context.Context, - decl project.EvaluatorDecl, - path string, -) (string, bool, error) { - // Validated before anything is published: a script with no top-level - // grade() is only rejected when a run executes, long after a version has - // been published and an eval bound to it. - script, err := evalcore.LoadCodeEvaluator(decl.Name, path) - if err != nil { - return "", false, err - } - - digest, err := codeEvaluatorDigest(decl, path) - if err != nil { - return "", false, err - } - - key := project.FingerprintKey("evaluator", decl.Name) - recordedVersion := r.ec.getEnvValue(ctx, versionKey("evaluator", decl.Name)) - if r.ec.getEnvValue(ctx, key) == digest && recordedVersion != "" { - // Unchanged since the last deploy, but that alone does not make the - // recorded version safe to reuse: someone may have published a newer - // one outside the repo, and binding the eval to the older one would - // quietly evaluate with superseded code. - if err := r.checkEvaluatorDrift(ctx, decl.Name, recordedVersion); err != nil { - return "", false, err - } - return recordedVersion, false, nil - } - - opts, err := codeEvaluatorOptions(codeEvaluatorFlags{ - imageTag: decl.ImageTag, - metrics: decl.Metrics, - dataSchema: decl.DataSchema, - initParams: decl.InitParameters, - }) - if err != nil { - return "", false, err - } - - created, err := r.ec.evalClient.CreateCodeEvaluatorVersion( - ctx, script, opts, ProjectEndpointAPIVersion, - ) - if err != nil { - return "", false, err - } - - r.awaitEvaluatorReadable(ctx, decl.Name, created.Version) - - _ = r.ec.setEnvValue(ctx, key, digest) - _ = r.ec.setEnvValue(ctx, versionKey("evaluator", decl.Name), created.Version) - return created.Version, true, nil -} - // evaluatorPropagation bounds the wait for a freshly published evaluator to // become usable. // diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler_digest_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler_digest_test.go deleted file mode 100644 index 5fd6d7521c7..00000000000 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler_digest_test.go +++ /dev/null @@ -1,81 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package cmd - -import ( - "os" - "path/filepath" - "testing" - - "azureaieval/internal/project" - - "github.com/stretchr/testify/require" -) - -// A code evaluator's published version depends on more than its script, so the -// digest that decides whether to republish has to cover the rest of it. -// Hashing only the source would leave a changed metric or image tag sitting in -// the config while `azd up` reported nothing to do. -func TestCodeEvaluatorDigest(t *testing.T) { - dir := t.TempDir() - - script := filepath.Join(dir, "grader.py") - require.NoError(t, os.WriteFile(script, - []byte("def grade(sample, item) -> float:\n return 1.0\n"), 0o600)) - - metrics := filepath.Join(dir, "metrics.json") - require.NoError(t, os.WriteFile(metrics, - []byte(`{"result":{"type":"continuous"}}`), 0o600)) - - base := project.EvaluatorDecl{Name: "answer_length"} - baseline, err := codeEvaluatorDigest(base, script) - require.NoError(t, err) - - t.Run("stable across calls", func(t *testing.T) { - again, err := codeEvaluatorDigest(base, script) - require.NoError(t, err) - require.Equal(t, baseline, again, - "an unchanged evaluator must not republish on every deploy") - }) - - t.Run("notices the script", func(t *testing.T) { - require.NoError(t, os.WriteFile(script, - []byte("def grade(sample, item) -> float:\n return 2.0\n"), 0o600)) - changed, err := codeEvaluatorDigest(base, script) - require.NoError(t, err) - require.NotEqual(t, baseline, changed) - }) - - t.Run("notices the image tag", func(t *testing.T) { - withImage := base - withImage.ImageTag = "python:3.12-slim" - changed, err := codeEvaluatorDigest(withImage, script) - require.NoError(t, err) - require.NotEqual(t, baseline, changed, - "changing the image changes what runs, so it must republish") - }) - - t.Run("notices the metrics file", func(t *testing.T) { - withMetrics := base - withMetrics.Metrics = metrics - before, err := codeEvaluatorDigest(withMetrics, script) - require.NoError(t, err) - - require.NoError(t, os.WriteFile(metrics, - []byte(`{"result":{"type":"ordinal","min_value":0,"max_value":1}}`), 0o600)) - after, err := codeEvaluatorDigest(withMetrics, script) - require.NoError(t, err) - require.NotEqual(t, before, after, - "editing metrics alone must republish, or the edit never deploys") - }) - - t.Run("reports a missing settings file", func(t *testing.T) { - missing := base - missing.DataSchema = filepath.Join(dir, "nope.json") - _, err := codeEvaluatorDigest(missing, script) - require.Error(t, err) - require.Contains(t, err.Error(), "data_schema", - "the error must name which setting could not be read") - }) -} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/results.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/results.go index 8b7728bb7bc..d2377f34671 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/results.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/results.go @@ -21,10 +21,20 @@ func newResultsCommand() *cobra.Command { Use: "results", Short: "Inspect evaluation results.", } - cmd.AddCommand(newResultsShowCommand(), newResultsExportCommand(), newResultsCompareCommand()) + cmd.AddCommand(newResultsShowCommand(), newResultsExportCommand()) return cmd } +// formatStat renders a statistic, showing an undefined one as a dash. A score +// the service left unset has no value, and printing the literal "NaN" in a +// results table reads like a failure rather than the arithmetic it is. +func formatStat(verb string, v eval_api.LenientFloat) string { + if !v.Defined() { + return "-" + } + return fmt.Sprintf(verb, float64(v)) +} + func newResultsShowCommand() *cobra.Command { var ( runID string diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops_test.go index 8063f09253b..05f2e32b773 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops_test.go @@ -77,7 +77,7 @@ func TestEvalCommandsAcceptIDAsAFlag(t *testing.T) { for _, name := range []string{ "run list", "run show", "run cancel", - "results show", "results export", "results compare", + "results show", "results export", } { cmd := subs[name] require.NotNil(t, cmd, "%s should exist", name) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/traces_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/traces_test.go deleted file mode 100644 index 078a62435b9..00000000000 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/traces_test.go +++ /dev/null @@ -1,119 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package cmd - -import ( - "bytes" - "encoding/json" - "testing" - "time" - - "azureaieval/internal/pkg/eval_api" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -// A window is only sent when one was asked for; the service defaults it -// otherwise. -func TestNewTracesDataSource_OmitsAnUnsetWindow(t *testing.T) { - ds := eval_api.NewTracesDataSource("support-agent", 0, time.Time{}, 0) - assert.Equal(t, eval_api.EvalRunDataSourceTypeTraces, ds.Type) - assert.Equal(t, "support-agent", ds.AgentName) - - raw, err := json.Marshal(ds) - require.NoError(t, err) - body := string(raw) - assert.NotContains(t, body, "lookback_hours") - assert.NotContains(t, body, "end_time") - assert.NotContains(t, body, "max_traces") - assert.NotContains(t, body, "input_messages", "traces carry no template") -} - -// The service reads `lookback_hours` and has no start bound. Sending a -// start_time is accepted and dropped, which silently leaves the default seven -// days in place, so the window has to travel as hours. -func TestNewTracesDataSource_SendsAWindowTheServiceReads(t *testing.T) { - ds := eval_api.NewTracesDataSource("support-agent", 30*24, time.Time{}, 25) - assert.Equal(t, 720, ds.LookbackHours) - assert.Equal(t, 25, ds.MaxTraces) - - raw, err := json.Marshal(ds) - require.NoError(t, err) - body := string(raw) - assert.Contains(t, body, `"lookback_hours":720`) - assert.NotContains(t, body, "start_time", - "the service drops start_time and falls back to its default window") -} - -// The reason a run failed is the only actionable part of the response, so it -// has to survive into the output. -func TestRunFailureMessage(t *testing.T) { - var run eval_api.OpenAIEvalRun - require.NoError(t, json.Unmarshal([]byte(`{ - "id": "evalrun_x", "status": "failed", - "error": { "code": "UserError", "message": " No trace data found for agent_name 'a'. " } - }`), &run)) - assert.Equal(t, "No trace data found for agent_name 'a'.", run.Failure()) - - // The field is present and null-valued on success, so presence alone - // must not read as failure. - var ok eval_api.OpenAIEvalRun - require.NoError(t, json.Unmarshal([]byte(`{ - "id": "evalrun_y", "status": "completed", - "error": { "code": null, "message": null } - }`), &ok)) - assert.Empty(t, ok.Failure()) - - var absent eval_api.OpenAIEvalRun - require.NoError(t, json.Unmarshal([]byte(`{"id":"evalrun_z","status":"completed"}`), &absent)) - assert.Empty(t, absent.Failure()) - - var nilRun *eval_api.OpenAIEvalRun - assert.Empty(t, nilRun.Failure()) -} - -// The ids travel as ordinary JSONL rows with a mapping pointing at the field -// that holds each one; that is how the service finds the chat history. -func TestNewResponsesDataSource(t *testing.T) { - ds := eval_api.NewResponsesDataSource([]string{"resp_a", "resp_b"}, 10) - assert.Equal(t, eval_api.EvalRunDataSourceTypeResponses, ds.Type) - require.NotNil(t, ds.ItemGenerationParams) - assert.Equal(t, "response_retrieval", ds.ItemGenerationParams.Type) - assert.Equal(t, 10, ds.ItemGenerationParams.MaxNumTurns) - assert.Equal(t, - map[string]string{"response_id": "{{item.response_id}}"}, - ds.ItemGenerationParams.DataMapping) - - raw, err := json.Marshal(ds) - require.NoError(t, err) - body := string(raw) - assert.Contains(t, body, `"response_id":"resp_a"`) - assert.Contains(t, body, `"response_id":"resp_b"`) - assert.NotContains(t, body, "agent_name", "responses carry no agent") -} - -// An unset turn limit is left to the service rather than sent as zero. -func TestNewResponsesDataSource_OmitsAnUnsetTurnLimit(t *testing.T) { - ds := eval_api.NewResponsesDataSource([]string{"resp_a"}, 0) - raw, err := json.Marshal(ds) - require.NoError(t, err) - assert.NotContains(t, string(raw), "max_num_turns") -} - -func TestRenderRun_ShowsTheFailureReason(t *testing.T) { - var buf bytes.Buffer - run := &eval_api.OpenAIEvalRun{ - ID: "evalrun_x", - Status: "failed", - Error: &eval_api.JobError{Code: "UserError", Message: "No trace data found."}, - } - require.NoError(t, renderRun(&buf, run)) - assert.Contains(t, buf.String(), "failed") - assert.Contains(t, buf.String(), "No trace data found.") - - var clean bytes.Buffer - require.NoError(t, renderRun(&clean, &eval_api.OpenAIEvalRun{ID: "evalrun_y", Status: "completed"})) - assert.NotContains(t, clean.String(), " \n", "a successful run gains no blank reason line") -} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/code_evaluator.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/code_evaluator.go deleted file mode 100644 index f9e1305eea6..00000000000 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/code_evaluator.go +++ /dev/null @@ -1,178 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package eval_api - -import ( - "context" - "encoding/json" - "fmt" - "net/http" - "net/url" - "strconv" - - "azureaieval/internal/pkg/evalcore" -) - -// CodeDefinitionType is the discriminator the service uses to deserialize a -// code evaluator definition. -// -// The wire shape is snake_case with a lowercase discriminator, matching -// CodeBasedEvaluatorDefinition in the Foundry data-plane OpenAPI document -// (`type` enum ["code"], plus code_text, image_tag, init_parameters, -// data_schema and metrics). An earlier draft documented a camelCase body with -// `type: "CodeBased"`; that shape is not what the deployed service accepts. -const CodeDefinitionType = "code" - -// evaluatorTypeCustom marks an evaluator as authored by the project rather -// than shipped by the platform. -const evaluatorTypeCustom = "custom" - -// foundryFeaturesHeader opts a request in to preview behaviour. The code -// definition's properties are declared as preview, so the header is sent with -// every call that sets one. -const ( - foundryFeaturesHeader = "Foundry-Features" - foundryFeatureEvalsV1 = "Evaluations=V1Preview" - defaultCodeMetricName = "result" - defaultCodeMetricType = "continuous" - defaultMetricDirection = "increase" -) - -// DefaultCodeMetrics is used when the caller declares none. -// -// The service rejects a code definition carrying no metrics, and the documented -// evaluator output is a JSON object whose `result` field holds the score, so -// this describes exactly that. It is a default, not a constraint: any declared -// metrics replace it wholesale. -var DefaultCodeMetrics = json.RawMessage(fmt.Sprintf( - `{%q:{"type":%q,"desirable_direction":%q,"is_primary":true}}`, - defaultCodeMetricName, defaultCodeMetricType, defaultMetricDirection, -)) - -// CodeEvaluatorOptions carries the parts of an evaluator version that do not -// come from the Python source itself. -type CodeEvaluatorOptions struct { - DisplayName string - Description string - Categories []string - ImageTag string - InitParameters json.RawMessage - DataSchema json.RawMessage - Metrics json.RawMessage -} - -// codeDefinition is the wire body of a code evaluator definition. -// -// code_text carries the whole evaluator. The contract's other source property, -// blob_uri, is deliberately absent: the definition is consumed as an OpenAI -// python grader, whose contract (GraderPython) is a single `Source` string -// with no notion of a folder, archive, file list or entry point. A definition -// published with blob_uri alone registers cleanly and then fails the run with -// "Invalid grader source: top-level grade() function not found in source", -// because nothing reads the blob back into Source. -// -// image_tag is how a grader gets dependencies. Only one file is ever sent, so -// a helper module cannot travel with it and anything beyond the standard -// library has to already be in the image. -type codeDefinition struct { - Type string `json:"type"` - CodeText string `json:"code_text,omitempty"` - ImageTag string `json:"image_tag,omitempty"` - InitParameters json.RawMessage `json:"init_parameters,omitempty"` - DataSchema json.RawMessage `json:"data_schema,omitempty"` - Metrics json.RawMessage `json:"metrics,omitempty"` -} - -// createEvaluatorVersionRequest is the POST body for a new evaluator version. -// The service assigns the version; it is not carried here. -type createEvaluatorVersionRequest struct { - Name string `json:"name,omitempty"` - DisplayName string `json:"display_name,omitempty"` - Description string `json:"description,omitempty"` - EvaluatorType string `json:"evaluator_type,omitempty"` - Categories []string `json:"categories,omitempty"` - Definition *codeDefinition `json:"definition"` -} - -// CreateCodeEvaluatorVersion publishes a Python script as a new version of a -// code evaluator. -// -// The source is sent inline. There is no upload step and no storage to -// reserve: the executor is handed a string of source, so a blob it would never -// read adds a round trip, a SAS write, and a failure mode in exchange for -// nothing that reaches the grader. -func (c *EvalClient) CreateCodeEvaluatorVersion( - ctx context.Context, - script *evalcore.CodeEvaluatorScript, - opts CodeEvaluatorOptions, - apiVersion string, -) (*EvaluatorVersion, error) { - if script == nil { - return nil, fmt.Errorf("no evaluator script to publish") - } - - definition := &codeDefinition{ - Type: CodeDefinitionType, - CodeText: script.Source, - ImageTag: opts.ImageTag, - InitParameters: opts.InitParameters, - DataSchema: opts.DataSchema, - Metrics: opts.Metrics, - } - if len(definition.Metrics) == 0 { - definition.Metrics = DefaultCodeMetrics - } - - body := &createEvaluatorVersionRequest{ - Name: script.Name, - DisplayName: opts.DisplayName, - Description: opts.Description, - EvaluatorType: evaluatorTypeCustom, - Categories: opts.Categories, - Definition: definition, - } - - path := pathEvaluators + "/" + url.PathEscape(script.Name) + "/versions" - respBody, err := c.doRequestWithHeaders( - ctx, http.MethodPost, path, nil, body, apiVersion, previewHeaders(), - ) - if err != nil { - return nil, err - } - - var created EvaluatorVersion - if len(respBody) > 0 { - if err := json.Unmarshal(respBody, &created); err != nil { - return nil, fmt.Errorf("failed to parse response: %w", err) - } - } - if created.Name == "" { - created.Name = script.Name - } - return &created, nil -} - -// LatestEvaluatorVersionNumber reports the newest registered version as an -// integer, or 0 when the evaluator is unknown or its versions are not numeric. -func (c *EvalClient) LatestEvaluatorVersionNumber( - ctx context.Context, - name string, - apiVersion string, -) int { - list, err := c.ListEvaluatorVersions(ctx, name, apiVersion) - if err != nil || list == nil || len(list.Value) == 0 { - return 0 - } - number, err := strconv.Atoi(pickLatestVersion(list.Value)) - if err != nil { - return 0 - } - return number -} - -// previewHeaders returns the opt-in header for the preview properties the code -// definition relies on. -func previewHeaders() map[string]string { - return map[string]string{foundryFeaturesHeader: foundryFeatureEvalsV1} -} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/evaluators.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/evaluators.go index 84ad2ac811f..7b3181b6ade 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/evaluators.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/evaluators.go @@ -158,6 +158,24 @@ func (c *EvalClient) ListEvaluatorVersions( ) } +// LatestEvaluatorVersionNumber reports the newest registered version as an +// integer, or 0 when the evaluator is unknown or its versions are not numeric. +func (c *EvalClient) LatestEvaluatorVersionNumber( + ctx context.Context, + name string, + apiVersion string, +) int { + list, err := c.ListEvaluatorVersions(ctx, name, apiVersion) + if err != nil || list == nil || len(list.Value) == 0 { + return 0 + } + number, err := strconv.Atoi(pickLatestVersion(list.Value)) + if err != nil { + return 0 + } + return number +} + // DeleteEvaluatorVersion removes a single evaluator version. func (c *EvalClient) DeleteEvaluatorVersion( ctx context.Context, diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/codescript.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/codescript.go deleted file mode 100644 index 2c01e791f09..00000000000 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/codescript.go +++ /dev/null @@ -1,113 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package evalcore - -import ( - "fmt" - "os" - "path/filepath" - "regexp" - "strings" -) - -// pythonExt is the extension a code evaluator's source carries. It is also -// what tells a code evaluator apart from a rubric, which is `.json`. -const pythonExt = ".py" - -// GradeFunctionName is the function the executor calls. -// -// A code evaluator runs as an OpenAI python grader, and the grader contract is -// a single string of source with one entry point: a top-level -// `grade(sample, item)` returning a float. There is no module, package, or -// import path, so nothing else in the file can be reached. -const GradeFunctionName = "grade" - -// CodeEvaluatorScript is a validated evaluator script ready to publish. -type CodeEvaluatorScript struct { - // Name is the evaluator name the script is published under. - Name string - // Path is where the script was read from, kept for error messages. - Path string - // Source is the whole file, which is what the service is sent. The grader - // takes source, not a location: there is nowhere for a second file to go. - Source string -} - -// IsCodeEvaluatorSource reports whether a declared `source:` names a code -// evaluator rather than a rubric. -// -// The decision is made from the extension alone and never touches the -// filesystem, so it answers the same for a path that has not been created yet -// — a config can be validated before the file it names exists. -func IsCodeEvaluatorSource(path string) bool { - return strings.EqualFold(filepath.Ext(path), pythonExt) -} - -// gradeDeclaration matches a top-level `def grade(` — optionally async, and -// anchored at column zero. -// -// Indentation is what makes this specific rather than a substring search: a -// `grade` nested inside a class is a method, and the grader only ever calls a -// module-level function, so an indented match would pass validation here and -// then fail at run time with "top-level grade() function not found". -var gradeDeclaration = regexp.MustCompile( - `(?m)^(?:async[ \t]+)?def[ \t]+` + regexp.QuoteMeta(GradeFunctionName) + `[ \t]*\(`) - -// LoadCodeEvaluator reads an evaluator script and checks it against the grader -// contract. -// -// The check is done here rather than left to the service because the service -// only discovers a missing entry point when a run executes — long after a -// version has been published and an eval bound to it. The failure it reports -// then is "Invalid grader source: top-level grade() function not found in -// source", which names neither the file nor the evaluator. -func LoadCodeEvaluator(name, path string) (*CodeEvaluatorScript, error) { - if name == "" { - return nil, fmt.Errorf("an evaluator name is required to publish %q", path) - } - if path == "" { - return nil, fmt.Errorf("evaluator %q has no source file to publish", name) - } - - info, err := os.Stat(path) - if err != nil { - return nil, fmt.Errorf("reading evaluator source %q: %w", path, err) - } - if info.IsDir() { - return nil, fmt.Errorf( - "evaluator source %q is a directory. A code evaluator is a single %s file: "+ - "it is published as the source of a python grader, which takes one script "+ - "and cannot import a helper module beside it", path, pythonExt) - } - if !IsCodeEvaluatorSource(path) { - return nil, fmt.Errorf( - "evaluator source %q must be a %s file", path, pythonExt) - } - - source, err := os.ReadFile(path) - if err != nil { - return nil, fmt.Errorf("reading evaluator source %q: %w", path, err) - } - if len(strings.TrimSpace(string(source))) == 0 { - return nil, fmt.Errorf("evaluator source %q is empty", path) - } - if !gradeDeclaration.Match(source) { - return nil, fmt.Errorf( - "%s does not declare a top-level %s(sample, item) function. A code evaluator "+ - "runs as a python grader, which calls exactly that and nothing else — a "+ - "class, a differently named function, or one nested inside another will "+ - "not be found. For example:\n\ndef %s(sample, item) -> float:\n"+ - " return float(len(item.get(\"response\", \"\")))\n\n"+ - "The script must also be self-contained: only the standard library and "+ - "whatever the image named by --image-tag provides are importable, so a "+ - "helper file next to it cannot be imported", - path, GradeFunctionName, GradeFunctionName) - } - - return &CodeEvaluatorScript{ - Name: name, - Path: path, - Source: string(source), - }, nil -} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/codescript_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/codescript_test.go deleted file mode 100644 index 7cb75014828..00000000000 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/codescript_test.go +++ /dev/null @@ -1,128 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package evalcore - -import ( - "os" - "path/filepath" - "testing" - - "github.com/stretchr/testify/require" -) - -// gradeSource is the minimal script that satisfies the grader contract. -const gradeSource = `def grade(sample, item) -> float: - return float(len((item or {}).get("response", ""))) -` - -func writeFile(t *testing.T, root, rel, content string) string { - t.Helper() - path := filepath.Join(root, filepath.FromSlash(rel)) - require.NoError(t, os.MkdirAll(filepath.Dir(path), 0o755)) - require.NoError(t, os.WriteFile(path, []byte(content), 0o600)) - return path -} - -// The extension decides which definition type to publish from the source's -// extension alone, and it has to answer for a path that does not exist yet so -// a config can be validated before its files are written. -func TestIsCodeEvaluatorSource(t *testing.T) { - require.True(t, IsCodeEvaluatorSource("tone.py")) - require.True(t, IsCodeEvaluatorSource("evaluators/tone.py")) - require.True(t, IsCodeEvaluatorSource(`evaluators\tone.PY`), - "the extension is matched case-insensitively") - require.True(t, IsCodeEvaluatorSource("/absent/never/created.py"), - "classification must not touch the filesystem") - - require.False(t, IsCodeEvaluatorSource("rubric.json")) - require.False(t, IsCodeEvaluatorSource("evaluators/rubric.json")) - require.False(t, IsCodeEvaluatorSource("evaluator"), - "a folder is no longer a code evaluator; the grader takes one script") - require.False(t, IsCodeEvaluatorSource("tone.python")) - require.False(t, IsCodeEvaluatorSource("")) -} - -func TestLoadCodeEvaluator_AcceptsATopLevelGrade(t *testing.T) { - dir := t.TempDir() - path := writeFile(t, dir, "tone.py", gradeSource) - - script, err := LoadCodeEvaluator("tone", path) - require.NoError(t, err) - require.Equal(t, "tone", script.Name) - require.Equal(t, path, script.Path) - require.Equal(t, gradeSource, script.Source, - "the whole file is what the grader is sent") -} - -// The grader is handed source and calls grade(); an async definition is still -// a top-level grade(). -func TestLoadCodeEvaluator_AcceptsAsyncAndAnnotatedForms(t *testing.T) { - for label, source := range map[string]string{ - "async": "async def grade(sample, item) -> float:\n return 1.0\n", - "spaced": "def grade (sample, item):\n return 1.0\n", - "no-annot": "def grade(sample, item):\n return 1.0\n", - "after-code": "import json\n\n\ndef grade(sample, item):\n return 1.0\n", - } { - dir := t.TempDir() - path := writeFile(t, dir, "tone.py", source) - _, err := LoadCodeEvaluator("tone", path) - require.NoError(t, err, "for %s", label) - } -} - -// Without this the failure surfaces only when a run executes, as "Invalid -// grader source: top-level grade() function not found in source" — long after -// a version has been published and an eval bound to it. -func TestLoadCodeEvaluator_ReportsAMissingGrade(t *testing.T) { - dir := t.TempDir() - path := writeFile(t, dir, "tone.py", - "class ToneEvaluator:\n def __call__(self, **kwargs):\n return {\"result\": 1}\n") - - _, err := LoadCodeEvaluator("tone", path) - require.Error(t, err) - require.Contains(t, err.Error(), "grade(sample, item)") - require.Contains(t, err.Error(), path) -} - -// A grade() nested inside a class is a method. The grader only ever calls a -// module-level function, so an indented match must not pass validation. -func TestLoadCodeEvaluator_RejectsANestedGrade(t *testing.T) { - dir := t.TempDir() - path := writeFile(t, dir, "tone.py", - "class ToneEvaluator:\n def grade(self, sample, item):\n return 1.0\n") - - _, err := LoadCodeEvaluator("tone", path) - require.Error(t, err) - require.Contains(t, err.Error(), "top-level") -} - -func TestLoadCodeEvaluator_RejectsAFolder(t *testing.T) { - dir := t.TempDir() - writeFile(t, dir, "tone.py", gradeSource) - - _, err := LoadCodeEvaluator("tone", dir) - require.Error(t, err) - require.Contains(t, err.Error(), "single") -} - -func TestLoadCodeEvaluator_RejectsBadInput(t *testing.T) { - dir := t.TempDir() - - _, err := LoadCodeEvaluator("", writeFile(t, dir, "tone.py", gradeSource)) - require.Error(t, err, "a name is required to publish under") - - _, err = LoadCodeEvaluator("tone", "") - require.Error(t, err) - - _, err = LoadCodeEvaluator("tone", filepath.Join(dir, "absent.py")) - require.Error(t, err) - - _, err = LoadCodeEvaluator("tone", writeFile(t, dir, "rubric.json", "{}")) - require.Error(t, err, "a rubric is not a code evaluator") - require.Contains(t, err.Error(), ".py") - - _, err = LoadCodeEvaluator("tone", writeFile(t, dir, "empty.py", " \n\n")) - require.Error(t, err) - require.Contains(t, err.Error(), "empty") -} diff --git a/cli/azd/extensions/azure.ai.evaluations/tests/cli/evaluator_test.go b/cli/azd/extensions/azure.ai.evaluations/tests/cli/evaluator_test.go index 79eabdd69eb..19112c2c95c 100644 --- a/cli/azd/extensions/azure.ai.evaluations/tests/cli/evaluator_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/tests/cli/evaluator_test.go @@ -7,7 +7,6 @@ package cli import ( "os" - "path/filepath" "strings" "testing" @@ -60,83 +59,13 @@ func TestCLIJSONListsAreBareArrays(t *testing.T) { } } -// TestCLICodeEvaluatorRoundTrip drives the whole custom evaluator lifecycle -// through the command surface: create from a script, read it back, list its -// versions, then delete it. -func TestCLICodeEvaluatorRoundTrip(t *testing.T) { - name := uniqueName("azdcli_code") - script := writeGrader(t, lengthGrader) - - requireSuccess(t, run(t, "evaluator", "create", "--name", name, "--file", script)) - t.Cleanup(func() { - run(t, "evaluator", "delete", "--name", name, "--version", "1") - }) - - shown := requireSuccess(t, run(t, "evaluator", "show", "--name", name, "-o", "json")) - var def struct { - Name string `json:"name"` - Definition struct { - Type string `json:"type"` - CodeText string `json:"code_text"` - } `json:"definition"` - } - shown.JSON(t, &def) - require.Equal(t, "code", def.Definition.Type, - "a script must register as a code definition") - require.Contains(t, def.Definition.CodeText, "def grade", - "the script's source must round-trip in code_text") - - listed := requireSuccess(t, run(t, "evaluator", "list", "--name", name, "-o", "json")) - var versions []struct { - Name string `json:"name"` - Version string `json:"version"` - } - listed.JSON(t, &versions) - require.NotEmpty(t, versions, "the evaluator must list its own versions") -} - -// TestCLIEvaluatorSourcesAreMutuallyExclusive covers the validation a user is -// most likely to trip, and asserts it costs nothing to find out — no version -// is published on the way to the error. -func TestCLIEvaluatorSourcesAreMutuallyExclusive(t *testing.T) { - script := writeGrader(t, lengthGrader) - rubric := filepath.Join(t.TempDir(), "rubric.json") - require.NoError(t, os.WriteFile(rubric, - []byte(`{"dimensions":[{"id":"tone","description":"polite","weight":5}]}`), 0o600)) - - both := requireFailure(t, run(t, "evaluator", "create", - "--name", uniqueName("azdcli_both"), "--file", script, "--rubric", rubric)) - require.Contains(t, strings.ToLower(both.Combined()), "rubric", - "the error must name the flags in conflict") - - neither := requireFailure(t, run(t, "evaluator", "create", - "--name", uniqueName("azdcli_neither"))) - require.NotEmpty(t, strings.TrimSpace(neither.Combined()), - "refusing without a source must explain itself") -} - -// TestCLIGraderIsValidatedBeforePublishing asserts the check that saves a user -// from a late failure: a script with no top-level grade() is refused locally, -// because the executor would otherwise accept the publish and fail the run. -func TestCLIGraderIsValidatedBeforePublishing(t *testing.T) { - script := writeGrader(t, `class AnswerLengthEvaluator: - def __call__(self, **kwargs): - return {"result": 1.0} -`) - - r := requireFailure(t, run(t, "evaluator", "create", - "--name", uniqueName("azdcli_noglade"), "--file", script)) - require.Contains(t, strings.ToLower(r.Combined()), "grade", - "the refusal must name the function the executor looks for") -} - // TestCLIUnknownEvaluatorIsBrief covers the failure a user hits by typo. // // The service answers with a long JSON body. Printing it verbatim buries the // one useful sentence, so the CLI shortens it, and a regression here is the // kind that only shows up in someone's terminal. func TestCLIUnknownEvaluatorIsBrief(t *testing.T) { - r := requireFailure(t, run(t, "evaluator", "show", "--name", "azdcli-does-not-exist-9999")) + r := requireFailure(t, run(t, "evaluator", "show", "azdcli-does-not-exist-9999")) require.Less(t, len(r.Combined()), 600, "a not-found must stay short, not dump the service body:\n%s", r.Combined()) } diff --git a/cli/azd/extensions/azure.ai.evaluations/tests/cli/fixture_test.go b/cli/azd/extensions/azure.ai.evaluations/tests/cli/fixture_test.go index af745cae250..e159f28ff4a 100644 --- a/cli/azd/extensions/azure.ai.evaluations/tests/cli/fixture_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/tests/cli/fixture_test.go @@ -9,16 +9,15 @@ import ( "context" "encoding/json" "fmt" + "net/http" "os" "os/exec" - "path/filepath" "strings" "sync" "testing" "time" "azureaieval/internal/pkg/eval_api" - "azureaieval/internal/pkg/evalcore" "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" "github.com/Azure/azure-sdk-for-go/sdk/azidentity" @@ -26,34 +25,47 @@ import ( // The command tests need an eval that has already been run, and building one // through the CLI is not possible: there is no command that creates an eval -// from flags, only `run`, which needs a config file and a deployed target. -// So the fixture is built with the client and every assertion is made against -// the binary. What is under test is the command surface; the eval is scenery. +// from flags, only `run start`, which needs a config file and a deployed +// target. So the fixture is built with the client and every assertion is made +// against the binary. What is under test is the command surface; the eval is +// scenery. // // It is built once for the whole package because two completed runs cost // minutes, and torn down in TestMain rather than t.Cleanup so that whichever // test happened to trigger the build does not take the fixture away from the // rest. +// +// It evaluates an agent with a built-in evaluator, because that is all M1 can +// run: a deterministic code grader over a target-less dataset would score the +// rows predictably, but code evaluators and no-target runs are both M2. The +// cost is that pass and fail are decided by a judge, so no test may assert how +// many rows failed — only that filtering by verdict is self-consistent. const fixtureAPIVersion = "2025-11-15-preview" -// scoringGrader splits the rows deterministically. A grader that scores every -// row the same way makes --failed-only and a comparison indistinguishable from -// a no-op, so "good" is the difference between a pass and a failure. -const scoringGrader = `def grade(sample, item) -> float: - response = (item or {}).get("response", "") - return 1.0 if "good" in response else 0.0 -` +const defaultFixtureModel = "gpt-4o-mini" + +// fixtureQueries are answered by the agent under evaluation. They are ordinary +// support questions: the fixture proves the command surface, not the agent. +var fixtureQueries = []string{ + "How do I reset my password?", + "How do I change my billing address?", + "What are your support hours?", +} -// evalFixture is one eval with two completed runs of the same criterion. +// evalFixture is one eval with two completed runs. type evalFixture struct { EvaluatorName string EvalID string - // Baseline scores worse than Treatment, so a comparison between them has - // a delta to report rather than zero. - BaselineRunID string - TreatmentRunID string + // The agent the runs evaluate, so that a test needing a further run does + // not have to resolve one again. + AgentName string + + // Two runs of the same eval, so that listing, limiting and defaulting to + // the most recent all have something to distinguish. + FirstRunID string + SecondRunID string } var ( @@ -162,6 +174,72 @@ func sharedEval(t *testing.T) *evalFixture { return fixture } +func fixtureModel() string { + if model := os.Getenv("AZURE_AI_EVAL_MODEL"); model != "" { + return model + } + return defaultFixtureModel +} + +// resolveFixtureAgent names the agent the fixture evaluates. +// +// It reads /agents, not /assistants: they are different collections, and an +// eval target resolves against the former. Naming an assistant is accepted by +// the create and then fails the run with "resources not found". +func resolveFixtureAgent(ctx context.Context) (string, error) { + if name := os.Getenv("AZURE_AI_EVAL_AGENT"); name != "" { + return name, nil + } + + // Builds the shared credential if it does not exist yet; the token below + // comes from it. + if _, err := liveClient(); err != nil { + return "", err + } + + token, err := cred.GetToken(ctx, policy.TokenRequestOptions{ + Scopes: []string{"https://ai.azure.com/.default"}, + }) + if err != nil { + return "", fmt.Errorf("acquiring a token to list agents: %w", err) + } + + req, err := http.NewRequestWithContext( + ctx, http.MethodGet, endpoint+"/agents?api-version="+fixtureAPIVersion, nil) + if err != nil { + return "", err + } + req.Header.Set("Authorization", "Bearer "+token.Token) + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return "", fmt.Errorf("listing the project's agents: %w", err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return "", fmt.Errorf( + "listing the project's agents returned %d; set AZURE_AI_EVAL_AGENT to name one", + resp.StatusCode) + } + + var listing struct { + Data []struct { + Name string `json:"name"` + } `json:"data"` + } + if err := json.NewDecoder(resp.Body).Decode(&listing); err != nil { + return "", err + } + for _, a := range listing.Data { + if a.Name != "" { + return a.Name, nil + } + } + return "", fmt.Errorf( + "this project has no agent in /agents, so an agent-target run cannot be built; " + + "deploy an agent or set AZURE_AI_EVAL_AGENT") +} + func buildFixture(logf func(string, ...any)) (*evalFixture, error) { ctx := context.Background() @@ -170,40 +248,33 @@ func buildFixture(logf func(string, ...any)) (*evalFixture, error) { return nil, fmt.Errorf("acquiring an azd credential: %w", err) } - name := strings.ReplaceAll(uniqueName("azdclifx"), "-", "_") - script, err := publishScoringEvaluator(ctx, client, name) + agent, err := resolveFixtureAgent(ctx) if err != nil { return nil, err } - logf("published code evaluator %s version %s", name, script.Version) - - if err := awaitEvaluatorListed(ctx, client, name, script.Version); err != nil { - return nil, err - } + logf("evaluating agent %q", agent) - evalID, err := createFixtureEval(ctx, client, name) + evaluatorName := "builtin.task_adherence" + evalID, err := createFixtureEval(ctx, client, evaluatorName) if err != nil { return nil, err } logf("created eval %s", evalID) - // Different pass rates so the comparison has something to measure. - baseline, err := startFixtureRun(ctx, client, evalID, "baseline", - []string{"a bad answer", "another bad answer", "a good answer"}) + first, err := startFixtureRun(ctx, client, evalID, agent, "first") if err != nil { return nil, err } - treatment, err := startFixtureRun(ctx, client, evalID, "treatment", - []string{"a good answer", "another good answer", "a third good answer"}) + second, err := startFixtureRun(ctx, client, evalID, agent, "second") if err != nil { return nil, err } - logf("started runs %s and %s", baseline, treatment) + logf("started runs %s and %s", first, second) // Polled together: they are independent, and serialising them doubles the // slowest part of the suite for nothing. errs := make(chan error, 2) - for _, runID := range []string{baseline, treatment} { + for _, runID := range []string{first, second} { go func(id string) { errs <- awaitCompleted(ctx, client, evalID, id, logf) }(runID) } for range 2 { @@ -213,109 +284,50 @@ func buildFixture(logf func(string, ...any)) (*evalFixture, error) { } return &evalFixture{ - EvaluatorName: name, - EvalID: evalID, - BaselineRunID: baseline, - TreatmentRunID: treatment, + // The criterion is named without the builtin. prefix, and that is the + // name results are reported under. + EvaluatorName: strings.TrimPrefix(evaluatorName, "builtin."), + EvalID: evalID, + AgentName: agent, + FirstRunID: first, + SecondRunID: second, }, nil } -func publishScoringEvaluator( - ctx context.Context, - client *eval_api.EvalClient, - name string, -) (*eval_api.EvaluatorVersion, error) { - dir, err := os.MkdirTemp("", "azdcli-grader") - if err != nil { - return nil, err - } - defer os.RemoveAll(dir) - - path := filepath.Join(dir, name+".py") - if err := os.WriteFile(path, []byte(scoringGrader), 0o600); err != nil { - return nil, err - } - script, err := evalcore.LoadCodeEvaluator(name, path) - if err != nil { - return nil, fmt.Errorf("loading the grader: %w", err) - } - - // Without a data_schema the criteria builder has no mapping to derive, so - // the schema is what makes the evaluator usable rather than merely - // publishable. - opts := eval_api.CodeEvaluatorOptions{ - DataSchema: json.RawMessage( - `{"type":"object","properties":{"response":{"type":"string"}},"required":["response"]}`), - Metrics: json.RawMessage( - `{"result":{"type":"continuous","desirable_direction":"increase","is_primary":true}}`), - } - var version *eval_api.EvaluatorVersion - if err := retryCredentialFlake(func() error { - var err error - version, err = client.CreateCodeEvaluatorVersion(ctx, script, opts, fixtureAPIVersion) - return err - }); err != nil { - return nil, fmt.Errorf("publishing the code evaluator: %w", err) - } - deferTeardown(func() { - _ = client.DeleteEvaluatorVersion( - context.Background(), name, version.Version, fixtureAPIVersion) - }) - return version, nil -} - -// awaitEvaluatorListed waits for the version listing to catch up, which is the -// view eval creation resolves against. The direct read goes consistent first, -// so waiting on that alone still leaves the create failing with "was not -// found". -func awaitEvaluatorListed( - ctx context.Context, - client *eval_api.EvalClient, - name, version string, -) error { - deadline := time.Now().Add(2 * time.Minute) - for { - list, err := client.ListEvaluatorVersions(ctx, name, fixtureAPIVersion) - if err == nil && list != nil { - for _, entry := range list.Value { - if entry.Version == version { - return nil - } - } - } - if time.Now().After(deadline) { - return fmt.Errorf("evaluator %s version %s never appeared in the listing", name, version) - } - time.Sleep(500 * time.Millisecond) - } -} - func createFixtureEval( ctx context.Context, client *eval_api.EvalClient, evaluatorName string, ) (string, error) { - // Hand-written rather than built with buildEvalRequest, which is - // unexported. That is safe only because the evaluator is one published - // here whose schema is a single `response` column; a built-in would need - // the shipping builder, since their input contracts differ per evaluator. + criterionName := strings.TrimPrefix(evaluatorName, "builtin.") + var group *eval_api.OpenAIEval if err := retryCredentialFlake(func() error { var err error group, err = client.CreateOpenAIEval(ctx, &eval_api.CreateOpenAIEvalRequest{ Name: uniqueName("azdcli-fixture"), DataSourceConfig: &eval_api.DataSourceConfig{ - Type: "custom", + Type: "custom", + IncludeSampleSchema: true, ItemSchema: map[string]any{ "type": "object", - "properties": map[string]any{"response": map[string]any{"type": "string"}}, + "properties": map[string]any{"query": map[string]any{"type": "string"}}, }, }, TestingCriteria: []eval_api.TestingCriterion{{ Type: "azure_ai_evaluator", - Name: evaluatorName, + Name: criterionName, EvaluatorName: evaluatorName, - DataMapping: map[string]string{"response": "{{item.response}}"}, + DataMapping: map[string]string{ + "query": "{{item.query}}", + "response": "{{sample.output_items}}", + "tool_calls": "{{sample.tool_calls}}", + "tool_definitions": "{{sample.tool_definitions}}", + }, + InitializationParameters: map[string]any{ + "model": fixtureModel(), + "deployment_name": fixtureModel(), + }, }}, }) return err @@ -331,15 +343,14 @@ func createFixtureEval( func startFixtureRun( ctx context.Context, client *eval_api.EvalClient, - evalID, label string, - responses []string, + evalID, agentName, label string, ) (string, error) { - rows := make([]map[string]any, 0, len(responses)) - for _, r := range responses { - rows = append(rows, map[string]any{"response": r}) + rows := make([]map[string]any, 0, len(fixtureQueries)) + for _, q := range fixtureQueries { + rows = append(rows, map[string]any{"query": q}) } - ds := eval_api.NewDatasetOnlyDataSource() + ds := eval_api.NewAgentTargetDataSource(agentName, nil) ds.SetFileContent(rows) var run *eval_api.OpenAIEvalRun diff --git a/cli/azd/extensions/azure.ai.evaluations/tests/cli/harness_test.go b/cli/azd/extensions/azure.ai.evaluations/tests/cli/harness_test.go index dec143ce139..1557ea4aa0d 100644 --- a/cli/azd/extensions/azure.ai.evaluations/tests/cli/harness_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/tests/cli/harness_test.go @@ -204,16 +204,3 @@ func requireFailure(t *testing.T, r result) result { func uniqueName(prefix string) string { return fmt.Sprintf("%s_%d", prefix, time.Now().UnixNano()) } - -// writeGrader lays down a code evaluator script and returns its path. -func writeGrader(t *testing.T, body string) string { - t.Helper() - dir := t.TempDir() - path := filepath.Join(dir, "grader.py") - require.NoError(t, os.WriteFile(path, []byte(body), 0o600)) - return path -} - -const lengthGrader = `def grade(sample, item) -> float: - return float(len((item or {}).get("response", ""))) -` diff --git a/cli/azd/extensions/azure.ai.evaluations/tests/cli/results_test.go b/cli/azd/extensions/azure.ai.evaluations/tests/cli/results_test.go index 8660b5eda26..77313c677dc 100644 --- a/cli/azd/extensions/azure.ai.evaluations/tests/cli/results_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/tests/cli/results_test.go @@ -15,6 +15,12 @@ import ( "github.com/stretchr/testify/require" ) +// The fixture is judged by a model, so no test here may assert how many rows +// passed. What is under test is the command, and the properties that hold +// whatever the judge decided: every dataset row comes back, every row carries a +// verdict and a score, and filtering by verdict returns a subset that agrees +// with the totals. + // resultsPayload is what `results show -o json` emits: the run and the rows. type resultsPayload struct { Run struct { @@ -46,88 +52,95 @@ type resultsPayload struct { // TestCLIResultsShowRendersTheRows is the difference between `results show` and // `run show`: the totals say how many failed, these say which. -// -// The fixture's baseline run scores one row and fails two, so the rendering is -// checked against known verdicts rather than against whatever came back. func TestCLIResultsShowRendersTheRows(t *testing.T) { f := sharedEval(t) - r := requireSuccess(t, run(t, "results", "show", f.EvalID, "--run-id", f.BaselineRunID)) + r := requireSuccess(t, run(t, "results", "show", f.EvalID, "--run-id", f.FirstRunID)) - require.Contains(t, r.Stdout, f.BaselineRunID) - require.Contains(t, r.Stdout, "Totals: 1 passed, 2 failed, 0 errored") + require.Contains(t, r.Stdout, f.FirstRunID) + require.Contains(t, r.Stdout, "Totals:") require.Contains(t, r.Stdout, "CRITERION") require.Contains(t, r.Stdout, "ITEM") require.Contains(t, r.Stdout, "EVALUATOR") require.Contains(t, r.Stdout, "SCORE") require.Contains(t, r.Stdout, f.EvaluatorName) - // Both verdicts, and the row's own input: a table that showed only the - // counts would satisfy every assertion above. - require.Contains(t, r.Stdout, "FAIL") - require.Contains(t, r.Stdout, "pass") - require.Contains(t, r.Stdout, "response=a good answer") - require.Contains(t, r.Stdout, "response=a bad answer") + // The row's own input, so that a table printing only counts would not + // satisfy every assertion above. + require.Contains(t, r.Stdout, "query=") require.Contains(t, r.Stdout, "Report:") } func TestCLIResultsShowJSON(t *testing.T) { f := sharedEval(t) - r := requireSuccess(t, run(t, "results", "show", f.EvalID, - "--run-id", f.BaselineRunID, "-o", "json")) - - var payload resultsPayload - r.JSON(t, &payload) + payload := resultsFor(t, f.EvalID, f.FirstRunID) - require.Equal(t, f.BaselineRunID, payload.Run.ID) + require.Equal(t, f.FirstRunID, payload.Run.ID) require.Equal(t, "completed", payload.Run.Status) - require.Equal(t, 3, payload.Run.ResultCounts.Total) - require.Equal(t, 1, payload.Run.ResultCounts.Passed) - require.Equal(t, 2, payload.Run.ResultCounts.Failed) - require.Zero(t, payload.Run.ResultCounts.Errored) + require.Equal(t, len(fixtureQueries), payload.Run.ResultCounts.Total) + require.Zero(t, payload.Run.ResultCounts.Errored, + "an errored row means the fixture measured nothing") require.Len(t, payload.Run.PerTestingCriteria, 1) require.Equal(t, f.EvaluatorName, payload.Run.PerTestingCriteria[0].TestingCriteria) // The rows are the reason this command exists, and a run reporting counts // while returning none would still satisfy everything above. - require.Len(t, payload.OutputItems, 3, "every dataset row must come back as an item") + require.Len(t, payload.OutputItems, len(fixtureQueries), + "every dataset row must come back as an item") passed := 0 for _, item := range payload.OutputItems { - require.NotEmpty(t, item.DataSourceItem["response"], + require.NotEmpty(t, item.DataSourceItem["query"], "each row must carry the column it was evaluated on") require.Len(t, item.Results, 1) require.Equal(t, f.EvaluatorName, item.Results[0].Name) require.NotNil(t, item.Results[0].Score, "a scored row must report its score") if item.Results[0].Passed { passed++ - require.Equal(t, 1.0, *item.Results[0].Score) - } else { - require.Equal(t, 0.0, *item.Results[0].Score) } } - require.Equal(t, 1, passed, "the per-row verdicts must agree with the totals") + require.Equal(t, payload.Run.ResultCounts.Passed, passed, + "the per-row verdicts must agree with the totals") } // TestCLIResultsShowFailedOnly asserts the filter removes rows rather than // merely relabelling them. +// +// The service has no verdict filter — its `status` selects on execution status, +// so `status=failed` returns errored rows, not failing ones — which makes this +// entirely the CLI's own work and worth testing directly. func TestCLIResultsShowFailedOnly(t *testing.T) { f := sharedEval(t) + payload := resultsFor(t, f.EvalID, f.FirstRunID) + failed := payload.Run.ResultCounts.Failed + r := requireSuccess(t, run(t, "results", "show", f.EvalID, - "--run-id", f.BaselineRunID, "--failed-only")) + "--run-id", f.FirstRunID, "--failed-only")) + + if failed == 0 { + // Saying so is not the same as printing an empty table. + require.Contains(t, r.Stdout, "No failing rows.") + return + } - require.Contains(t, r.Stdout, "response=a bad answer") - require.NotContains(t, r.Stdout, "response=a good answer", + require.NotContains(t, r.Stdout, " pass ", "--failed-only must drop the rows that passed") + require.Contains(t, r.Stdout, "FAIL") + require.Equal(t, failed, strings.Count(r.Stdout, "FAIL"), + "every failing row must appear exactly once") +} - // The passing run has nothing to show, and saying so is not the same as - // printing an empty table. - empty := requireSuccess(t, run(t, "results", "show", f.EvalID, - "--run-id", f.TreatmentRunID, "--failed-only")) - require.Contains(t, empty.Stdout, "No failing rows.") +// resultsFor reads a run's results as JSON, which several tests need before +// they can decide what the rendered output should say. +func resultsFor(t *testing.T, evalID, runID string) resultsPayload { + t.Helper() + r := requireSuccess(t, run(t, "results", "show", evalID, "--run-id", runID, "-o", "json")) + var payload resultsPayload + r.JSON(t, &payload) + return payload } func TestCLIResultsExport(t *testing.T) { @@ -135,34 +148,35 @@ func TestCLIResultsExport(t *testing.T) { t.Run("json to stdout", func(t *testing.T) { r := requireSuccess(t, run(t, "results", "export", f.EvalID, - "--run-id", f.BaselineRunID, "--format", "json")) + "--run-id", f.FirstRunID, "--format", "json")) var exported struct { ID string `json:"id"` Status string `json:"status"` ResultCounts struct { + Total int `json:"total"` Passed int `json:"passed"` Failed int `json:"failed"` } `json:"result_counts"` } r.JSON(t, &exported) - require.Equal(t, f.BaselineRunID, exported.ID) + require.Equal(t, f.FirstRunID, exported.ID) require.Equal(t, "completed", exported.Status) - require.Equal(t, 1, exported.ResultCounts.Passed) - require.Equal(t, 2, exported.ResultCounts.Failed) + require.Equal(t, len(fixtureQueries), exported.ResultCounts.Total) }) t.Run("csv to stdout", func(t *testing.T) { r := requireSuccess(t, run(t, "results", "export", f.EvalID, - "--run-id", f.BaselineRunID, "--format", "csv")) + "--run-id", f.FirstRunID, "--format", "csv")) rows, err := csv.NewReader(strings.NewReader(r.Stdout)).ReadAll() require.NoError(t, err, "--format csv must emit parseable CSV:\n%s", r.Stdout) require.Len(t, rows, 2, "a header and one row per criterion") require.Equal(t, []string{"run_id", "status", "criterion", "passed", "failed"}, rows[0]) - require.Equal(t, - []string{f.BaselineRunID, "completed", f.EvaluatorName, "1", "2"}, rows[1]) + require.Equal(t, f.FirstRunID, rows[1][0]) + require.Equal(t, "completed", rows[1][1]) + require.Equal(t, f.EvaluatorName, rows[1][2]) }) t.Run("out-file writes the path instead of stdout", func(t *testing.T) { @@ -170,148 +184,25 @@ func TestCLIResultsExport(t *testing.T) { path := filepath.Join(dir, "results.csv") r := requireSuccess(t, runIn(t, dir, "results", "export", f.EvalID, - "--run-id", f.BaselineRunID, "--format", "csv", "-O", path)) + "--run-id", f.FirstRunID, "--format", "csv", "-O", path)) require.Empty(t, strings.TrimSpace(r.Stdout), "-O redirects the payload; leaving it on stdout too would double it") body, err := os.ReadFile(path) require.NoError(t, err) require.Contains(t, string(body), "run_id,status,criterion,passed,failed") - require.Contains(t, string(body), f.BaselineRunID) + require.Contains(t, string(body), f.FirstRunID) }) t.Run("an unknown format is refused", func(t *testing.T) { r := requireFailure(t, run(t, "results", "export", f.EvalID, - "--run-id", f.BaselineRunID, "--format", "xml")) + "--run-id", f.FirstRunID, "--format", "xml")) require.Contains(t, r.Combined(), "json or csv") }) } -// comparison is the shape `results compare -o json` emits. -type comparison struct { - State string `json:"state"` - Request struct { - EvalID string `json:"evalId"` - BaselineRunID string `json:"baselineRunId"` - TreatmentRunIDs []string `json:"treatmentRunIds"` - } `json:"request"` - Result struct { - Method string `json:"method"` - Comparisons []struct { - TestingCriteria string `json:"testingCriteria"` - Metric string `json:"metric"` - BaselineRunSummary struct { - RunID string `json:"runId"` - SampleCount int `json:"sampleCount"` - Average float64 `json:"average"` - } `json:"baselineRunSummary"` - CompareItems []struct { - TreatmentRunSummary struct { - RunID string `json:"runId"` - SampleCount int `json:"sampleCount"` - Average float64 `json:"average"` - } `json:"treatmentRunSummary"` - DeltaEstimate float64 `json:"deltaEstimate"` - TreatmentEffect string `json:"treatmentEffect"` - } `json:"compareItems"` - } `json:"comparisons"` - } `json:"result"` -} - -// TestCLIResultsCompare needs two completed runs of the same eval that scored -// differently, which is why the fixture seeds one run to fail two of three -// rows: comparing two identical runs reports a zero delta, and a comparison -// that computed nothing would look the same. -func TestCLIResultsCompare(t *testing.T) { - f := sharedEval(t) - - t.Run("rendered columns", func(t *testing.T) { - r := requireSuccess(t, run(t, "results", "compare", f.EvalID, - "--baseline", f.BaselineRunID, "--treatment", f.TreatmentRunID)) - - for _, header := range []string{ - "METRIC", "TREATMENT RUN", "BASELINE", "TREATMENT", "DELTA", "P-VALUE", "EFFECT", - } { - require.Containsf(t, r.Stdout, header, "the comparison table lost its %s column", header) - } - require.Contains(t, r.Stdout, "Method:") - require.Contains(t, r.Stdout, f.TreatmentRunID) - require.Contains(t, r.Stdout, f.EvaluatorName) - - // One in three against three in three. The delta is signed, which is - // the whole point of naming a baseline. - require.Contains(t, r.Stdout, "0.333") - require.Contains(t, r.Stdout, "1.000") - require.Contains(t, r.Stdout, "+0.667") - }) - - t.Run("json shape", func(t *testing.T) { - r := requireSuccess(t, run(t, "results", "compare", f.EvalID, - "--baseline", f.BaselineRunID, "--treatment", f.TreatmentRunID, "-o", "json")) - - var got comparison - r.JSON(t, &got) - - require.Equal(t, "Succeeded", got.State) - require.Equal(t, f.EvalID, got.Request.EvalID) - require.Equal(t, f.BaselineRunID, got.Request.BaselineRunID) - require.Equal(t, []string{f.TreatmentRunID}, got.Request.TreatmentRunIDs) - - require.NotEmpty(t, got.Result.Method) - require.Len(t, got.Result.Comparisons, 1) - c := got.Result.Comparisons[0] - require.Equal(t, f.EvaluatorName, c.TestingCriteria) - require.Equal(t, f.BaselineRunID, c.BaselineRunSummary.RunID) - require.Equal(t, 3, c.BaselineRunSummary.SampleCount) - require.InDelta(t, 1.0/3.0, c.BaselineRunSummary.Average, 0.001) - - require.Len(t, c.CompareItems, 1) - item := c.CompareItems[0] - require.Equal(t, f.TreatmentRunID, item.TreatmentRunSummary.RunID) - require.Equal(t, 1.0, item.TreatmentRunSummary.Average) - require.InDelta(t, 2.0/3.0, item.DeltaEstimate, 0.001) - require.NotEmpty(t, item.TreatmentEffect, - "the effect classifies the result, including when there are too few samples") - }) - - // Naming neither run is the common case — "did my last change help?" — so - // the defaults are asserted against the listing rather than against the - // fixture's own ids, which is what the command itself resolves from. - t.Run("defaults to the two most recent completed runs", func(t *testing.T) { - listed := requireSuccess(t, run(t, "run", "list", f.EvalID, "-o", "json")) - var runs []struct { - ID string `json:"id"` - Status string `json:"status"` - } - listed.JSON(t, &runs) - - completed := make([]string, 0, len(runs)) - for _, r := range runs { - if r.Status == "completed" { - completed = append(completed, r.ID) - } - } - require.GreaterOrEqual(t, len(completed), 2, - "comparing needs two completed runs of the same eval") - - r := requireSuccess(t, run(t, "results", "compare", f.EvalID, "-o", "json")) - var got comparison - r.JSON(t, &got) - - require.Equal(t, completed[0], got.Request.TreatmentRunIDs[0], - "the treatment defaults to the most recent completed run") - require.Equal(t, completed[1], got.Request.BaselineRunID, - "the baseline defaults to the one before it") - require.Equal(t, "Succeeded", got.State) - }) -} - -// TestCLIResultsUnknownEvalIsBrief covers the failure a user hits by typo. The -// service answers with a long JSON body; printing it verbatim buries the one -// useful sentence. func TestCLIResultsUnknownEvalIsBrief(t *testing.T) { - r := requireFailure(t, run(t, "results", "show", "eval_azdcli_does_not_exist")) - require.Less(t, len(r.Combined()), 600, - "a not-found must stay short, not dump the service body:\n%s", r.Combined()) - require.Contains(t, r.Combined(), "eval_azdcli_does_not_exist") + r := requireFailure(t, run(t, "results", "show", "eval_does_not_exist")) + require.Contains(t, r.Combined(), "eval_does_not_exist") + require.NotContains(t, r.Combined(), "RESPONSE 404") } diff --git a/cli/azd/extensions/azure.ai.evaluations/tests/cli/rubric_test.go b/cli/azd/extensions/azure.ai.evaluations/tests/cli/rubric_test.go index ebe5b49bd24..c104f369e46 100644 --- a/cli/azd/extensions/azure.ai.evaluations/tests/cli/rubric_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/tests/cli/rubric_test.go @@ -56,13 +56,13 @@ func TestCLIRubricRoundTrip(t *testing.T) { {"id":"accuracy","description":"Is the answer correct?","weight":10} ]`) - created := requireSuccess(t, run(t, "evaluator", "create", "--name", name, "--rubric", rubric)) + created := requireSuccess(t, run(t, "evaluator", "create", name, "--from-file", rubric)) require.Contains(t, created.Stdout, "version 1") t.Cleanup(func() { - run(t, "evaluator", "delete", "--name", name, "--version", "1") + run(t, "evaluator", "delete", name, "--version", "1") }) - shown := requireSuccess(t, run(t, "evaluator", "show", "--name", name)) + shown := requireSuccess(t, run(t, "evaluator", "show", name)) var doc evaluatorDocument shown.JSON(t, &doc) @@ -91,18 +91,18 @@ func TestCLIRubricRoundTrip(t *testing.T) { require.NotEmpty(t, doc.Definition.Metrics) require.NotEmpty(t, doc.SupportedEvaluationLevels) - // Every registration publishes a new immutable version, which is why there - // is no separate update command. - republished := requireSuccess(t, run(t, "evaluator", "create", "--name", name, "--rubric", rubric)) + // Every registration publishes a new immutable version, which is what + // `update` means for an evaluator. + republished := requireSuccess(t, run(t, "evaluator", "update", name, "--from-file", rubric)) require.Contains(t, republished.Stdout, "version 2", - "re-registering must advance the version rather than overwrite") + "updating must advance the version rather than overwrite") t.Cleanup(func() { - run(t, "evaluator", "delete", "--name", name, "--version", "2") + run(t, "evaluator", "delete", name, "--version", "2") }) // The earlier version stays reachable, which is what makes a published // version safe to reference from a config. - pinned := requireSuccess(t, run(t, "evaluator", "show", "--name", name, "--version", "1")) + pinned := requireSuccess(t, run(t, "evaluator", "show", name, "--version", "1")) var first evaluatorDocument pinned.JSON(t, &first) require.Equal(t, "1", first.Version) @@ -131,7 +131,7 @@ func TestCLIRubricWeightMustBeAnIntegerFromOneToTen(t *testing.T) { `[{"id":"tone","description":"Is the answer polite?","weight":`+tc.weight+`}]`) r := requireFailure(t, run(t, "evaluator", "create", - "--name", uniqueName("azdcli_badweight"), "--rubric", rubric)) + uniqueName("azdcli_badweight"), "--from-file", rubric)) require.Contains(t, r.Combined(), "between 1 and 10", "the refusal must say what a legal weight is") }) @@ -141,9 +141,9 @@ func TestCLIRubricWeightMustBeAnIntegerFromOneToTen(t *testing.T) { // weight rather than on the rubric shape they share. name := uniqueName("azdcli_goodweight") ok := writeRubric(t, `[{"id":"tone","description":"Is the answer polite?","weight":1}]`) - requireSuccess(t, run(t, "evaluator", "create", "--name", name, "--rubric", ok)) + requireSuccess(t, run(t, "evaluator", "create", name, "--from-file", ok)) t.Cleanup(func() { - run(t, "evaluator", "delete", "--name", name, "--version", "1") + run(t, "evaluator", "delete", name, "--version", "1") }) } @@ -154,25 +154,10 @@ func TestCLIRubricNeedsDimensions(t *testing.T) { require.NoError(t, os.WriteFile(path, []byte(`{"criteria":[]}`), 0o600)) r := requireFailure(t, run(t, "evaluator", "create", - "--name", uniqueName("azdcli_nodims"), "--rubric", path)) + uniqueName("azdcli_nodims"), "--from-file", path)) require.Contains(t, r.Combined(), "dimensions") } -// TestCLIRubricRejectsCodeOnlyFlags asserts the settings that would be -// accepted and then dropped are refused instead. A rubric runs no code and its -// schemas come from the service, so carrying them would be a silent no-op. -func TestCLIRubricRejectsCodeOnlyFlags(t *testing.T) { - rubric := writeRubric(t, `[{"id":"tone","description":"polite","weight":5}]`) - schema := filepath.Join(t.TempDir(), "schema.json") - require.NoError(t, os.WriteFile(schema, []byte(`{"type":"object"}`), 0o600)) - - r := requireFailure(t, run(t, "evaluator", "create", - "--name", uniqueName("azdcli_rubricflags"), "--rubric", rubric, - "--data-schema", schema)) - require.Contains(t, r.Combined(), "--data-schema") - require.Contains(t, r.Combined(), "--file") -} - // TestCLIEvaluatorShowAcceptsAFullDocument proves `evaluator show` emits JSON a // script can consume, whatever the definition kind. It renders the service's // body rather than a typed struct, so nothing else pins that it stays parseable. @@ -186,12 +171,12 @@ func TestCLIEvaluatorShowAcceptsAFullDocument(t *testing.T) { `{"name":"ignored","definition":{"dimensions":[{"id":"tone","description":"polite","weight":3}]}}`, ), 0o600)) - requireSuccess(t, run(t, "evaluator", "create", "--name", name, "--rubric", path)) + requireSuccess(t, run(t, "evaluator", "create", name, "--from-file", path)) t.Cleanup(func() { - run(t, "evaluator", "delete", "--name", name, "--version", "1") + run(t, "evaluator", "delete", name, "--version", "1") }) - shown := requireSuccess(t, run(t, "evaluator", "show", "--name", name)) + shown := requireSuccess(t, run(t, "evaluator", "show", name)) var raw map[string]any require.NoError(t, json.Unmarshal([]byte(shown.Stdout), &raw), "evaluator show must emit parseable JSON:\n%s", shown.Stdout) diff --git a/cli/azd/extensions/azure.ai.evaluations/tests/cli/run_ops_test.go b/cli/azd/extensions/azure.ai.evaluations/tests/cli/run_ops_test.go index 77628b90270..3f37b32462f 100644 --- a/cli/azd/extensions/azure.ai.evaluations/tests/cli/run_ops_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/tests/cli/run_ops_test.go @@ -32,9 +32,9 @@ func TestCLIRunList(t *testing.T) { for _, header := range []string{"RUN ID", "NAME", "STATUS", "RESULTS"} { require.Containsf(t, r.Stdout, header, "the listing lost its %s column", header) } - require.Contains(t, r.Stdout, f.BaselineRunID) - require.Contains(t, r.Stdout, f.TreatmentRunID) - require.Contains(t, r.Stdout, "1 passed, 2 failed, 0 errored", + require.Contains(t, r.Stdout, f.FirstRunID) + require.Contains(t, r.Stdout, f.SecondRunID) + require.Regexp(t, `\d+ passed, \d+ failed, \d+ errored`, r.Stdout, "the listing must summarise each run's counts, not just its status") }) @@ -51,11 +51,13 @@ func TestCLIRunList(t *testing.T) { for _, entry := range runs { byID[entry.ID] = entry } - baseline, ok := byID[f.BaselineRunID] + first, ok := byID[f.FirstRunID] require.True(t, ok, "the eval's own run is missing from its listing") - require.Equal(t, "completed", baseline.Status) - require.NotNil(t, baseline.ResultCounts) - require.Equal(t, 2, baseline.ResultCounts.Failed) + require.Equal(t, "completed", first.Status) + require.NotNil(t, first.ResultCounts) + require.Equal(t, len(fixtureQueries), + first.ResultCounts.Passed+first.ResultCounts.Failed, + "every dataset row must be accounted for by a verdict") }) // The client has always taken a limit; until recently the command did not @@ -79,11 +81,11 @@ func TestCLIRunShow(t *testing.T) { f := sharedEval(t) t.Run("by run id", func(t *testing.T) { - r := requireSuccess(t, run(t, "run", "show", f.EvalID, "--run-id", f.BaselineRunID)) - require.Contains(t, r.Stdout, f.BaselineRunID) + r := requireSuccess(t, run(t, "run", "show", f.EvalID, "--run-id", f.FirstRunID)) + require.Contains(t, r.Stdout, f.FirstRunID) require.Contains(t, r.Stdout, "status") require.Contains(t, r.Stdout, "completed") - require.Contains(t, r.Stdout, "1 passed, 2 failed, 0 errored") + require.Regexp(t, `\d+ passed, \d+ failed, \d+ errored`, r.Stdout) require.Contains(t, r.Stdout, "report") }) @@ -115,7 +117,7 @@ func TestCLIRunShow(t *testing.T) { r := requireFailure(t, run(t, "run", "show", f.EvalID, "--run-id", "evalrun_azdcli_nope")) require.Contains(t, r.Combined(), "evalrun_azdcli_nope", "the failure must name the run that was asked for") - require.NotContains(t, r.Combined(), f.BaselineRunID, + require.NotContains(t, r.Combined(), f.FirstRunID, "an explicit --run-id must not fall back to another run") }) } @@ -130,7 +132,7 @@ func TestCLIRunCancelAndDelete(t *testing.T) { f := sharedEval(t) t.Run("a finished run is refused", func(t *testing.T) { - r := requireFailure(t, run(t, "run", "cancel", f.EvalID, "--run-id", f.BaselineRunID)) + r := requireFailure(t, run(t, "run", "cancel", f.EvalID, "--run-id", f.FirstRunID)) require.Contains(t, r.Combined(), "already finished") require.Contains(t, r.Combined(), "completed") }) @@ -185,21 +187,17 @@ func TestCLIRunCancelAndDelete(t *testing.T) { // startCancellableRun adds a run to the fixture's eval and returns it before it // can finish. // -// The rows are padded so the run cannot complete inside the second it takes to -// issue the cancel; a run that finished first would turn the cancel test into -// an assertion about the guard it is not testing. +// An agent-target run invokes the agent once per row and is judged after that, +// which takes far longer than the second it takes to issue the cancel; a run +// that finished first would turn the cancel test into an assertion about the +// guard it is not testing. func startCancellableRun(t *testing.T, f *evalFixture) string { t.Helper() client, err := liveClient() require.NoError(t, err) - responses := make([]string, 0, 40) - for i := range 40 { - responses = append(responses, strings.Repeat("a good answer ", i%5+1)) - } - - runID, err := startFixtureRun(context.Background(), client, f.EvalID, "cancelme", responses) + runID, err := startFixtureRun(context.Background(), client, f.EvalID, f.AgentName, "cancelme") require.NoError(t, err, "starting a run to cancel") t.Cleanup(func() { _ = client.DeleteOpenAIEvalRun(context.Background(), f.EvalID, runID) diff --git a/cli/azd/extensions/azure.ai.evaluations/tests/live/code_evaluator_run_test.go b/cli/azd/extensions/azure.ai.evaluations/tests/live/code_evaluator_run_test.go deleted file mode 100644 index fbdfe1b2b87..00000000000 --- a/cli/azd/extensions/azure.ai.evaluations/tests/live/code_evaluator_run_test.go +++ /dev/null @@ -1,347 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -//go:build live - -package live - -import ( - "context" - "encoding/json" - "os" - "path/filepath" - "strings" - "testing" - "time" - - "azureaieval/internal/pkg/eval_api" - "azureaieval/internal/pkg/evalcore" - - "github.com/stretchr/testify/require" -) - -// These tests close the gap between publishing a code evaluator and using one. -// -// The publish path was verified on its own first, and passing that proves less -// than it appears to: a script publishes cleanly while carrying no -// data_schema, because only metrics are defaulted. Nothing then tells the -// caller that the evaluator cannot be wired into an eval. The criteria builder -// derives data_mapping from the schema the evaluator publishes, so no schema -// means no mapping, and the service refuses a criterion with none. That -// failure would surface at run time, long after the publish that caused it. -// -// So there are two tests. The first runs a code evaluator end to end and -// requires it to score a sample. The second publishes without a schema and -// records what the service actually does about it, rather than leaving the -// consequence to inference. - -// writeCodeEvaluator writes an evaluator that scores the length of a response. -// -// It is one self-contained script. A code evaluator runs as an OpenAI python -// grader, whose contract is a single Source string with one entry point: a -// top-level grade(sample, item). There is no package and no import path, so a -// helper module beside it could not be reached even if it were published. -func writeCodeEvaluator(t *testing.T, name string) string { - t.Helper() - dir := t.TempDir() - path := filepath.Join(dir, name+".py") - - source := `def grade(sample, item) -> float: - response = (item or {}).get("response", "") - return float(len(response)) -` - require.NoError(t, os.WriteFile(path, []byte(source), 0o600)) - return path -} - -// dataSchemaForResponse is the schema the criteria builder needs to derive a -// data_mapping. Only the caller can supply it: nothing about it is inferable -// from Python source. -func dataSchemaForResponse(t *testing.T) json.RawMessage { - t.Helper() - raw, err := json.Marshal(map[string]any{ - "type": "object", - "properties": map[string]any{"response": map[string]any{"type": "string"}}, - "required": []string{"response"}, - }) - require.NoError(t, err) - return raw -} - -// publishCodeEvaluator registers the script and returns the version, after -// confirming it is readable. -func publishCodeEvaluator( - t *testing.T, - env *liveEnv, - name string, - withSchema bool, -) *eval_api.EvaluatorVersion { - t.Helper() - ctx := context.Background() - - path := writeCodeEvaluator(t, name) - script, err := evalcore.LoadCodeEvaluator(name, path) - require.NoError(t, err, "loading the evaluator script") - - var opts eval_api.CodeEvaluatorOptions - if withSchema { - opts.DataSchema = dataSchemaForResponse(t) - opts.Metrics = json.RawMessage( - `{"result":{"type":"continuous","desirable_direction":"increase","is_primary":true}}`) - } - - version, err := env.evalClient.CreateCodeEvaluatorVersion(ctx, script, opts, projectAPIVersion) - require.NoError(t, err, "publishing the code evaluator") - t.Cleanup(func() { - _ = env.evalClient.DeleteEvaluatorVersion( - context.Background(), name, version.Version, projectAPIVersion) - }) - t.Logf("published code evaluator %s version %s", name, version.Version) - - awaitEvaluatorResolvable(t, env, name, version.Version) - return version -} - -// awaitEvaluatorResolvable waits for a published version the way the -// reconciler does, and reports how long each of the two views took. -// -// The numbers are the point. The direct read goes consistent almost at once -// while the listing lags it, and the eval-create resolver follows the slower -// one: a publish was observed reading back at 03:06:58 and still failing eval -// creation at 03:06:59. Logging both is what keeps the reconciler's tolerance -// honest instead of guessed, and asserting on the listing here is what proves -// the gate it waits on is the right one. -func awaitEvaluatorResolvable(t *testing.T, env *liveEnv, name, version string) { - t.Helper() - ctx := context.Background() - - start := time.Now() - var readable time.Duration - - for { - if readable == 0 { - if _, err := env.evalClient.GetEvaluatorRaw( - ctx, name, version, projectAPIVersion, - ); err == nil { - readable = time.Since(start) - t.Logf("evaluator %s readable after %s", - name, readable.Round(time.Millisecond)) - } - } - if readable != 0 && evaluatorVersionListed(ctx, env, name, version) { - t.Logf("evaluator %s listed after %s", - name, time.Since(start).Round(time.Millisecond)) - return - } - if time.Since(start) > 2*time.Minute { - t.Fatalf("evaluator %s never became resolvable", name) - } - time.Sleep(250 * time.Millisecond) - } -} - -func evaluatorVersionListed( - ctx context.Context, - env *liveEnv, - name, version string, -) bool { - list, err := env.evalClient.ListEvaluatorVersions(ctx, name, projectAPIVersion) - if err != nil || list == nil { - return false - } - for _, entry := range list.Value { - if entry.Version == version { - return true - } - } - return false -} - -// createEvalReferencing creates an eval naming a custom evaluator, tolerating -// the window in which the evaluator is published but not yet resolvable. -// -// The delay is reported so a run that hits it leaves evidence of how long it -// took, which is the number the reconciler's own tolerance has to be built on. -func createEvalReferencing( - t *testing.T, - env *liveEnv, - req *eval_api.CreateOpenAIEvalRequest, - within time.Duration, -) (*eval_api.OpenAIEval, error) { - t.Helper() - ctx := context.Background() - - start := time.Now() - for { - group, err := env.evalClient.CreateOpenAIEval(ctx, req) - if err == nil { - t.Logf("eval accepted the evaluator after %s", time.Since(start).Round(time.Millisecond)) - return group, nil - } - if !strings.Contains(strings.ToLower(err.Error()), "was not found") { - return nil, err - } - if time.Since(start) > within { - t.Logf("the evaluator was still unresolvable after %s", within) - return nil, err - } - time.Sleep(5 * time.Second) - } -} - -// TestLiveCodeEvaluatorScoresARun is the test the publish tests could not be: -// it requires the evaluator to actually run and return a score. -// -// No agent is involved. A code evaluator reads item fields, so the run uses a -// dataset-only source and the rows are supplied inline. That keeps the test -// about the evaluator rather than about a target being reachable. -func TestLiveCodeEvaluatorScoresARun(t *testing.T) { - env := setup(t) - ctx := context.Background() - - name := strings.ReplaceAll(uniqueName("azdcoderun"), "-", "_") - publishCodeEvaluator(t, env, name, true) - - group, err := createEvalReferencing(t, env, &eval_api.CreateOpenAIEvalRequest{ - Name: uniqueName("azd-code-eval"), - DataSourceConfig: &eval_api.DataSourceConfig{ - Type: "custom", - ItemSchema: map[string]any{ - "type": "object", - "properties": map[string]any{"response": map[string]any{"type": "string"}}, - }, - }, - TestingCriteria: []eval_api.TestingCriterion{{ - Type: "azure_ai_evaluator", - Name: name, - EvaluatorName: name, - DataMapping: map[string]string{"response": "{{item.response}}"}, - }}, - }, 3*time.Minute) - require.NoError(t, err, "creating an eval that references the code evaluator") - t.Logf("created eval %s", group.ID) - - ds := eval_api.NewDatasetOnlyDataSource() - ds.SetFileContent([]map[string]any{ - {"response": "a short answer"}, - {"response": "a considerably longer answer than the first one"}, - }) - - run, err := env.evalClient.CreateOpenAIEvalRun(ctx, group.ID, &eval_api.CreateOpenAIEvalRunRequest{ - Name: uniqueName("code-run"), - DataSource: ds, - }) - require.NoError(t, err, "starting the run") - t.Cleanup(func() { - _, _ = env.evalClient.CancelOpenAIEvalRun(context.Background(), group.ID, run.ID) - }) - t.Logf("started run %s", run.ID) - - final := awaitRun(t, env, group.ID, run.ID, 10*time.Minute) - - // Reaching a terminal state is not the same as having evaluated anything: - // a run whose every sample errors still reports completed. - require.Equal(t, "completed", strings.ToLower(final.Status), - "the run must complete rather than fail or cancel") - require.NotNil(t, final.ResultCounts, "a completed run must report counts") - require.Zero(t, final.ResultCounts.Errored, - "an errored sample means the code evaluator did not run") - require.Positive(t, final.ResultCounts.Passed+final.ResultCounts.Failed, - "the run must score at least one sample; scoring nothing means the rows "+ - "never reached the evaluator") -} - -// TestLiveCodeEvaluatorWithoutSchemaIsAccepted pins down what happens to a -// script published with no data_schema, which is the shape most people's -// their first evaluator will produce. -// -// It was expected to be refused. The reasoning was that the criteria builder -// derives data_mapping from the evaluator's schema, so no schema means no -// mapping, and the service rejects a criterion with none. An earlier run -// appeared to confirm it. That was wrong: the refusal was the propagation 404 -// in disguise, read as a mapping error because it arrived at the same call. -// With the publish properly gated, a schema-less evaluator is accepted and an -// empty data_mapping is allowed, so this is not the trap it looked like. -// -// The create is deliberately not retried. publishCodeEvaluator has already -// waited on the same condition the reconciler waits on, so a "was not found" -// here would mean that gate is the wrong one — which is worth failing on, -// because a retry would hide it. -func TestLiveCodeEvaluatorWithoutSchemaIsAccepted(t *testing.T) { - env := setup(t) - ctx := context.Background() - - name := strings.ReplaceAll(uniqueName("azdcodenoschema"), "-", "_") - published := publishCodeEvaluator(t, env, name, false) - require.NotEmpty(t, published.Version, - "a script with no schema still publishes; the schema is not required to register") - - _, err := env.evalClient.CreateOpenAIEval(ctx, &eval_api.CreateOpenAIEvalRequest{ - Name: uniqueName("azd-code-eval-noschema"), - DataSourceConfig: &eval_api.DataSourceConfig{ - Type: "custom", - ItemSchema: map[string]any{ - "type": "object", - "properties": map[string]any{"response": map[string]any{"type": "string"}}, - }, - }, - TestingCriteria: []eval_api.TestingCriterion{{ - Type: "azure_ai_evaluator", - Name: name, - EvaluatorName: name, - // Deliberately empty: this is what the criteria builder produces - // for an evaluator that publishes no data_schema. - DataMapping: map[string]string{}, - }}, - }) - - if err == nil { - t.Log("an evaluator with no data_schema was accepted with an empty data_mapping; " + - "publishing without a schema is not by itself a blocker") - return - } - require.NotContains(t, strings.ToLower(err.Error()), "was not found", - "the evaluator was published and waited for, so a not-found here means the "+ - "propagation gate the reconciler uses does not cover eval creation") - t.Logf("an evaluator with no data_schema was refused: %v", err) - require.Contains(t, strings.ToLower(err.Error()), "mapping", - "the refusal should name the mapping, so the CLI can explain it at publish time") -} - -// awaitRun polls until the run reaches a terminal state or the deadline passes. -func awaitRun( - t *testing.T, - env *liveEnv, - evalID string, - runID string, - within time.Duration, -) *eval_api.OpenAIEvalRun { - t.Helper() - ctx := context.Background() - - terminal := map[string]bool{ - "completed": true, "failed": true, "canceled": true, "cancelled": true, "error": true, - } - deadline := time.Now().Add(within) - for { - current, err := env.evalClient.GetOpenAIEvalRun(ctx, evalID, runID) - require.NoError(t, err, "polling the run") - if terminal[strings.ToLower(current.Status)] { - if current.ResultCounts != nil { - t.Logf("run %s reached %s: passed=%d failed=%d errored=%d", - runID, current.Status, - current.ResultCounts.Passed, - current.ResultCounts.Failed, - current.ResultCounts.Errored) - } - body, _ := json.MarshalIndent(current.PerTestingCriteria, "", " ") - t.Logf("per-criteria results: %s", string(body)) - return current - } - if time.Now().After(deadline) { - t.Fatalf("run %s did not finish within %s (last status %q)", - runID, within, current.Status) - } - time.Sleep(10 * time.Second) - } -} From 34f9c0c3405d9c9d848ea2b83ae6c52cfb24f324 Mon Sep 17 00:00:00 2001 From: mohessie Date: Mon, 3 Aug 2026 04:15:15 +0300 Subject: [PATCH 074/320] Leave M1 with the one target it ships Trace-backed, stored-response, model-target and target-less runs are all deferred, so their data sources, the flags that reached them, and the config that declared them come out. What remains is an agent target over a dataset, which is what the four hero scenarios use. An eval that names no target is now an error rather than a target-less run. It used to mean the rows already held both sides of the exchange, and that is exactly the deferred case, so accepting it silently would ship the feature through the back door. The code evaluator settings leave EvaluatorDecl for the same reason: image_tag, metrics, data_schema and init_parameters only ever fed a python grader. --- .../internal/cmd/build.go | 9 +-- .../azure.ai.evaluations/internal/cmd/run.go | 76 +----------------- .../internal/pkg/eval_api/models.go | 79 ------------------- .../internal/project/eval_config.go | 27 ++----- .../internal/project/service_target_eval.go | 3 - 5 files changed, 10 insertions(+), 184 deletions(-) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build.go index 6c938f32a70..d5008b57b9d 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build.go @@ -56,18 +56,11 @@ var sampleBindings = map[string]string{ // modelSampleBindings are what a model target produces. A model answers as // plain text and calls no tools, so binding an agent's richer output would // leave the evaluator waiting on fields the run never produces. -var modelSampleBindings = map[string]string{ - "response": "{{sample.output_text}}", -} - // sampleBindingsFor returns the run-time bindings a target of this kind can // satisfy. An empty target kind means nothing is invoked, so nothing is bound. func sampleBindingsFor(targetType string) map[string]string { - switch targetType { - case project.TargetTypeAgent: + if targetType == project.TargetTypeAgent { return sampleBindings - case project.TargetTypeModel: - return modelSampleBindings } return nil } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go index e6087ae0e38..b9ccd7d398a 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go @@ -54,11 +54,6 @@ func buildRunCommand(use, short string) *cobra.Command { runName string level string maxSamples int - fromTraces bool - traceWindow string - maxTraces int - responseIDs []string - maxTurns int wait bool endpointFlg string ) @@ -107,10 +102,6 @@ func buildRunCommand(use, short string) *cobra.Command { // target and dataset comes from the group's previous run. var dataSource *eval_api.EvalRunDataSource switch { - case len(responseIDs) > 0: - dataSource = eval_api.NewResponsesDataSource(responseIDs, maxTurns) - case fromTraces: - dataSource, err = buildTracesDataSource(ctx, ec, group, evalID, traceWindow, maxTraces) case group == nil: dataSource, err = ec.reuseDataSourceFromLastRun(ctx, evalID) default: @@ -188,16 +179,6 @@ func buildRunCommand(use, short string) *cobra.Command { "Scoring granularity: turn or conversation. Defaults to the service default (turn).") cmd.Flags().IntVar(&maxSamples, "max-samples", 0, "Cap the rows sent from the dataset.") - cmd.Flags().BoolVar(&fromTraces, "from-traces", false, - "Evaluate the agent's recorded traces instead of the dataset.") - cmd.Flags().StringVar(&traceWindow, "trace-window", "", - "How far back to read traces, for example 7d. Defaults to the service's window.") - cmd.Flags().IntVar(&maxTraces, "max-traces", 0, "Cap the traces evaluated.") - cmd.Flags().StringSliceVar(&responseIDs, "response-id", nil, - "Evaluate stored responses by id instead of the dataset; repeatable.") - cmd.Flags().IntVar(&maxTurns, "max-turns", 0, - "Turns of chat history to pull back per response. Defaults to the service's limit.") - cmd.MarkFlagsMutuallyExclusive("from-traces", "response-id") cmd.Flags().BoolVar(&wait, "wait", true, "Block until the run reaches a terminal state.") // The spec documents --no-wait, and cobra does not derive it from a bool. var noWait bool @@ -367,47 +348,6 @@ func (ec *evalContext) reuseDataSourceFromLastRun( return list.Data[0].DataSource, nil } -// buildTracesDataSource evaluates what the agent has already done, rather than -// asking it fresh questions from a dataset. -// -// The service reads the traces from Application Insights, so the agent has to -// be emitting gen_ai.input.messages / gen_ai.output.messages for anything to be -// found; when it is not, the run fails with the service saying so. -func buildTracesDataSource( - ctx context.Context, - ec *evalContext, - group *project.Eval, - evalID, window string, - maxTraces int, -) (*eval_api.EvalRunDataSource, error) { - agent := "" - switch { - case group != nil && group.Target != nil: - agent = group.Target.Name - default: - // With --eval-id there is no config, so the agent comes from whatever - // the group ran against last. - last, err := ec.reuseDataSourceFromLastRun(ctx, evalID) - if err != nil { - return nil, err - } - if last != nil && last.Target != nil { - agent = last.Target.Name - } - } - if agent == "" { - return nil, fmt.Errorf( - "--from-traces needs to know whose traces to read, and the eval does not " + - "name an agent. Declare target.type: agent on the eval") - } - - var lookbackHours int - if days := parseWindowDays(window); days > 0 { - lookbackHours = days * 24 - } - return eval_api.NewTracesDataSource(agent, lookbackHours, time.Time{}, maxTraces), nil -} - // buildRunDataSource binds the dataset to the run. The eval carries no // dataset today, so it is supplied here. func (ec *evalContext) buildRunDataSource( @@ -419,20 +359,12 @@ func (ec *evalContext) buildRunDataSource( if group == nil { return nil, fmt.Errorf("no eval to run") } - - // A group with no target scores a dataset that already holds the exchange, - // so there is nothing to invoke. That is how recorded conversations are - // evaluated. - var ds *eval_api.EvalRunDataSource - switch { - case group.Target == nil || group.Target.Name == "": - ds = eval_api.NewDatasetOnlyDataSource() - case group.Target.Type == project.TargetTypeModel: - ds = eval_api.NewModelTargetDataSource(group.Target.Name) - default: - ds = eval_api.NewAgentTargetDataSource(group.Target.Name, nil) + if group.Target == nil || group.Target.Name == "" { + return nil, fmt.Errorf("eval %q does not name a target agent", group.Name) } + ds := eval_api.NewAgentTargetDataSource(group.Target.Name, nil) + if group.Dataset == "" { return nil, fmt.Errorf("eval %q does not reference a dataset", group.Name) } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/models.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/models.go index 2b9fa2ca9d1..6441568db72 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/models.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/models.go @@ -8,7 +8,6 @@ import ( "fmt" "sort" "strings" - "time" ) // --------------------------------------------------------------------------- @@ -422,84 +421,6 @@ func NewAgentTargetDataSource(agentName string, agentVersion *string) *EvalRunDa } } -// NewTracesDataSource evaluates an agent's recorded traces instead of a dataset. -// -// The window is a lookback in hours. The service's own field is -// `lookback_hours` and it has no start bound: a `start_time` is accepted and -// dropped, leaving the default seven days in place, so the conversion happens -// here rather than being left to look like it worked. -func NewTracesDataSource(agentName string, lookbackHours int, end time.Time, maxTraces int) *EvalRunDataSource { - ds := &EvalRunDataSource{ - Type: EvalRunDataSourceTypeTraces, - AgentName: agentName, - LookbackHours: lookbackHours, - MaxTraces: maxTraces, - } - if !end.IsZero() { - ds.EndTime = end.Unix() - } - return ds -} - -// NewDatasetOnlyDataSource scores the dataset as it stands, invoking nothing. -// -// Used when a group declares no target: the rows already hold both sides of -// the exchange, which is how a recorded conversation is evaluated. -func NewDatasetOnlyDataSource() *EvalRunDataSource { - return &EvalRunDataSource{Type: EvalRunDataSourceTypeJSONL} -} - -// NewModelTargetDataSource sends the dataset's questions straight to a model -// deployment, with no agent in front of it. -// -// The model answers as plain text, so a group evaluating one has to bind its -// response to {{sample.output_text}} rather than the richer output an agent -// produces. -func NewModelTargetDataSource(model string) *EvalRunDataSource { - return &EvalRunDataSource{ - Type: EvalRunDataSourceTypeAgentTarget, - InputMessages: &EvalRunInputMessages{ - Type: "template", - Template: []EvalRunMessageTemplate{ - { - Role: "user", - Content: "{{item.query}}", - Type: "message", - }, - }, - }, - Target: &EvalRunTarget{ - Type: "azure_ai_model", - Model: model, - }, - } -} - -// NewResponsesDataSource evaluates responses the project already stored. -// -// The ids travel as ordinary JSONL rows and a data_mapping points the service -// at the field holding each one, which is how it retrieves the chat history -// behind the response. -func NewResponsesDataSource(responseIDs []string, maxTurns int) *EvalRunDataSource { - rows := make([]map[string]any, 0, len(responseIDs)) - for _, id := range responseIDs { - rows = append(rows, map[string]any{"item": map[string]any{"response_id": id}}) - } - - return &EvalRunDataSource{ - Type: EvalRunDataSourceTypeResponses, - ItemGenerationParams: &ItemGenerationParams{ - Type: "response_retrieval", - MaxNumTurns: maxTurns, - DataMapping: map[string]string{"response_id": "{{item.response_id}}"}, - Source: &EvalRunDataContent{ - Type: EvalRunDataContentTypeFileContent, - Content: rows, - }, - }, - } -} - // SetFileContent sets the data source to use inline file content. func (ds *EvalRunDataSource) SetFileContent(items []map[string]any) { ds.Source = &EvalRunDataContent{ diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go index 76b0d4bf7cb..0c75f38fdf0 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go @@ -34,23 +34,12 @@ type DatasetDecl struct { // EvaluatorDecl declares a custom evaluator. Built-ins are referenced directly // from an eval and never declared here. // -// Source decides which kind of evaluator this is, by extension: a `.py` file is -// a single self-contained Python script and publishes a code evaluator, a -// `.json` file holds a rubric. A code evaluator cannot name a folder — it runs -// as a python grader, which is handed one script's source and cannot import a -// helper module beside it. +// Source names a `.json` file holding a rubric: a list of weighted scoring +// dimensions. type EvaluatorDecl struct { Name string `yaml:"name" json:"name"` Source string `yaml:"source,omitempty" json:"source,omitempty"` Version string `yaml:"version,omitempty" json:"version,omitempty"` - - // Code evaluators only. The three schema fields name JSON files beside the - // script, resolved like Source, because they are edited as files rather - // than written inline in YAML. - ImageTag string `yaml:"image_tag,omitempty" json:"image_tag,omitempty"` - Metrics string `yaml:"metrics,omitempty" json:"metrics,omitempty"` - DataSchema string `yaml:"data_schema,omitempty" json:"data_schema,omitempty"` - InitParameters string `yaml:"init_parameters,omitempty" json:"init_parameters,omitempty"` } // Eval is a run definition: evaluators plus options, bound to a dataset. @@ -72,11 +61,6 @@ type Target struct { const TargetTypeAgent = "agent" -// TargetTypeModel evaluates a model deployment directly, with no agent in -// front of it. A model answers as plain text, so a group targeting one binds -// its response differently from a group targeting an agent. -const TargetTypeModel = "model" - // Options are run settings carried on the group. type Options struct { EvalModel string `yaml:"eval_model,omitempty" json:"eval_model,omitempty"` @@ -175,11 +159,10 @@ func (c *EvalConfig) Validate() error { i, g.Name, ref.Name, evalcore.BuiltinPrefix) } } - if g.Target != nil && g.Target.Type != "" && - g.Target.Type != TargetTypeAgent && g.Target.Type != TargetTypeModel { + if g.Target != nil && g.Target.Type != "" && g.Target.Type != TargetTypeAgent { return fmt.Errorf( - "evals[%d] (%s): target.type %q is not supported; use %q or %q", - i, g.Name, g.Target.Type, TargetTypeAgent, TargetTypeModel) + "evals[%d] (%s): target.type %q is not supported; use %q", + i, g.Name, g.Target.Type, TargetTypeAgent) } if g.Options != nil { switch g.Options.EvaluationLevel { diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go index 5f6786e954e..f4902f236a3 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go @@ -162,9 +162,6 @@ func (p *EvalServiceTargetProvider) Deploy( for _, decl := range cfg.Evaluators { report(progress, fmt.Sprintf("Reconciling evaluator %s", decl.Name)) localPath := resolveSource(baseDir, decl.Source) - decl.Metrics = resolveSource(baseDir, decl.Metrics) - decl.DataSchema = resolveSource(baseDir, decl.DataSchema) - decl.InitParameters = resolveSource(baseDir, decl.InitParameters) version, changed, err := reconciler.EnsureEvaluator(ctx, decl, localPath) if err != nil { return nil, fmt.Errorf("evaluator %q: %w", decl.Name, err) From 4bbcd923198641dca6a137cbd0a78375066f0751 Mon Sep 17 00:00:00 2001 From: mohessie Date: Mon, 3 Aug 2026 04:28:52 +0300 Subject: [PATCH 075/320] Give datasets the same surface as evaluators Name positionally, content through --from-file, and versions under their own subcommand so that list means the project's datasets rather than one dataset's history. create and update split the same upload the way they do for evaluators. Also drops a stray copy of eval_api/schedules.go from the working tree. It was left behind by the checkout that built the M2 branch, was never tracked here, and had been compiling into M1 ever since - dead code that M1 does not call and that its own history does not contain. --- .../internal/cmd/dataset.go | 178 +++++++++++------- .../tests/cli/dataset_test.go | 62 +++--- 2 files changed, 149 insertions(+), 91 deletions(-) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset.go index 9ea7c5b3a48..daecc8c3d05 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset.go @@ -23,52 +23,51 @@ func newDatasetCommand() *cobra.Command { } cmd.AddCommand( newDatasetCreateCommand(), + newDatasetUpdateCommand(), newDatasetListCommand(), newDatasetShowCommand(), newDatasetDeleteCommand(), + newDatasetVersionsCommand(), ) return cmd } -// newDatasetCreateCommand builds `dataset create`. -// -// There is no separate `update`: every registration publishes a new immutable -// version and the server auto-increments, so `create` covers both the first -// version and every later one. +// newDatasetCreateCommand builds `dataset create `, which registers a +// dataset that does not exist yet. func newDatasetCreateCommand() *cobra.Command { + return newDatasetWriteCommand("create", "Register a dataset, publishing its first version.") +} + +// newDatasetUpdateCommand builds `dataset update `, which publishes a +// further version of one that does. +func newDatasetUpdateCommand() *cobra.Command { + return newDatasetWriteCommand("update", "Publish a new version of a dataset.") +} + +// newDatasetWriteCommand builds create and update. Both run the same upload, +// and the existence check is the only thing that separates them: a version is +// brought into being by startPendingUpload, which neither knows nor cares +// whether the name was already in use. +func newDatasetWriteCommand(verb, short string) *cobra.Command { var ( - name string - file string + fromFile string version string endpointFlg string ) - use := "create" - short := "Register a dataset, publishing a new version." - cmd := &cobra.Command{ - Use: use, + Use: verb + " ", Short: short, + Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { - if name == "" { - return requireFlag("name") - } - if file == "" { - return requireFlag("file") + name := args[0] + if fromFile == "" { + return requireFlag("from-file") } - info, err := os.Stat(file) + localDir, err := datasetUploadDir(fromFile) if err != nil { - return fmt.Errorf("reading --file %q: %w", file, err) - } - // The upload helper scans a directory for the first .jsonl, so pass - // the containing directory when given a file path. - localDir := file - if !info.IsDir() { - if !strings.EqualFold(filepath.Ext(file), ".jsonl") { - return fmt.Errorf("--file must be a .jsonl file or a directory containing one, got %q", file) - } - localDir = filepath.Dir(file) + return err } ctx := cmd.Context() @@ -78,6 +77,14 @@ func newDatasetCreateCommand() *cobra.Command { } defer ec.Close() + existing, err := ec.datasetClient.ListDatasetVersions( + ctx, name, ProjectEndpointAPIVersion, + ) + exists := err == nil && existing != nil && len(existing.Value) > 0 + if err := checkAssetExistence(verb, "dataset", name, exists); err != nil { + return err + } + ds, err := ec.datasetClient.UploadNextVersion( ctx, name, version, localDir, ProjectEndpointAPIVersion, ) @@ -103,23 +110,37 @@ func newDatasetCreateCommand() *cobra.Command { }, } - cmd.Flags().StringVar(&name, "name", "", "Name of the dataset.") - cmd.Flags().StringVar(&file, "file", "", "Path to a .jsonl file, or a directory containing one.") + cmd.Flags().StringVar(&fromFile, "from-file", "", + "Path to a .jsonl file, or a directory containing one.") cmd.Flags().StringVar(&version, "version", "", "Current version to increment from. Omit to increment from the latest registered version.") cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") return cmd } +// datasetUploadDir resolves what was named into the directory the upload scans. +func datasetUploadDir(path string) (string, error) { + info, err := os.Stat(path) + if err != nil { + return "", fmt.Errorf("reading --from-file %q: %w", path, err) + } + if info.IsDir() { + return path, nil + } + if !strings.EqualFold(filepath.Ext(path), ".jsonl") { + return "", fmt.Errorf( + "--from-file must be a .jsonl file or a directory containing one, got %q", path) + } + return filepath.Dir(path), nil +} + func newDatasetListCommand() *cobra.Command { - var ( - name string - endpointFlg string - ) + var endpointFlg string cmd := &cobra.Command{ Use: "list", - Short: "List registered datasets, or the versions of one dataset.", + Short: "List the project's datasets.", + Args: cobra.NoArgs, RunE: func(cmd *cobra.Command, args []string) error { ctx := cmd.Context() ec, err := newEvalContext(ctx, endpointFlg) @@ -128,50 +149,85 @@ func newDatasetListCommand() *cobra.Command { } defer ec.Close() - var list *dataset_api.DatasetList - if name != "" { - list, err = ec.datasetClient.ListDatasetVersions(ctx, name, ProjectEndpointAPIVersion) - } else { - list, err = ec.datasetClient.ListDatasets(ctx, ProjectEndpointAPIVersion) - } + list, err := ec.datasetClient.ListDatasets(ctx, ProjectEndpointAPIVersion) if err != nil { return fmt.Errorf("listing datasets: %w", err) } + return renderDatasets(cmd, list) + }, + } - if isJSON(cmd) { - return emitJSONList(cmd.OutOrStdout(), list.Value) - } - rows := make([][]string, 0, len(list.Value)) - for _, d := range list.Value { - rows = append(rows, []string{d.Name, d.Version, d.Format}) + cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") + return cmd +} + +// newDatasetVersionsCommand groups the version listing, so that `list` means +// the assets rather than the history of one of them. +func newDatasetVersionsCommand() *cobra.Command { + cmd := &cobra.Command{ + Use: "versions", + Short: "Inspect the versions of one dataset.", + } + cmd.AddCommand(newDatasetVersionsListCommand()) + return cmd +} + +func newDatasetVersionsListCommand() *cobra.Command { + var endpointFlg string + + cmd := &cobra.Command{ + Use: "list ", + Short: "List the versions of a dataset.", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + name := args[0] + + ctx := cmd.Context() + ec, err := newEvalContext(ctx, endpointFlg) + if err != nil { + return err } - if len(rows) == 0 { - fmt.Fprintln(cmd.OutOrStdout(), "No datasets found.") - return nil + defer ec.Close() + + list, err := ec.datasetClient.ListDatasetVersions(ctx, name, ProjectEndpointAPIVersion) + if err != nil { + return fmt.Errorf("listing versions of dataset %q: %w", name, err) } - return emitTable(cmd.OutOrStdout(), []string{"NAME", "VERSION", "FORMAT"}, rows) + return renderDatasets(cmd, list) }, } - cmd.Flags().StringVar(&name, "name", "", "Limit the listing to versions of this dataset.") cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") return cmd } +func renderDatasets(cmd *cobra.Command, list *dataset_api.DatasetList) error { + if isJSON(cmd) { + return emitJSONList(cmd.OutOrStdout(), list.Value) + } + rows := make([][]string, 0, len(list.Value)) + for _, d := range list.Value { + rows = append(rows, []string{d.Name, d.Version, d.Format}) + } + if len(rows) == 0 { + fmt.Fprintln(cmd.OutOrStdout(), "No datasets found.") + return nil + } + return emitTable(cmd.OutOrStdout(), []string{"NAME", "VERSION", "FORMAT"}, rows) +} + func newDatasetShowCommand() *cobra.Command { var ( - name string version string endpointFlg string ) cmd := &cobra.Command{ - Use: "show", + Use: "show ", Short: "Show a dataset version.", + Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { - if name == "" { - return requireFlag("name") - } + name := args[0] ctx := cmd.Context() ec, err := newEvalContext(ctx, endpointFlg) @@ -211,7 +267,6 @@ func newDatasetShowCommand() *cobra.Command { }, } - cmd.Flags().StringVar(&name, "name", "", "Name of the dataset.") cmd.Flags().StringVar(&version, "version", "", "Version to show. Omit for the latest.") cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") return cmd @@ -219,18 +274,16 @@ func newDatasetShowCommand() *cobra.Command { func newDatasetDeleteCommand() *cobra.Command { var ( - name string version string endpointFlg string ) cmd := &cobra.Command{ - Use: "delete", + Use: "delete ", Short: "Delete a dataset version.", + Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { - if name == "" { - return requireFlag("name") - } + name := args[0] if version == "" { return requireFlag("version") } @@ -262,7 +315,6 @@ func newDatasetDeleteCommand() *cobra.Command { }, } - cmd.Flags().StringVar(&name, "name", "", "Name of the dataset.") cmd.Flags().StringVar(&version, "version", "", "Version to delete.") cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") return cmd diff --git a/cli/azd/extensions/azure.ai.evaluations/tests/cli/dataset_test.go b/cli/azd/extensions/azure.ai.evaluations/tests/cli/dataset_test.go index 3b0dbeb1856..d82415517fc 100644 --- a/cli/azd/extensions/azure.ai.evaluations/tests/cli/dataset_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/tests/cli/dataset_test.go @@ -59,9 +59,15 @@ func registerDataset(t *testing.T, versions int) *registeredDataset { require.NoError(t, os.WriteFile(path, []byte(datasetRows), 0o600)) ds := ®isteredDataset{Name: uniqueName("azdcli_ds")} - for range versions { - r := requireSuccess(t, run(t, "dataset", "create", - "--name", ds.Name, "--file", path, "-o", "json")) + for i := range versions { + // The first publish is a create; every later one is an update, which is + // the only difference between them. + verb := "update" + if i == 0 { + verb = "create" + } + r := requireSuccess(t, run(t, "dataset", verb, + ds.Name, "--from-file", path, "-o", "json")) var created datasetSummary r.JSON(t, &created) @@ -70,12 +76,12 @@ func registerDataset(t *testing.T, versions int) *registeredDataset { version := created.Version deferTeardown(func() { - runQuietly("dataset", "delete", "--name", ds.Name, "--version", version) + runQuietly("dataset", "delete", ds.Name, "--version", version) }) } require.Len(t, ds.Versions, versions) require.NotEqual(t, ds.Versions[0], ds.Versions[len(ds.Versions)-1], - "re-registering must advance the version rather than overwrite") + "updating must advance the version rather than overwrite") return ds } @@ -83,17 +89,17 @@ func TestCLIDatasetList(t *testing.T) { ds := sharedDataset(t) t.Run("table", func(t *testing.T) { - r := requireSuccess(t, run(t, "dataset", "list", "--name", ds.Name)) + r := requireSuccess(t, run(t, "dataset", "versions", "list", ds.Name)) for _, header := range []string{"NAME", "VERSION", "FORMAT"} { require.Containsf(t, r.Stdout, header, "the listing lost its %s column", header) } require.Contains(t, r.Stdout, ds.Name) }) - // --name is what makes the listing usable once a project holds more than a - // screenful: it narrows to one dataset's versions. - t.Run("name scopes the listing to one dataset's versions", func(t *testing.T) { - r := requireSuccess(t, run(t, "dataset", "list", "--name", ds.Name, "-o", "json")) + // `versions list` is what makes the listing usable once a project holds more + // than a screenful: it narrows to one dataset's versions. + t.Run("versions list scopes to one dataset's versions", func(t *testing.T) { + r := requireSuccess(t, run(t, "dataset", "versions", "list", ds.Name, "-o", "json")) var listed []datasetSummary r.JSON(t, &listed) require.NotEmpty(t, listed) @@ -101,7 +107,7 @@ func TestCLIDatasetList(t *testing.T) { seen := map[string]bool{} for _, v := range listed { require.Equalf(t, ds.Name, v.Name, - "--name must return only that dataset's versions; got %q", v.Name) + "the listing must return only that dataset's versions; got %q", v.Name) seen[v.Version] = true } for _, want := range ds.Versions { @@ -127,8 +133,8 @@ func TestCLIDatasetList(t *testing.T) { }) t.Run("an unknown name lists nothing rather than failing", func(t *testing.T) { - r := requireSuccess(t, run(t, "dataset", "list", - "--name", "azdcli-no-such-dataset", "-o", "json")) + r := requireSuccess(t, run(t, "dataset", "versions", "list", + "azdcli-no-such-dataset", "-o", "json")) var listed []datasetSummary r.JSON(t, &listed) require.Empty(t, listed) @@ -142,7 +148,7 @@ func TestCLIDatasetShow(t *testing.T) { // Omitting the version means the latest, which is the only sensible // default for a name that gains a version on every registration. t.Run("defaults to the latest version", func(t *testing.T) { - r := requireSuccess(t, run(t, "dataset", "show", "--name", ds.Name, "-o", "json")) + r := requireSuccess(t, run(t, "dataset", "show", ds.Name, "-o", "json")) var shown datasetSummary r.JSON(t, &shown) require.Equal(t, ds.Name, shown.Name) @@ -151,7 +157,7 @@ func TestCLIDatasetShow(t *testing.T) { t.Run("version pins an earlier one", func(t *testing.T) { r := requireSuccess(t, run(t, "dataset", "show", - "--name", ds.Name, "--version", ds.Versions[0], "-o", "json")) + ds.Name, "--version", ds.Versions[0], "-o", "json")) var shown datasetSummary r.JSON(t, &shown) require.Equal(t, ds.Versions[0], shown.Version) @@ -159,7 +165,7 @@ func TestCLIDatasetShow(t *testing.T) { }) t.Run("table", func(t *testing.T) { - r := requireSuccess(t, run(t, "dataset", "show", "--name", ds.Name)) + r := requireSuccess(t, run(t, "dataset", "show", ds.Name)) for _, header := range []string{"NAME", "VERSION", "FORMAT", "URI"} { require.Containsf(t, r.Stdout, header, "the table lost its %s column", header) } @@ -168,11 +174,11 @@ func TestCLIDatasetShow(t *testing.T) { t.Run("the name is required", func(t *testing.T) { r := requireFailure(t, run(t, "dataset", "show")) - require.Contains(t, r.Combined(), "--name is required") + require.Contains(t, r.Combined(), "accepts 1 arg") }) t.Run("an unknown dataset is brief", func(t *testing.T) { - r := requireFailure(t, run(t, "dataset", "show", "--name", "azdcli-no-such-dataset")) + r := requireFailure(t, run(t, "dataset", "show", "azdcli-no-such-dataset")) require.Less(t, len(r.Combined()), 600, "a not-found must stay short, not dump the service body:\n%s", r.Combined()) require.Contains(t, r.Combined(), "azdcli-no-such-dataset") @@ -180,19 +186,19 @@ func TestCLIDatasetShow(t *testing.T) { t.Run("an unknown version of a real dataset is refused", func(t *testing.T) { r := requireFailure(t, run(t, "dataset", "show", - "--name", ds.Name, "--version", "9999")) + ds.Name, "--version", "9999")) require.Contains(t, r.Combined(), "9999") require.Less(t, len(r.Combined()), 600, r.Combined()) }) } func TestCLIDatasetDelete(t *testing.T) { - t.Run("both flags are required", func(t *testing.T) { + t.Run("the name and version are both required", func(t *testing.T) { require.Contains(t, requireFailure(t, run(t, "dataset", "delete", "--version", "1")).Combined(), - "--name is required") + "accepts 1 arg") require.Contains(t, - requireFailure(t, run(t, "dataset", "delete", "--name", "whatever")).Combined(), + requireFailure(t, run(t, "dataset", "delete", "whatever")).Combined(), "--version is required") }) @@ -204,11 +210,11 @@ func TestCLIDatasetDelete(t *testing.T) { // entitled to know it means "gone", not "was there and is now gone". t.Run("deleting an unregistered dataset is idempotent, not an error", func(t *testing.T) { r := requireSuccess(t, run(t, "dataset", "delete", - "--name", "azdcli-no-such-dataset", "--version", "1")) + "azdcli-no-such-dataset", "--version", "1")) require.Contains(t, r.Stdout, "Deleted dataset") - listed := requireSuccess(t, run(t, "dataset", "list", - "--name", "azdcli-no-such-dataset", "-o", "json")) + listed := requireSuccess(t, run(t, "dataset", "versions", "list", + "azdcli-no-such-dataset", "-o", "json")) var remaining []datasetSummary listed.JSON(t, &remaining) require.Empty(t, remaining, "nothing was there to delete in the first place") @@ -222,12 +228,12 @@ func TestCLIDatasetDelete(t *testing.T) { gone, kept := ds.Versions[0], ds.Versions[1] r := requireSuccess(t, run(t, "dataset", "delete", - "--name", ds.Name, "--version", gone)) + ds.Name, "--version", gone)) require.Contains(t, r.Stdout, "Deleted dataset") require.Contains(t, r.Stdout, ds.Name) - listed := requireSuccess(t, run(t, "dataset", "list", - "--name", ds.Name, "-o", "json")) + listed := requireSuccess(t, run(t, "dataset", "versions", "list", + ds.Name, "-o", "json")) var remaining []datasetSummary listed.JSON(t, &remaining) From f7ae5632a9df953d216b545924f8620c2b75f5bd Mon Sep 17 00:00:00 2001 From: mohessie Date: Mon, 3 Aug 2026 04:50:31 +0300 Subject: [PATCH 076/320] Add the eval and job groups Evals gain list, show and delete at the root; creation stays with azd up so that reconciliation has one owner. Jobs gain list, show and cancel, which is the resume path for a generate started with --no-wait. Both job routes turned out to carry more than the spec's table records: list and cancel exist for datasets and evaluators alike, verified against the live project. Cancel is the colon form - {id}:cancel reaches the action while {id}/cancel is a 404 - and needs a body, because without a content type the route answers 415. A job id does not say which kind started it, so show and cancel try both rather than making the caller remember. --- .../internal/cmd/eval_group.go | 134 ++++++++++++ .../azure.ai.evaluations/internal/cmd/job.go | 201 ++++++++++++++++++ .../azure.ai.evaluations/internal/cmd/root.go | 4 + .../internal/pkg/eval_api/operations.go | 56 +++++ 4 files changed, 395 insertions(+) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/eval_group.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/job.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/eval_group.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/eval_group.go new file mode 100644 index 00000000000..8f7566c00cf --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/eval_group.go @@ -0,0 +1,134 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "fmt" + + "azureaieval/internal/pkg/eval_api" + + "github.com/spf13/cobra" +) + +// The eval group is read and delete only. Creation belongs to `azd up`, which +// owns reconciliation: a second creation path would drift from the declared +// config, and reconciliation could not then tell whether to adopt an eval it +// found or replace it. + +func newEvalListCommand() *cobra.Command { + var ( + limit int + endpointFlg string + ) + + cmd := &cobra.Command{ + Use: "list", + Short: "List the project's evals.", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + ctx := cmd.Context() + ec, err := newEvalContext(ctx, endpointFlg) + if err != nil { + return err + } + defer ec.Close() + + list, err := ec.evalClient.ListOpenAIEvals(ctx, limit) + if err != nil { + return fmt.Errorf("listing evals: %w", err) + } + + if isJSON(cmd) { + return emitJSONList(cmd.OutOrStdout(), list.Data) + } + if len(list.Data) == 0 { + fmt.Fprintln(cmd.OutOrStdout(), "No evals found.") + return nil + } + rows := make([][]string, 0, len(list.Data)) + for _, e := range list.Data { + rows = append(rows, []string{e.ID, e.Name}) + } + return emitTable(cmd.OutOrStdout(), []string{"EVAL ID", "NAME"}, rows) + }, + } + + cmd.Flags().IntVar(&limit, "limit", 0, "Cap the number of evals returned.") + cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") + return cmd +} + +func newEvalShowCommand() *cobra.Command { + var endpointFlg string + + cmd := &cobra.Command{ + Use: "show ", + Short: "Show an eval definition.", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + evalID := args[0] + + ctx := cmd.Context() + ec, err := newEvalContext(ctx, endpointFlg) + if err != nil { + return err + } + defer ec.Close() + + group, err := ec.evalClient.GetOpenAIEval(ctx, evalID) + if err != nil { + if eval_api.IsNotFound(err) { + return fmt.Errorf( + "no eval %q in this project; "+ + "`azd ai eval list` shows the ones there are", evalID) + } + return fmt.Errorf("reading eval %q: %w", evalID, err) + } + return emitJSON(cmd.OutOrStdout(), group) + }, + } + + cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") + return cmd +} + +func newEvalDeleteCommand() *cobra.Command { + var endpointFlg string + + cmd := &cobra.Command{ + Use: "delete ", + Short: "Delete an eval and everything under it.", + Long: "Delete an eval and everything under it.\n\n" + + "An eval owns its runs, so deleting one discards their results too.", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + evalID := args[0] + + ctx := cmd.Context() + ec, err := newEvalContext(ctx, endpointFlg) + if err != nil { + return err + } + defer ec.Close() + + if err := ec.evalClient.DeleteOpenAIEval(ctx, evalID); err != nil { + if eval_api.IsNotFound(err) { + return fmt.Errorf("no eval %q in this project", evalID) + } + return fmt.Errorf("deleting eval %q: %w", evalID, err) + } + + if isJSON(cmd) { + return emitJSON(cmd.OutOrStdout(), map[string]string{ + "id": evalID, "status": "deleted", + }) + } + fmt.Fprintf(cmd.OutOrStdout(), "Deleted eval %s\n", evalID) + return nil + }, + } + + cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") + return cmd +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/job.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/job.go new file mode 100644 index 00000000000..1c094e20c77 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/job.go @@ -0,0 +1,201 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "context" + "fmt" + + "azureaieval/internal/pkg/eval_api" + + "github.com/spf13/cobra" +) + +// Generation runs as two independent long-running resources — one for datasets, +// one for evaluators — and a job id does not say which it came from. Rather +// than make the caller remember, every command here tries both. + +const ( + jobKindDataset = "dataset" + jobKindEvaluator = "evaluator" +) + +func newJobCommand() *cobra.Command { + cmd := &cobra.Command{ + Use: "job", + Short: "Inspect and cancel generation jobs.", + Long: "Inspect and cancel generation jobs.\n\n" + + "This is the resume path for `dataset generate` and `evaluator generate`: " + + "a job started with --no-wait, or one whose client was interrupted, is " + + "reattached to here rather than restarted.", + } + cmd.AddCommand( + newJobListCommand(), + newJobShowCommand(), + newJobCancelCommand(), + ) + return cmd +} + +func newJobListCommand() *cobra.Command { + var endpointFlg string + + cmd := &cobra.Command{ + Use: "list", + Short: "List the project's generation jobs.", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + ctx := cmd.Context() + ec, err := newEvalContext(ctx, endpointFlg) + if err != nil { + return err + } + defer ec.Close() + + datasets, err := ec.evalClient.ListDataGenerationJobs(ctx, ProjectEndpointAPIVersion) + if err != nil { + return fmt.Errorf("listing dataset generation jobs: %w", err) + } + evaluators, err := ec.evalClient.ListEvaluatorGenerationJobs(ctx, ProjectEndpointAPIVersion) + if err != nil { + return fmt.Errorf("listing evaluator generation jobs: %w", err) + } + + type jobRow struct { + ID string `json:"id"` + Kind string `json:"kind"` + Status string `json:"status"` + } + rows := make([]jobRow, 0, len(datasets.Data)+len(evaluators.Data)) + for _, j := range datasets.Data { + rows = append(rows, jobRow{ID: j.ID, Kind: jobKindDataset, Status: j.Status}) + } + for _, j := range evaluators.Data { + rows = append(rows, jobRow{ID: j.ID, Kind: jobKindEvaluator, Status: j.Status}) + } + + if isJSON(cmd) { + return emitJSONList(cmd.OutOrStdout(), rows) + } + if len(rows) == 0 { + fmt.Fprintln(cmd.OutOrStdout(), "No generation jobs found.") + return nil + } + table := make([][]string, 0, len(rows)) + for _, r := range rows { + table = append(table, []string{r.ID, r.Kind, r.Status}) + } + return emitTable(cmd.OutOrStdout(), []string{"JOB ID", "KIND", "STATUS"}, table) + }, + } + + cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") + return cmd +} + +func newJobShowCommand() *cobra.Command { + var endpointFlg string + + cmd := &cobra.Command{ + Use: "show ", + Short: "Show a generation job.", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + jobID := args[0] + + ctx := cmd.Context() + ec, err := newEvalContext(ctx, endpointFlg) + if err != nil { + return err + } + defer ec.Close() + + job, _, err := findGenerationJob(ctx, ec, jobID) + if err != nil { + return err + } + + if isJSON(cmd) { + return emitJSON(cmd.OutOrStdout(), job) + } + fmt.Fprintf(cmd.OutOrStdout(), "%s %s\n", job.ID, job.Status) + if job.Error != nil && job.Error.Message != "" { + fmt.Fprintf(cmd.OutOrStdout(), "error: %s\n", job.Error.Message) + } + return nil + }, + } + + cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") + return cmd +} + +func newJobCancelCommand() *cobra.Command { + var endpointFlg string + + cmd := &cobra.Command{ + Use: "cancel ", + Short: "Cancel an in-flight generation job.", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + jobID := args[0] + + ctx := cmd.Context() + ec, err := newEvalContext(ctx, endpointFlg) + if err != nil { + return err + } + defer ec.Close() + + _, kind, err := findGenerationJob(ctx, ec, jobID) + if err != nil { + return err + } + + var canceled *eval_api.GenerationJob + if kind == jobKindDataset { + canceled, err = ec.evalClient.CancelDataGenerationJob( + ctx, jobID, ProjectEndpointAPIVersion) + } else { + canceled, err = ec.evalClient.CancelEvaluatorGenerationJob( + ctx, jobID, ProjectEndpointAPIVersion) + } + if err != nil { + return fmt.Errorf("cancelling job %s: %w", jobID, err) + } + + if isJSON(cmd) { + return emitJSON(cmd.OutOrStdout(), canceled) + } + fmt.Fprintf(cmd.OutOrStdout(), "Cancelled %s generation job %s (%s)\n", + kind, jobID, canceled.Status) + return nil + }, + } + + cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") + return cmd +} + +// findGenerationJob resolves an id against both job types and reports which one +// answered, so that a caller never has to know which command started it. +func findGenerationJob( + ctx context.Context, + ec *evalContext, + jobID string, +) (*eval_api.GenerationJob, string, error) { + if job, err := ec.evalClient.GetDataGenerationJob( + ctx, jobID, ProjectEndpointAPIVersion, + ); err == nil { + return job, jobKindDataset, nil + } + if job, err := ec.evalClient.GetEvaluatorGenerationJob( + ctx, jobID, ProjectEndpointAPIVersion, + ); err == nil { + return job, jobKindEvaluator, nil + } + return nil, "", fmt.Errorf( + "no generation job %s in this project; "+ + "`azd ai eval job list` shows the ones there are", jobID) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/root.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/root.go index 40724c5a0a7..af9f5f58156 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/root.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/root.go @@ -51,6 +51,10 @@ func NewRootCommand() *cobra.Command { newRunCommand(), newEvaluatorCommand(), newResultsCommand(), + newJobCommand(), + newEvalListCommand(), + newEvalShowCommand(), + newEvalDeleteCommand(), newListenCommand(), ) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/operations.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/operations.go index 4ba2d628994..cf61480d16b 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/operations.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/operations.go @@ -115,6 +115,62 @@ func (c *EvalClient) GetEvaluatorGenerationJob( return doRequestTyped[GenerationJob](c, ctx, http.MethodGet, path, nil, nil, apiVersion) } +// GenerationJobList is the listing envelope both job types answer with. It is +// `data`, not the `value` the dataset and evaluator routes use. +type GenerationJobList struct { + Data []GenerationJob `json:"data"` +} + +// ListDataGenerationJobs returns the project's dataset generation jobs. +func (c *EvalClient) ListDataGenerationJobs( + ctx context.Context, + apiVersion string, +) (*GenerationJobList, error) { + return doRequestTyped[GenerationJobList]( + c, ctx, http.MethodGet, pathDataGenerationJobs, nil, nil, apiVersion) +} + +// ListEvaluatorGenerationJobs returns the project's evaluator generation jobs. +func (c *EvalClient) ListEvaluatorGenerationJobs( + ctx context.Context, + apiVersion string, +) (*GenerationJobList, error) { + return doRequestTyped[GenerationJobList]( + c, ctx, http.MethodGet, pathEvaluatorGenerationJobs, nil, nil, apiVersion) +} + +// CancelDataGenerationJob stops a dataset generation job. +func (c *EvalClient) CancelDataGenerationJob( + ctx context.Context, + operationID string, + apiVersion string, +) (*GenerationJob, error) { + return c.cancelGenerationJob(ctx, pathDataGenerationJobs, operationID, apiVersion) +} + +// CancelEvaluatorGenerationJob stops an evaluator generation job. +func (c *EvalClient) CancelEvaluatorGenerationJob( + ctx context.Context, + operationID string, + apiVersion string, +) (*GenerationJob, error) { + return c.cancelGenerationJob(ctx, pathEvaluatorGenerationJobs, operationID, apiVersion) +} + +// cancelGenerationJob posts to the colon form of the route. +// +// The separator is a colon, not a path segment: `{id}/cancel` is a 404 while +// `{id}:cancel` reaches the action. The empty object is what carries a content +// type, without which the route answers 415. +func (c *EvalClient) cancelGenerationJob( + ctx context.Context, + basePath, operationID, apiVersion string, +) (*GenerationJob, error) { + path := basePath + "/" + url.PathEscape(operationID) + ":cancel" + return doRequestTyped[GenerationJob]( + c, ctx, http.MethodPost, path, nil, json.RawMessage(`{}`), apiVersion) +} + // GetAgent reads an agent from the project's catalog. // // Only the newest version is returned, which is the one generation is seeded From 975e1a559979b7e916545a6dec44aad95fc1684a Mon Sep 17 00:00:00 2001 From: mohessie Date: Mon, 3 Aug 2026 05:13:08 +0300 Subject: [PATCH 077/320] Turn results into run output, and make run a group The per-sample views move under the run they belong to: run output list, show and export. The positional argument is now the run rather than the eval, since the eval is what --eval names and the API path nests runs under it. run output show is new, and reads one row whole. The listing truncates the input and the reason to keep a table readable, so there was no way to see either in full. run stops being executable. Once run output exists, a bare run would make azd ai eval run list read as run the thing called list. --- .../azure.ai.evaluations/internal/cmd/root.go | 1 - .../azure.ai.evaluations/internal/cmd/run.go | 13 ++- .../internal/cmd/run_ops.go | 53 ++++----- .../internal/cmd/run_ops_test.go | 24 +++-- .../cmd/{results.go => run_output.go} | 102 ++++++++++++++---- .../internal/pkg/eval_api/evaluators.go | 12 +++ .../tests/cli/run_ops_test.go | 32 +++--- .../{results_test.go => run_output_test.go} | 26 ++--- 8 files changed, 177 insertions(+), 86 deletions(-) rename cli/azd/extensions/azure.ai.evaluations/internal/cmd/{results.go => run_output.go} (78%) rename cli/azd/extensions/azure.ai.evaluations/tests/cli/{results_test.go => run_output_test.go} (87%) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/root.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/root.go index af9f5f58156..24323e839b9 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/root.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/root.go @@ -50,7 +50,6 @@ func NewRootCommand() *cobra.Command { newDatasetCommand(), newRunCommand(), newEvaluatorCommand(), - newResultsCommand(), newJobCommand(), newEvalListCommand(), newEvalShowCommand(), diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go index b9ccd7d398a..73b963960b4 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go @@ -35,17 +35,22 @@ var terminalRunStates = map[string]bool{ // newRunCommand builds the composite `azd ai eval run` and attaches the atomic // run operations, including `run start` which the spec lists as the atomic form // of this same command. +// newRunCommand builds the run group. +// +// `run` is a group, not an executable verb: once `run output` exists, a bare +// `run` would make `azd ai eval run list` read as "run the thing called list". func newRunCommand() *cobra.Command { - cmd := buildRunCommand( - "run", "Run an evaluation, creating the eval if it does not exist yet.") + cmd := &cobra.Command{ + Use: "run", + Short: "Start and inspect evaluation runs.", + } addRunSubcommands(cmd) cmd.AddCommand(buildRunCommand( "start", "Start a run, creating the eval if it does not exist yet.")) return cmd } -// buildRunCommand is shared by `run` and `run start` so the two forms cannot -// drift apart. +// buildRunCommand builds `run start`. func buildRunCommand(use, short string) *cobra.Command { var ( configPath string diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go index fbfdb2c8271..71af58eccab 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go @@ -22,6 +22,7 @@ func addRunSubcommands(cmd *cobra.Command) { newRunShowCommand(), newRunCancelCommand(), newRunDeleteCommand(), + newRunOutputCommand(), ) } @@ -33,9 +34,9 @@ func newRunListCommand() *cobra.Command { ) cmd := &cobra.Command{ - Use: "list [eval-id]", + Use: "list", Short: "List runs for an eval.", - Args: cobra.MaximumNArgs(1), + Args: cobra.NoArgs, RunE: func(cmd *cobra.Command, args []string) error { ctx := cmd.Context() ec, err := newEvalContext(ctx, endpointFlg) @@ -44,7 +45,7 @@ func newRunListCommand() *cobra.Command { } defer ec.Close() - evalID, err := resolveEvalID(cmd, ec, args, groupName) + evalID, err := resolveEvalID(cmd, ec, nil, groupName) if err != nil { return err } @@ -87,13 +88,12 @@ func newRunListCommand() *cobra.Command { func newRunShowCommand() *cobra.Command { var ( - runID string endpointFlg string groupName string ) cmd := &cobra.Command{ - Use: "show [eval-id]", + Use: "show [run]", Short: "Show a single run.", Args: cobra.MaximumNArgs(1), RunE: func(cmd *cobra.Command, args []string) error { @@ -104,12 +104,13 @@ func newRunShowCommand() *cobra.Command { } defer ec.Close() - evalID, err := resolveEvalID(cmd, ec, args, groupName) + evalID, err := resolveEvalID(cmd, ec, nil, groupName) if err != nil { return err } - run, err := ec.latestOrNamedRun(cmd, evalID, runID) + runID := firstArg(args) + run, err := ec.latestOrNamedRun(cmd, evalID, runID, runID != "") if err != nil { return err } @@ -130,21 +131,27 @@ func newRunShowCommand() *cobra.Command { return nil }, } - cmd.Flags().StringVar(&runID, "run-id", "", "Run to show. Defaults to the most recent run.") addEvalFlags(cmd, &groupName) cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") return cmd } +// firstArg returns the positional argument, or empty when none was given. +func firstArg(args []string) string { + if len(args) > 0 { + return args[0] + } + return "" +} + func newRunCancelCommand() *cobra.Command { var ( - runID string endpointFlg string groupName string ) cmd := &cobra.Command{ - Use: "cancel [eval-id]", + Use: "cancel [run]", Short: "Cancel an in-flight run.", Args: cobra.MaximumNArgs(1), RunE: func(cmd *cobra.Command, args []string) error { @@ -155,12 +162,13 @@ func newRunCancelCommand() *cobra.Command { } defer ec.Close() - evalID, err := resolveEvalID(cmd, ec, args, groupName) + evalID, err := resolveEvalID(cmd, ec, nil, groupName) if err != nil { return err } - target, err := ec.latestOrNamedRun(cmd, evalID, runID) + runID := firstArg(args) + target, err := ec.latestOrNamedRun(cmd, evalID, runID, runID != "") if err != nil { return err } @@ -186,7 +194,6 @@ func newRunCancelCommand() *cobra.Command { return nil }, } - cmd.Flags().StringVar(&runID, "run-id", "", "Run to cancel. Defaults to the most recent run.") addEvalFlags(cmd, &groupName) cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") return cmd @@ -194,33 +201,30 @@ func newRunCancelCommand() *cobra.Command { // newRunDeleteCommand removes a run. // -// Runs accumulate — every `azd ai eval run` adds one — and a run that evaluated -// the wrong dataset or target is noise in every later listing and comparison. -// The id is required rather than defaulted to the most recent run, because -// deleting is not undoable and "the latest one" is a poor thing to guess at. +// Runs accumulate — every `run start` adds one — and a run that evaluated the +// wrong dataset or target is noise in every later listing. The run is required +// rather than defaulted to the most recent, because deleting is not undoable +// and "the latest one" is a poor thing to guess at. func newRunDeleteCommand() *cobra.Command { var ( - runID string endpointFlg string groupName string ) cmd := &cobra.Command{ - Use: "delete [eval-id]", + Use: "delete ", Short: "Delete a run.", - Args: cobra.MaximumNArgs(1), + Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { ctx := cmd.Context() - if runID == "" { - return requireFlag("run-id") - } + runID := args[0] ec, err := newEvalContext(ctx, endpointFlg) if err != nil { return err } defer ec.Close() - evalID, err := resolveEvalID(cmd, ec, args, groupName) + evalID, err := resolveEvalID(cmd, ec, nil, groupName) if err != nil { return err } @@ -241,7 +245,6 @@ func newRunDeleteCommand() *cobra.Command { return nil }, } - cmd.Flags().StringVar(&runID, "run-id", "", "Run to delete.") addEvalFlags(cmd, &groupName) cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") return cmd diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops_test.go index 05f2e32b773..1a5bd5ac978 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops_test.go @@ -70,14 +70,16 @@ func TestEvalCommandsAcceptIDAsAFlag(t *testing.T) { subs := map[string]*cobra.Command{} for _, sub := range newRunCommand().Commands() { subs["run "+sub.Name()] = sub - } - for _, sub := range newResultsCommand().Commands() { - subs["results "+sub.Name()] = sub + if sub.Name() == "output" { + for _, leaf := range sub.Commands() { + subs["run output "+leaf.Name()] = leaf + } + } } for _, name := range []string{ "run list", "run show", "run cancel", - "results show", "results export", + "run output list", "run output show", "run output export", } { cmd := subs[name] require.NotNil(t, cmd, "%s should exist", name) @@ -87,9 +89,15 @@ func TestEvalCommandsAcceptIDAsAFlag(t *testing.T) { } // --no-wait is documented in the spec, and cobra does not derive it from the -// --wait bool. +// --wait bool. It belongs to `run start`: `run` itself is a group. func TestRunCommandAcceptsNoWait(t *testing.T) { - cmd := newRunCommand() - require.NotNil(t, cmd.Flags().Lookup("no-wait"), "run should accept --no-wait") - require.NotNil(t, cmd.Flags().Lookup("wait"), "run should keep --wait") + var start *cobra.Command + for _, sub := range newRunCommand().Commands() { + if sub.Name() == "start" { + start = sub + } + } + require.NotNil(t, start) + require.NotNil(t, start.Flags().Lookup("no-wait"), "run start should accept --no-wait") + require.NotNil(t, start.Flags().Lookup("wait"), "run start should keep --wait") } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/results.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output.go similarity index 78% rename from cli/azd/extensions/azure.ai.evaluations/internal/cmd/results.go rename to cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output.go index d2377f34671..4a30bcd200d 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/results.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output.go @@ -16,12 +16,20 @@ import ( "github.com/spf13/cobra" ) -func newResultsCommand() *cobra.Command { +// newRunOutputCommand groups the per-sample views of a run. +// +// `run show` is the summary - how many passed. These are the rows: which ones +// failed, and why. +func newRunOutputCommand() *cobra.Command { cmd := &cobra.Command{ - Use: "results", - Short: "Inspect evaluation results.", + Use: "output", + Short: "Inspect the per-sample results of a run.", } - cmd.AddCommand(newResultsShowCommand(), newResultsExportCommand()) + cmd.AddCommand( + newRunOutputListCommand(), + newRunOutputShowCommand(), + newRunOutputExportCommand(), + ) return cmd } @@ -35,9 +43,8 @@ func formatStat(verb string, v eval_api.LenientFloat) string { return fmt.Sprintf(verb, float64(v)) } -func newResultsShowCommand() *cobra.Command { +func newRunOutputListCommand() *cobra.Command { var ( - runID string failedOnly bool outFile string endpointFlg string @@ -45,8 +52,8 @@ func newResultsShowCommand() *cobra.Command { ) cmd := &cobra.Command{ - Use: "show ", - Short: "Show per-sample results for a run.", + Use: "list [run]", + Short: "List the per-sample results of a run.", Args: cobra.MaximumNArgs(1), RunE: func(cmd *cobra.Command, args []string) error { ctx := cmd.Context() @@ -56,12 +63,13 @@ func newResultsShowCommand() *cobra.Command { } defer ec.Close() - evalID, err := resolveEvalID(cmd, ec, args, groupName) + evalID, err := resolveEvalID(cmd, ec, nil, groupName) if err != nil { return err } - run, err := ec.latestOrNamedRun(cmd, evalID, runID) + runID := firstArg(args) + run, err := ec.latestOrNamedRun(cmd, evalID, runID, runID != "") if err != nil { return err } @@ -101,7 +109,6 @@ func newResultsShowCommand() *cobra.Command { }, } - cmd.Flags().StringVar(&runID, "run-id", "", "Run to show. Defaults to the most recent run.") cmd.Flags().BoolVar(&failedOnly, "failed-only", false, "Show only the rows that failed.") cmd.Flags().StringVarP(&outFile, "out-file", "O", "", "Write JSON results to this path.") addEvalFlags(cmd, &groupName) @@ -109,9 +116,63 @@ func newResultsShowCommand() *cobra.Command { return cmd } -func newResultsExportCommand() *cobra.Command { +// newRunOutputShowCommand reads one evaluated row by its id. +// +// The listing truncates the input and the reason to keep a table readable, so +// this is how the whole of either is seen. +func newRunOutputShowCommand() *cobra.Command { var ( runID string + endpointFlg string + groupName string + ) + + cmd := &cobra.Command{ + Use: "show ", + Short: "Show a single evaluated row.", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + itemID := args[0] + + ctx := cmd.Context() + ec, err := newEvalContext(ctx, endpointFlg) + if err != nil { + return err + } + defer ec.Close() + + evalID, err := resolveEvalID(cmd, ec, nil, groupName) + if err != nil { + return err + } + + run, err := ec.latestOrNamedRun(cmd, evalID, runID, runID != "") + if err != nil { + return err + } + + item, err := ec.evalClient.GetOutputItem(ctx, evalID, run.ID, itemID) + if err != nil { + if eval_api.IsNotFound(err) { + return fmt.Errorf( + "no output item %q on run %s; "+ + "`azd ai eval run output list` shows the ones there are", + itemID, run.ID) + } + return fmt.Errorf("reading output item %q: %w", itemID, err) + } + return emitJSON(cmd.OutOrStdout(), item) + }, + } + + cmd.Flags().StringVar(&runID, "run", "", "Run the item belongs to. Defaults to the most recent run.") + addEvalFlags(cmd, &groupName) + cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") + return cmd +} + +func newRunOutputExportCommand() *cobra.Command { + var ( format string outFile string endpointFlg string @@ -119,7 +180,7 @@ func newResultsExportCommand() *cobra.Command { ) cmd := &cobra.Command{ - Use: "export ", + Use: "export [run]", Short: "Export run results as JSON or CSV.", Args: cobra.MaximumNArgs(1), RunE: func(cmd *cobra.Command, args []string) error { @@ -135,12 +196,13 @@ func newResultsExportCommand() *cobra.Command { } defer ec.Close() - evalID, err := resolveEvalID(cmd, ec, args, groupName) + evalID, err := resolveEvalID(cmd, ec, nil, groupName) if err != nil { return err } - run, err := ec.latestOrNamedRun(cmd, evalID, runID) + runID := firstArg(args) + run, err := ec.latestOrNamedRun(cmd, evalID, runID, runID != "") if err != nil { return err } @@ -162,7 +224,6 @@ func newResultsExportCommand() *cobra.Command { }, } - cmd.Flags().StringVar(&runID, "run-id", "", "Run to export. Defaults to the most recent run.") cmd.Flags().StringVar(&format, "format", "json", "Output format: json or csv.") cmd.Flags().StringVarP(&outFile, "out-file", "O", "", "Write to this path instead of stdout.") addEvalFlags(cmd, &groupName) @@ -226,9 +287,14 @@ func addEvalFlags(cmd *cobra.Command, target *string) { } // latestOrNamedRun returns the named run, or the most recent one for the eval. +// +// explicit says whether the caller named the run rather than leaving it to +// default. A remembered run that no longer resolves is worth falling through +// on; one that was asked for by name is not. func (ec *evalContext) latestOrNamedRun( cmd *cobra.Command, evalID, runID string, + explicit bool, ) (*eval_api.OpenAIEvalRun, error) { ctx := cmd.Context() @@ -243,9 +309,7 @@ func (ec *evalContext) latestOrNamedRun( if err == nil { return run, nil } - // A remembered run that no longer resolves is not worth failing on: - // fall through to whatever the group has now. - if cmd.Flag("run-id") != nil && cmd.Flag("run-id").Changed { + if explicit { return nil, fmt.Errorf("reading run %s: %w", runID, err) } } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/evaluators.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/evaluators.go index 7b3181b6ade..9f0a6f17b35 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/evaluators.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/evaluators.go @@ -239,3 +239,15 @@ func (c *EvalClient) ListOutputItems( ) return doRequestTyped[OutputItemList](c, ctx, http.MethodGet, path, query, nil, "") } + +// GetOutputItem reads a single evaluated row. +func (c *EvalClient) GetOutputItem( + ctx context.Context, + evalID, runID, itemID string, +) (*OutputItem, error) { + path := fmt.Sprintf( + "%s/%s/runs/%s/output_items/%s", + pathOpenAIEvals, url.PathEscape(evalID), url.PathEscape(runID), url.PathEscape(itemID), + ) + return doRequestTyped[OutputItem](c, ctx, http.MethodGet, path, nil, nil, "") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/tests/cli/run_ops_test.go b/cli/azd/extensions/azure.ai.evaluations/tests/cli/run_ops_test.go index 3f37b32462f..626ab9dbccd 100644 --- a/cli/azd/extensions/azure.ai.evaluations/tests/cli/run_ops_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/tests/cli/run_ops_test.go @@ -28,7 +28,7 @@ func TestCLIRunList(t *testing.T) { f := sharedEval(t) t.Run("table", func(t *testing.T) { - r := requireSuccess(t, run(t, "run", "list", f.EvalID)) + r := requireSuccess(t, run(t, "run", "list", "--eval-id", f.EvalID)) for _, header := range []string{"RUN ID", "NAME", "STATUS", "RESULTS"} { require.Containsf(t, r.Stdout, header, "the listing lost its %s column", header) } @@ -39,7 +39,7 @@ func TestCLIRunList(t *testing.T) { }) t.Run("json", func(t *testing.T) { - r := requireSuccess(t, run(t, "run", "list", f.EvalID, "-o", "json")) + r := requireSuccess(t, run(t, "run", "list", "--eval-id", f.EvalID, "-o", "json")) require.True(t, strings.HasPrefix(strings.TrimSpace(r.Stdout), "["), "a list must be a bare array, not the service's envelope") @@ -63,14 +63,14 @@ func TestCLIRunList(t *testing.T) { // The client has always taken a limit; until recently the command did not // expose one, so a service-side truncation would have passed unnoticed. t.Run("limit", func(t *testing.T) { - r := requireSuccess(t, run(t, "run", "list", f.EvalID, "--limit", "1", "-o", "json")) + r := requireSuccess(t, run(t, "run", "list", "--eval-id", f.EvalID, "--limit", "1", "-o", "json")) var runs []runSummary r.JSON(t, &runs) require.Len(t, runs, 1, "--limit must reach the service") }) t.Run("unknown eval is brief", func(t *testing.T) { - r := requireFailure(t, run(t, "run", "list", "eval_azdcli_no_such_eval")) + r := requireFailure(t, run(t, "run", "list", "--eval-id", "eval_azdcli_no_such_eval")) require.Less(t, len(r.Combined()), 600, "a not-found must stay short, not dump the service body:\n%s", r.Combined()) require.Contains(t, r.Combined(), "eval_azdcli_no_such_eval") @@ -81,7 +81,7 @@ func TestCLIRunShow(t *testing.T) { f := sharedEval(t) t.Run("by run id", func(t *testing.T) { - r := requireSuccess(t, run(t, "run", "show", f.EvalID, "--run-id", f.FirstRunID)) + r := requireSuccess(t, run(t, "run", "show", f.FirstRunID, "--eval-id", f.EvalID)) require.Contains(t, r.Stdout, f.FirstRunID) require.Contains(t, r.Stdout, "status") require.Contains(t, r.Stdout, "completed") @@ -93,12 +93,12 @@ func TestCLIRunShow(t *testing.T) { // environment there is no remembered id to fall back on, so what is // exercised is the listing path. t.Run("defaults to the most recent run", func(t *testing.T) { - listed := requireSuccess(t, run(t, "run", "list", f.EvalID, "--limit", "1", "-o", "json")) + listed := requireSuccess(t, run(t, "run", "list", "--eval-id", f.EvalID, "--limit", "1", "-o", "json")) var newest []runSummary listed.JSON(t, &newest) require.Len(t, newest, 1) - r := requireSuccess(t, run(t, "run", "show", f.EvalID, "-o", "json")) + r := requireSuccess(t, run(t, "run", "show", "--eval-id", f.EvalID, "-o", "json")) var shown runSummary r.JSON(t, &shown) require.Equal(t, newest[0].ID, shown.ID, @@ -114,7 +114,7 @@ func TestCLIRunShow(t *testing.T) { // about 1700 characters of raw JSON — recorded in the report rather than // pinned here, since pinning it would make the length a requirement. t.Run("an unknown run id is reported, not silently replaced", func(t *testing.T) { - r := requireFailure(t, run(t, "run", "show", f.EvalID, "--run-id", "evalrun_azdcli_nope")) + r := requireFailure(t, run(t, "run", "show", "evalrun_azdcli_nope", "--eval-id", f.EvalID)) require.Contains(t, r.Combined(), "evalrun_azdcli_nope", "the failure must name the run that was asked for") require.NotContains(t, r.Combined(), f.FirstRunID, @@ -132,7 +132,7 @@ func TestCLIRunCancelAndDelete(t *testing.T) { f := sharedEval(t) t.Run("a finished run is refused", func(t *testing.T) { - r := requireFailure(t, run(t, "run", "cancel", f.EvalID, "--run-id", f.FirstRunID)) + r := requireFailure(t, run(t, "run", "cancel", f.FirstRunID, "--eval-id", f.EvalID)) require.Contains(t, r.Combined(), "already finished") require.Contains(t, r.Combined(), "completed") }) @@ -148,21 +148,21 @@ func TestCLIRunCancelAndDelete(t *testing.T) { t.Run("an in-flight run is cancelled, and the delete is accepted", func(t *testing.T) { runID := startCancellableRun(t, f) - cancelled := requireSuccess(t, run(t, "run", "cancel", f.EvalID, "--run-id", runID)) + cancelled := requireSuccess(t, run(t, "run", "cancel", runID, "--eval-id", f.EvalID)) require.Contains(t, cancelled.Stdout, runID) require.Contains(t, cancelled.Stdout, "is now") - shown := requireSuccess(t, run(t, "run", "show", f.EvalID, "--run-id", runID, "-o", "json")) + shown := requireSuccess(t, run(t, "run", "show", runID, "--eval-id", f.EvalID, "-o", "json")) var after runSummary shown.JSON(t, &after) require.NotEqual(t, "completed", after.Status, "a cancelled run must not go on to complete") - deleted := requireSuccess(t, run(t, "run", "delete", f.EvalID, "--run-id", runID)) + deleted := requireSuccess(t, run(t, "run", "delete", runID, "--eval-id", f.EvalID)) require.Contains(t, deleted.Stdout, "Deleted run") require.Contains(t, deleted.Stdout, runID) - still := requireSuccess(t, run(t, "run", "show", f.EvalID, "--run-id", runID, "-o", "json")) + still := requireSuccess(t, run(t, "run", "show", runID, "--eval-id", f.EvalID, "-o", "json")) var survivor runSummary still.JSON(t, &survivor) t.Logf("the run is still readable after a successful delete (status %q); "+ @@ -172,12 +172,12 @@ func TestCLIRunCancelAndDelete(t *testing.T) { // Deleting is not undoable, so the id is required rather than defaulted to // whichever run happens to be newest. t.Run("delete requires the run id", func(t *testing.T) { - r := requireFailure(t, run(t, "run", "delete", f.EvalID)) - require.Contains(t, r.Combined(), "--run-id is required") + r := requireFailure(t, run(t, "run", "delete", "--eval-id", f.EvalID)) + require.Contains(t, r.Combined(), "accepts 1 arg") }) t.Run("deleting an unknown run is reported briefly", func(t *testing.T) { - r := requireFailure(t, run(t, "run", "delete", f.EvalID, "--run-id", "evalrun_azdcli_nope")) + r := requireFailure(t, run(t, "run", "delete", "evalrun_azdcli_nope", "--eval-id", f.EvalID)) require.Contains(t, r.Combined(), "evalrun_azdcli_nope") require.Less(t, len(r.Combined()), 600, "a not-found must stay short, not dump the service body:\n%s", r.Combined()) diff --git a/cli/azd/extensions/azure.ai.evaluations/tests/cli/results_test.go b/cli/azd/extensions/azure.ai.evaluations/tests/cli/run_output_test.go similarity index 87% rename from cli/azd/extensions/azure.ai.evaluations/tests/cli/results_test.go rename to cli/azd/extensions/azure.ai.evaluations/tests/cli/run_output_test.go index 77313c677dc..9b9d71e9c91 100644 --- a/cli/azd/extensions/azure.ai.evaluations/tests/cli/results_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/tests/cli/run_output_test.go @@ -55,7 +55,7 @@ type resultsPayload struct { func TestCLIResultsShowRendersTheRows(t *testing.T) { f := sharedEval(t) - r := requireSuccess(t, run(t, "results", "show", f.EvalID, "--run-id", f.FirstRunID)) + r := requireSuccess(t, run(t, "run", "output", "list", f.FirstRunID, "--eval-id", f.EvalID)) require.Contains(t, r.Stdout, f.FirstRunID) require.Contains(t, r.Stdout, "Totals:") @@ -117,8 +117,8 @@ func TestCLIResultsShowFailedOnly(t *testing.T) { payload := resultsFor(t, f.EvalID, f.FirstRunID) failed := payload.Run.ResultCounts.Failed - r := requireSuccess(t, run(t, "results", "show", f.EvalID, - "--run-id", f.FirstRunID, "--failed-only")) + r := requireSuccess(t, run(t, "run", "output", "list", f.FirstRunID, + "--eval-id", f.EvalID, "--failed-only")) if failed == 0 { // Saying so is not the same as printing an empty table. @@ -137,7 +137,7 @@ func TestCLIResultsShowFailedOnly(t *testing.T) { // they can decide what the rendered output should say. func resultsFor(t *testing.T, evalID, runID string) resultsPayload { t.Helper() - r := requireSuccess(t, run(t, "results", "show", evalID, "--run-id", runID, "-o", "json")) + r := requireSuccess(t, run(t, "run", "output", "list", runID, "--eval-id", evalID, "-o", "json")) var payload resultsPayload r.JSON(t, &payload) return payload @@ -147,8 +147,8 @@ func TestCLIResultsExport(t *testing.T) { f := sharedEval(t) t.Run("json to stdout", func(t *testing.T) { - r := requireSuccess(t, run(t, "results", "export", f.EvalID, - "--run-id", f.FirstRunID, "--format", "json")) + r := requireSuccess(t, run(t, "run", "output", "export", f.FirstRunID, + "--eval-id", f.EvalID, "--format", "json")) var exported struct { ID string `json:"id"` @@ -166,8 +166,8 @@ func TestCLIResultsExport(t *testing.T) { }) t.Run("csv to stdout", func(t *testing.T) { - r := requireSuccess(t, run(t, "results", "export", f.EvalID, - "--run-id", f.FirstRunID, "--format", "csv")) + r := requireSuccess(t, run(t, "run", "output", "export", f.FirstRunID, + "--eval-id", f.EvalID, "--format", "csv")) rows, err := csv.NewReader(strings.NewReader(r.Stdout)).ReadAll() require.NoError(t, err, "--format csv must emit parseable CSV:\n%s", r.Stdout) @@ -183,8 +183,8 @@ func TestCLIResultsExport(t *testing.T) { dir := t.TempDir() path := filepath.Join(dir, "results.csv") - r := requireSuccess(t, runIn(t, dir, "results", "export", f.EvalID, - "--run-id", f.FirstRunID, "--format", "csv", "-O", path)) + r := requireSuccess(t, runIn(t, dir, "run", "output", "export", f.FirstRunID, + "--eval-id", f.EvalID, "--format", "csv", "-O", path)) require.Empty(t, strings.TrimSpace(r.Stdout), "-O redirects the payload; leaving it on stdout too would double it") @@ -195,14 +195,14 @@ func TestCLIResultsExport(t *testing.T) { }) t.Run("an unknown format is refused", func(t *testing.T) { - r := requireFailure(t, run(t, "results", "export", f.EvalID, - "--run-id", f.FirstRunID, "--format", "xml")) + r := requireFailure(t, run(t, "run", "output", "export", f.FirstRunID, + "--eval-id", f.EvalID, "--format", "xml")) require.Contains(t, r.Combined(), "json or csv") }) } func TestCLIResultsUnknownEvalIsBrief(t *testing.T) { - r := requireFailure(t, run(t, "results", "show", "eval_does_not_exist")) + r := requireFailure(t, run(t, "run", "output", "list", "--eval-id", "eval_does_not_exist")) require.Contains(t, r.Combined(), "eval_does_not_exist") require.NotContains(t, r.Combined(), "RESPONSE 404") } From 6dc159425f98169d75911f479f3310aa4e6309f5 Mon Sep 17 00:00:00 2001 From: mohessie Date: Mon, 3 Aug 2026 16:43:19 +0300 Subject: [PATCH 078/320] Split generation per artifact, and add CI gating Generation becomes dataset generate and evaluator generate, because the service splits it the same way: two long-running resources, one per artifact. A single composite verb left partial failure undefined, could not regenerate one artifact after the other was hand-edited, and gave --no-wait nothing to reattach to. Neither command edits azure.yaml any more - init declares where the artifacts live, these fill them in, so a generation run produces a data-file-only diff. --fail-on gates a build on quality: any-failure, or pass-rate=<0..1>. It is opt-in, because failing samples are the expected output of a working evaluation and a default that returned non-zero would break a build the first time a grader disagreed. Errored rows count against the rate - they are in the total and did not pass - and a run that scored nothing breaches rather than dividing by zero, so a broken evaluation cannot hold a gate open. The exit code matters more than the flag: 2 says the evaluation regressed, 1 says it could not run. Those have different owners. It exits from applyGate because the extension SDK collapses every returned error to 1. --- .../internal/cmd/dataset.go | 1 + .../internal/cmd/evaluator.go | 1 + .../internal/cmd/gating.go | 116 ++++++++++ .../internal/cmd/gating_test.go | 70 ++++++ .../internal/cmd/generate.go | 146 ------------- .../internal/cmd/generate_commands.go | 199 ++++++++++++++++++ .../azure.ai.evaluations/internal/cmd/root.go | 1 - .../azure.ai.evaluations/internal/cmd/run.go | 22 +- .../internal/cmd/run_ops.go | 28 ++- .../tests/cli/generate_test.go | 122 +++++------ 10 files changed, 482 insertions(+), 224 deletions(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/gating.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/gating_test.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_commands.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset.go index daecc8c3d05..ef4361b8cc8 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset.go @@ -24,6 +24,7 @@ func newDatasetCommand() *cobra.Command { cmd.AddCommand( newDatasetCreateCommand(), newDatasetUpdateCommand(), + newDatasetGenerateCommand(), newDatasetListCommand(), newDatasetShowCommand(), newDatasetDeleteCommand(), diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator.go index 72f431536ed..a4750af5a00 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator.go @@ -21,6 +21,7 @@ func newEvaluatorCommand() *cobra.Command { cmd.AddCommand( newEvaluatorCreateCommand(), newEvaluatorUpdateCommand(), + newEvaluatorGenerateCommand(), newEvaluatorListCommand(), newEvaluatorShowCommand(), newEvaluatorDeleteCommand(), diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/gating.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/gating.go new file mode 100644 index 00000000000..c7754fa5064 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/gating.go @@ -0,0 +1,116 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "fmt" + "os" + "strconv" + "strings" + + "azureaieval/internal/pkg/eval_api" + + "github.com/spf13/cobra" +) + +// Gating is opt-in. A completed run with failing samples exits 0 without +// --fail-on: failing samples are the expected output of a working evaluation, +// not a tool error, and `run start` is used constantly in the inner loop. A +// default that returned non-zero on any failure would break a build the first +// time a noisy grader disagreed. +// +// The separate exit code matters more than the flag. It lets a pipeline tell +// "the evaluation regressed" from "the evaluation could not run", which are +// different failures with different owners. + +// exitCodeGateBreached is returned when a run completed but missed its +// threshold. +const exitCodeGateBreached = 2 + +// gate is a parsed --fail-on threshold. +type gate struct { + set bool + anyFailure bool + passRate float64 +} + +// parseGate reads the --fail-on value. An empty value means no gating. +func parseGate(spec string) (gate, error) { + spec = strings.TrimSpace(spec) + if spec == "" { + return gate{}, nil + } + if spec == "any-failure" { + return gate{set: true, anyFailure: true}, nil + } + + rate, ok := strings.CutPrefix(spec, "pass-rate=") + if !ok { + return gate{}, fmt.Errorf( + "--fail-on must be any-failure or pass-rate=<0..1>, got %q", spec) + } + value, err := strconv.ParseFloat(rate, 64) + if err != nil { + return gate{}, fmt.Errorf("--fail-on pass-rate must be a number, got %q", rate) + } + if value < 0 || value > 1 { + return gate{}, fmt.Errorf("--fail-on pass-rate must be between 0 and 1, got %v", value) + } + return gate{set: true, passRate: value}, nil +} + +// breach reports why the run missed the threshold, or empty when it met it. +// +// Errored and skipped rows count against the pass rate: they are in the total +// and they did not pass. A run that scored nothing at all breaches every +// threshold rather than dividing by zero — "no rows passed" is the honest +// reading of an empty result, and treating it as 100% would let a broken +// evaluation hold a gate open. +func (g gate) breach(counts *eval_api.EvalRunResultCounts) string { + if !g.set { + return "" + } + if counts == nil { + return "the run reported no result counts, so the threshold cannot be checked" + } + if g.anyFailure { + unpassed := counts.Total - counts.Passed + if unpassed > 0 { + return fmt.Sprintf("%d of %d samples did not pass", unpassed, counts.Total) + } + return "" + } + if counts.Total == 0 { + return "the run scored no rows, so its pass rate is below any threshold" + } + actual := float64(counts.Passed) / float64(counts.Total) + if actual < g.passRate { + return fmt.Sprintf("pass rate %.3f is below the required %.3f (%d of %d passed)", + actual, g.passRate, counts.Passed, counts.Total) + } + return "" +} + +// applyGate ends the process with exit code 2 when the run missed its +// threshold. +// +// It exits here rather than returning an error because the extension SDK's +// Run collapses every error to exit 1, and the whole point of the flag is a +// code a pipeline can tell apart from an operational failure. +func applyGate(cmd *cobra.Command, g gate, run *eval_api.OpenAIEvalRun) { + if run == nil { + return + } + reason := g.breach(run.ResultCounts) + if reason == "" { + return + } + fmt.Fprintf(os.Stderr, "Evaluation gate failed: %s\n", reason) + os.Exit(exitCodeGateBreached) +} + +func addFailOnFlag(cmd *cobra.Command, target *string) { + cmd.Flags().StringVar(target, "fail-on", "", + "Exit 2 when the run misses this threshold: any-failure, or pass-rate=<0..1>.") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/gating_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/gating_test.go new file mode 100644 index 00000000000..fb612cd6aca --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/gating_test.go @@ -0,0 +1,70 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "testing" + + "azureaieval/internal/pkg/eval_api" + + "github.com/stretchr/testify/require" +) + +func TestParseGate(t *testing.T) { + t.Run("empty means no gating", func(t *testing.T) { + g, err := parseGate("") + require.NoError(t, err) + require.False(t, g.set) + require.Empty(t, g.breach(&eval_api.EvalRunResultCounts{Total: 3}), + "an unset gate must never breach") + }) + + t.Run("any-failure", func(t *testing.T) { + g, err := parseGate("any-failure") + require.NoError(t, err) + require.True(t, g.anyFailure) + }) + + t.Run("pass-rate", func(t *testing.T) { + g, err := parseGate("pass-rate=0.8") + require.NoError(t, err) + require.InDelta(t, 0.8, g.passRate, 1e-9) + }) + + for _, bad := range []string{"passrate=0.8", "pass-rate=abc", "pass-rate=1.5", "pass-rate=-1", "sometimes"} { + t.Run("refuses "+bad, func(t *testing.T) { + _, err := parseGate(bad) + require.Error(t, err) + }) + } +} + +func TestGateBreach(t *testing.T) { + anyFailure, err := parseGate("any-failure") + require.NoError(t, err) + eighty, err := parseGate("pass-rate=0.8") + require.NoError(t, err) + + t.Run("any-failure passes only when every row passed", func(t *testing.T) { + require.Empty(t, anyFailure.breach(&eval_api.EvalRunResultCounts{Total: 2, Passed: 2})) + require.NotEmpty(t, anyFailure.breach(&eval_api.EvalRunResultCounts{Total: 2, Passed: 1, Failed: 1})) + }) + + // Errored rows are in the total and did not pass, so they count against the + // threshold the same way a failing row does. + t.Run("errored rows count against the rate", func(t *testing.T) { + counts := &eval_api.EvalRunResultCounts{Total: 10, Passed: 8, Errored: 2} + require.Empty(t, eighty.breach(counts), "0.8 exactly meets a 0.8 threshold") + + counts = &eval_api.EvalRunResultCounts{Total: 10, Passed: 7, Errored: 3} + require.NotEmpty(t, eighty.breach(counts)) + }) + + // A run that scored nothing has no defensible pass rate, and treating it as + // 100% would let a broken evaluation hold a gate open. + t.Run("a run that scored nothing breaches", func(t *testing.T) { + require.NotEmpty(t, eighty.breach(&eval_api.EvalRunResultCounts{Total: 0})) + require.NotEmpty(t, eighty.breach(nil)) + }) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go index 1b33daeba1d..86cd9c7727c 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go @@ -17,8 +17,6 @@ import ( "azureaieval/internal/pkg/eval_api" "azureaieval/internal/project" - - "github.com/spf13/cobra" ) // generatePollBudget replaces the inherited 2s x 300 (10 minute) client budget. @@ -29,150 +27,6 @@ var generatePollBudget = eval_api.PollerOptions{ MaxAttempts: 720, // one hour } -func newGenerateCommand() *cobra.Command { - var ( - configPath string - deployPath string - target string - instruction string - instructionFile string - datasetFlag string - evaluators []string - maxSamples int - traceDays int - evalModel string - noWait bool - endpointFlg string - ) - - cmd := &cobra.Command{ - Use: "generate", - Short: "Generate a rubric and dataset, download them, and reference them from the deployment spec.", - RunE: func(cmd *cobra.Command, args []string) error { - ctx := cmd.Context() - out := cmd.OutOrStdout() - - instruction, err := resolveInstruction(instruction, instructionFile) - if err != nil { - return err - } - - cfg, err := resolveGenerateConfig( - configPath, target, evalModel, datasetFlag, maxSamples, traceDays, - ) - if err != nil { - return err - } - if err := cfg.Validate(); err != nil { - return err - } - // Written to stdout because azd does not surface an extension's - // stderr, and guarded so `-o json` stays parseable. - if !isJSON(cmd) { - warnIgnoredTraceFields(cfg, out) - } - - // Both jobs are billed against a model deployment. Checking before - // any network work keeps the failure at the flag the caller can act - // on, instead of a service rejection partway through the command. - generatingRubric := len(evaluators) == 0 && cfg.Generate.Rubric != nil - generatingDataset := datasetFlag == "" && cfg.Generate.Dataset != nil - if (generatingRubric || generatingDataset) && generationModel(cfg) == "" { - return fmt.Errorf( - "a model deployment is required to generate: pass --generation-model, " + - "or set generate.rubric.model in the generation spec") - } - - ec, err := newEvalContext(ctx, endpointFlg) - if err != nil { - return err - } - defer ec.Close() - - instruction, err = ec.resolveGenerationInstruction( - ctx, cfg, instruction, configPath, out, isJSON(cmd), - ) - if err != nil { - return err - } - - baseDir := filepath.Dir(deployPath) - var datasetRefs, evaluatorRefs []project.ArtifactRef - - // Supplied evaluators are honored: their generation is skipped. - if len(evaluators) > 0 { - fmt.Fprintf(out, "Using the supplied evaluators; skipping rubric generation.\n") - } else if cfg.Generate.Rubric != nil { - ref, err := ec.generateRubric(ctx, cfg, instruction, baseDir, out, noWait) - if err != nil { - return err - } - if ref != nil { - evaluatorRefs = append(evaluatorRefs, *ref) - } - } - - // --dataset means use this one, whether it names a local file or a - // dataset already registered on the project. Either way there is - // nothing to generate, which is how --evaluator behaves too. - if datasetFlag != "" { - fmt.Fprintf(out, "Using the supplied dataset; skipping data generation.\n") - } else if cfg.Generate.Dataset != nil { - ref, err := ec.generateDataset(ctx, cfg, instruction, baseDir, out, noWait) - if err != nil { - return err - } - if ref != nil { - datasetRefs = append(datasetRefs, *ref) - } - } - - if len(datasetRefs) == 0 && len(evaluatorRefs) == 0 { - // With --no-wait the jobs were submitted and nothing was - // downloaded, which is success, not an empty result. - if noWait { - fmt.Fprintln(out, - "\nJobs submitted. Re-run without --no-wait to download the artifacts "+ - "and reference them from the deployment spec.") - return nil - } - fmt.Fprintln(out, "Nothing was generated.") - return nil - } - - if err := project.MergeArtifactRefs(deployPath, datasetRefs, evaluatorRefs); err != nil { - return err - } - fmt.Fprintf(out, "\nUpdated %s\n", deployPath) - fmt.Fprintln(out, "Review the generated artifacts, then run: azd up && azd ai eval run") - return nil - }, - } - - cmd.Flags().StringVar(&configPath, "config", project.DefaultGenerateConfig, - "Path to the generation spec. Optional; flags alone are sufficient.") - cmd.Flags().StringVar(&deployPath, "deploy-config", project.DefaultDeployConfig, - "Deployment spec to write source references into.") - cmd.Flags().StringVar(&target, "target", "", "Agent whose context seeds generation.") - cmd.Flags().StringVar(&instruction, "agent-instruction", "", - "What the agent does and what to test.") - cmd.Flags().StringVar(&instructionFile, "agent-instruction-file", "", - "Read the agent instruction from this file. Mutually exclusive with --agent-instruction.") - cmd.MarkFlagsMutuallyExclusive("agent-instruction", "agent-instruction-file") - cmd.Flags().StringVar(&datasetFlag, "dataset", "", - "Use this dataset instead of generating one.") - cmd.Flags().StringArrayVar(&evaluators, "evaluator", nil, - "Use these evaluators instead of generating a rubric; repeatable.") - cmd.Flags().IntVar(&maxSamples, "max-samples", 0, - fmt.Sprintf("Rows to synthesize (%d-%d).", project.MinSampleSize, project.MaxSampleSize)) - cmd.Flags().IntVar(&traceDays, "trace-days", 0, - "Days of traces to seed rubric generation. 0 disables.") - cmd.Flags().StringVar(&evalModel, "generation-model", "", "Model deployment that generates the dataset and rubric.") - cmd.Flags().BoolVar(&noWait, "no-wait", false, "Submit the jobs and return without polling.") - cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") - return cmd -} - // warnIgnoredTraceFields reports generation settings that are accepted but have // no effect yet. // diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_commands.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_commands.go new file mode 100644 index 00000000000..b3c7b86c374 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_commands.go @@ -0,0 +1,199 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "fmt" + + "azureaieval/internal/project" + + "github.com/spf13/cobra" +) + +// Generation is split per artifact because the service splits it: datasets and +// evaluators are separate long-running resources. One composite verb leaves +// partial failure undefined, cannot regenerate one artifact after the other has +// been hand-edited, and gives --no-wait nothing to reattach to. +// +// Neither command edits azure.yaml. `init` declares where the artifacts live; +// these fill them in, so a generation run produces a data-file-only diff. + +// generateFlags are the settings both generate commands share. +type generateFlags struct { + configPath string + target string + instruction string + instructionFile string + model string + outputDir string + noWait bool + endpoint string +} + +func addGenerateFlags(cmd *cobra.Command, f *generateFlags) { + cmd.Flags().StringVar(&f.configPath, "config", project.DefaultGenerateConfig, + "Path to the generation spec. Optional; flags alone are sufficient.") + cmd.Flags().StringVar(&f.target, "target", "", "Agent whose context seeds generation.") + cmd.Flags().StringVar(&f.instruction, "agent-instruction", "", + "What the agent does and what to test.") + cmd.Flags().StringVar(&f.instructionFile, "agent-instruction-file", "", + "Read the agent instruction from this file. Mutually exclusive with --agent-instruction.") + cmd.MarkFlagsMutuallyExclusive("agent-instruction", "agent-instruction-file") + cmd.Flags().StringVar(&f.model, "generation-model", "", + "Model deployment that generates the artifact.") + cmd.Flags().StringVar(&f.outputDir, "output-dir", project.DefaultEvalDir, + "Directory the generated artifact is written under.") + cmd.Flags().BoolVar(&f.noWait, "no-wait", false, + "Submit the job and return its id without polling.") + cmd.Flags().StringVar(&f.endpoint, "project-endpoint", "", "Foundry project endpoint.") +} + +// prepareGeneration resolves everything both commands need before they diverge. +// +// The model check happens here rather than at the service, because a generation +// job is billed against a deployment and a rejection partway through the +// command says less than a refusal at the flag that caused it. +func prepareGeneration( + cmd *cobra.Command, + f *generateFlags, + maxSamples, traceDays int, +) (*evalContext, *project.GenerateConfig, string, error) { + instruction, err := resolveInstruction(f.instruction, f.instructionFile) + if err != nil { + return nil, nil, "", err + } + + cfg, err := resolveGenerateConfig( + f.configPath, f.target, f.model, "", maxSamples, traceDays, + ) + if err != nil { + return nil, nil, "", err + } + if err := cfg.Validate(); err != nil { + return nil, nil, "", err + } + if !isJSON(cmd) { + warnIgnoredTraceFields(cfg, cmd.OutOrStdout()) + } + if generationModel(cfg) == "" { + return nil, nil, "", fmt.Errorf( + "a model deployment is required to generate: pass --generation-model, " + + "or set it in the generation spec") + } + + ctx := cmd.Context() + ec, err := newEvalContext(ctx, f.endpoint) + if err != nil { + return nil, nil, "", err + } + + instruction, err = ec.resolveGenerationInstruction( + ctx, cfg, instruction, f.configPath, cmd.OutOrStdout(), isJSON(cmd), + ) + if err != nil { + ec.Close() + return nil, nil, "", err + } + return ec, cfg, instruction, nil +} + +func newDatasetGenerateCommand() *cobra.Command { + var ( + flags generateFlags + maxSamples int + ) + + cmd := &cobra.Command{ + Use: "generate ", + Short: "Generate a dataset and download it.", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + name := args[0] + + ec, cfg, instruction, err := prepareGeneration(cmd, &flags, maxSamples, 0) + if err != nil { + return err + } + defer ec.Close() + + if cfg.Generate.Dataset == nil { + return fmt.Errorf("the generation spec declares no dataset to generate") + } + cfg.Generate.Dataset.Name = name + + ref, err := ec.generateDataset( + cmd.Context(), cfg, instruction, flags.outputDir, cmd.OutOrStdout(), flags.noWait) + if err != nil { + return err + } + return reportGenerated(cmd, ref, flags.noWait) + }, + } + + cmd.Flags().IntVar(&maxSamples, "max-samples", 0, + fmt.Sprintf("Rows to synthesize (%d-%d).", project.MinSampleSize, project.MaxSampleSize)) + addGenerateFlags(cmd, &flags) + return cmd +} + +func newEvaluatorGenerateCommand() *cobra.Command { + var ( + flags generateFlags + traceDays int + ) + + cmd := &cobra.Command{ + Use: "generate ", + Short: "Generate a rubric evaluator and download it.", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + name := args[0] + + ec, cfg, instruction, err := prepareGeneration(cmd, &flags, 0, traceDays) + if err != nil { + return err + } + defer ec.Close() + + if cfg.Generate.Rubric == nil { + return fmt.Errorf("the generation spec declares no rubric to generate") + } + cfg.Generate.Rubric.Name = name + + ref, err := ec.generateRubric( + cmd.Context(), cfg, instruction, flags.outputDir, cmd.OutOrStdout(), flags.noWait) + if err != nil { + return err + } + return reportGenerated(cmd, ref, flags.noWait) + }, + } + + cmd.Flags().IntVar(&traceDays, "trace-days", 0, + "Days of traces to seed generation. 0 disables.") + addGenerateFlags(cmd, &flags) + return cmd +} + +// reportGenerated closes out either command. +// +// With --no-wait nothing was downloaded and there is no ref, which is success: +// the job id was printed and `job show` reattaches to it. +func reportGenerated(cmd *cobra.Command, ref *project.ArtifactRef, noWait bool) error { + out := cmd.OutOrStdout() + if ref == nil { + if noWait { + fmt.Fprintln(out, + "\nSubmitted. `azd ai eval job show ` reports its progress.") + return nil + } + fmt.Fprintln(out, "Nothing was generated.") + return nil + } + if isJSON(cmd) { + return emitJSON(out, ref) + } + fmt.Fprintf(out, "\nReference it from your eval config as: %s\n", ref.Source) + return nil +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/root.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/root.go index 24323e839b9..5be5fcc9e46 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/root.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/root.go @@ -46,7 +46,6 @@ func NewRootCommand() *cobra.Command { rootCmd.AddCommand( newInitCommand(), - newGenerateCommand(), newDatasetCommand(), newRunCommand(), newEvaluatorCommand(), diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go index 73b963960b4..7c2ab84e458 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go @@ -60,6 +60,7 @@ func buildRunCommand(use, short string) *cobra.Command { level string maxSamples int wait bool + failOn string endpointFlg string ) @@ -70,6 +71,13 @@ func buildRunCommand(use, short string) *cobra.Command { ctx := cmd.Context() out := cmd.OutOrStdout() + // Parsed before any network work, so a malformed threshold costs + // nothing to find out about. + threshold, err := parseGate(failOn) + if err != nil { + return err + } + ec, err := newEvalContext(ctx, endpointFlg) if err != nil { return err @@ -167,9 +175,18 @@ func buildRunCommand(use, short string) *cobra.Command { } if isJSON(cmd) { - return emitJSON(out, final) + if err := emitJSON(out, final); err != nil { + return err + } + } else if err := renderRun(out, final); err != nil { + return err } - return renderRun(out, final) + + // Last, so that the results are reported whether or not the gate + // holds: a pipeline that only learns it failed is worse off than + // one that can see by how much. + applyGate(cmd, threshold, final) + return nil }, } @@ -185,6 +202,7 @@ func buildRunCommand(use, short string) *cobra.Command { cmd.Flags().IntVar(&maxSamples, "max-samples", 0, "Cap the rows sent from the dataset.") cmd.Flags().BoolVar(&wait, "wait", true, "Block until the run reaches a terminal state.") + addFailOnFlag(cmd, &failOn) // The spec documents --no-wait, and cobra does not derive it from a bool. var noWait bool cmd.Flags().BoolVar(&noWait, "no-wait", false, "Submit the run and return immediately.") diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go index 71af58eccab..e7e537e544c 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go @@ -90,6 +90,8 @@ func newRunShowCommand() *cobra.Command { var ( endpointFlg string groupName string + wait bool + failOn string ) cmd := &cobra.Command{ @@ -98,6 +100,12 @@ func newRunShowCommand() *cobra.Command { Args: cobra.MaximumNArgs(1), RunE: func(cmd *cobra.Command, args []string) error { ctx := cmd.Context() + + threshold, err := parseGate(failOn) + if err != nil { + return err + } + ec, err := newEvalContext(ctx, endpointFlg) if err != nil { return err @@ -114,8 +122,22 @@ func newRunShowCommand() *cobra.Command { if err != nil { return err } + + // Reattaching to a run started asynchronously: the pipeline that + // gates on it is often not the one that started it. + if wait { + run, err = ec.pollRun(ctx, evalID, run.ID, cmd.OutOrStdout(), isJSON(cmd)) + if err != nil { + return err + } + } + if isJSON(cmd) { - return emitJSON(cmd.OutOrStdout(), run) + if err := emitJSON(cmd.OutOrStdout(), run); err != nil { + return err + } + applyGate(cmd, threshold, run) + return nil } out := cmd.OutOrStdout() @@ -128,9 +150,13 @@ func newRunShowCommand() *cobra.Command { if run.ReportURL != "" { fmt.Fprintf(out, " report : %s\n", run.ReportURL) } + applyGate(cmd, threshold, run) return nil }, } + cmd.Flags().BoolVar(&wait, "wait", false, + "Block until the run reaches a terminal state before reporting.") + addFailOnFlag(cmd, &failOn) addEvalFlags(cmd, &groupName) cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") return cmd diff --git a/cli/azd/extensions/azure.ai.evaluations/tests/cli/generate_test.go b/cli/azd/extensions/azure.ai.evaluations/tests/cli/generate_test.go index ff3f37b0a31..cc4eed3edac 100644 --- a/cli/azd/extensions/azure.ai.evaluations/tests/cli/generate_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/tests/cli/generate_test.go @@ -8,21 +8,21 @@ package cli import ( "os" "path/filepath" - "strings" "testing" "github.com/stretchr/testify/require" ) -// `generate` submits two jobs that cost model time and take minutes, so what -// is exercised here is everything up to that point: the flag combinations it -// refuses, the spec it parses, and the two flags that mean "I already have -// this one, do not make another". None of these tests submits a job — the last -// one reaches the service and deliberately generates nothing. +// Generation submits a job that costs model time and takes minutes, so what is +// exercised here is everything up to that point: the flag combinations each +// command refuses and the spec it parses. No test here submits a job. +// +// There is one command per artifact, so nothing suppresses anything: a caller +// who already has a dataset simply does not run `dataset generate`. // TestCLIGenerateRefusesBadFlagCombinations covers the mistakes that must cost -// nothing to make. Each of these is decided locally, so a user finds out -// before a job is billed. +// nothing to make. Each is decided locally, so a user finds out before a job is +// billed. func TestCLIGenerateRefusesBadFlagCombinations(t *testing.T) { dir := t.TempDir() instruction := filepath.Join(dir, "instruction.md") @@ -34,40 +34,56 @@ func TestCLIGenerateRefusesBadFlagCombinations(t *testing.T) { want string }{{ name: "the two instruction sources are mutually exclusive", - args: []string{"--target", "a", "--agent-instruction", "inline", - "--agent-instruction-file", instruction}, + args: []string{"dataset", "generate", "d", "--target", "a", + "--agent-instruction", "inline", "--agent-instruction-file", instruction}, want: "agent-instruction-file", }, { name: "below the minimum sample size", - args: []string{"--target", "a", "--max-samples", "14"}, + args: []string{"dataset", "generate", "d", "--target", "a", "--max-samples", "14"}, want: "between 15 and 1000", }, { name: "above the maximum sample size", - args: []string{"--target", "a", "--max-samples", "1001"}, + args: []string{"dataset", "generate", "d", "--target", "a", "--max-samples", "1001"}, want: "between 15 and 1000", }, { name: "a missing instruction file names the flag", - args: []string{"--target", "a", "--agent-instruction-file", - filepath.Join(dir, "absent.md")}, + args: []string{"dataset", "generate", "d", "--target", "a", + "--agent-instruction-file", filepath.Join(dir, "absent.md")}, want: "--agent-instruction-file", }, { - name: "generating needs a model deployment", - args: []string{"--target", "a", "--agent-instruction", "inline"}, + name: "generating a dataset needs a model deployment", + args: []string{"dataset", "generate", "d", "--target", "a", "--agent-instruction", "inline"}, + want: "--generation-model", + }, { + name: "generating an evaluator needs a model deployment", + args: []string{"evaluator", "generate", "e", "--target", "a", "--agent-instruction", "inline"}, want: "--generation-model", }} for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { - r := requireFailure(t, runIn(t, dir, append([]string{"generate"}, tc.args...)...)) + r := requireFailure(t, runIn(t, dir, tc.args...)) require.Contains(t, r.Combined(), tc.want) }) } } +// TestCLIGenerateNamesTheArtifact pins the positional argument. Without it the +// name would come from the spec, and two runs would quietly overwrite the same +// artifact. +func TestCLIGenerateNamesTheArtifact(t *testing.T) { + for _, group := range []string{"dataset", "evaluator"} { + t.Run(group, func(t *testing.T) { + r := requireFailure(t, runIn(t, t.TempDir(), group, "generate")) + require.Contains(t, r.Combined(), "accepts 1 arg") + }) + } +} + // TestCLIGenerateNoPromptNamesWhatIsMissing is the CI case: with no target and // nothing to prompt with, the process has to end saying which flag to pass. func TestCLIGenerateNoPromptNamesWhatIsMissing(t *testing.T) { - r := requireFailure(t, runIn(t, t.TempDir(), "generate", "--no-prompt")) + r := requireFailure(t, runIn(t, t.TempDir(), "dataset", "generate", "d", "--no-prompt")) require.Contains(t, r.Combined(), "--target is required") require.Contains(t, r.Combined(), "--no-prompt", "the message must say why it could not be resolved") @@ -91,68 +107,26 @@ generate: strategy: from-traces `), 0o600)) - r := requireFailure(t, runIn(t, dir, "generate", "--config", spec)) + r := requireFailure(t, runIn(t, dir, "dataset", "generate", "d", "--config", spec)) require.Contains(t, r.Combined(), "from-traces") require.Contains(t, r.Combined(), "agent.context.traces.window", "the refusal must point at the field that does seed generation from traces") } -// TestCLIGenerateSkipsWhatWasSupplied is the one generate test that reaches the -// service, and it is here because the skip decision is made in the command -// body rather than in the config resolver. -// -// With both artifacts supplied there is nothing left to generate, so the whole -// command runs without submitting a job — which is what makes it affordable to -// assert on. A regression that stopped honouring either flag would show up as -// a generation job starting instead of this returning. -func TestCLIGenerateSkipsWhatWasSupplied(t *testing.T) { +// TestCLIGenerateFlagsAreScopedToTheirArtifact asserts the two commands do not +// share settings that only one of them can honour. A sample count means nothing +// to a rubric, and a trace window means nothing to a synthetic dataset; either +// would be accepted and dropped. +func TestCLIGenerateFlagsAreScopedToTheirArtifact(t *testing.T) { dir := t.TempDir() - r := requireSuccess(t, runIn(t, dir, "generate", - "--target", "azd-eval-probe-agent", - "--agent-instruction", "answer questions about orders", - "--evaluator", "already-published", - "--dataset", "already-registered")) - - require.Contains(t, r.Stdout, "skipping rubric generation") - require.Contains(t, r.Stdout, "skipping data generation") - require.Contains(t, r.Stdout, "Nothing was generated.") - - // The deployment spec is only rewritten when something was produced, and - // writing an empty reference into it would be worse than not writing. - entries, err := os.ReadDir(dir) - require.NoError(t, err) - require.Empty(t, entries, "a generate that produced nothing must write nothing") -} - -// TestCLIGenerateSuppressionIsPerArtifact pins the two flags apart: neither -// may suppress the artifact it does not name. -// -// Each case supplies one artifact and leaves the other to be generated, and is -// stopped at the model check that precedes submission. Reaching that error is -// the proof: it is only raised when something is still going to be generated, -// so it says the unsupplied artifact survived the other flag. -func TestCLIGenerateSuppressionIsPerArtifact(t *testing.T) { - cases := []struct { - name string - supplied []string - survives string - }{ - {"a supplied evaluator leaves the dataset", []string{"--evaluator", "already-published"}, "dataset"}, - {"a supplied dataset leaves the rubric", []string{"--dataset", "already-registered"}, "rubric"}, - } + r := requireFailure(t, runIn(t, dir, "evaluator", "generate", "e", + "--target", "a", "--max-samples", "20")) + require.Contains(t, r.Combined(), "max-samples", + "--max-samples belongs to dataset generate") - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - args := append([]string{"generate", - "--target", "azd-eval-probe-agent", - "--agent-instruction", "answer questions about orders"}, tc.supplied...) - - r := requireFailure(t, runIn(t, t.TempDir(), args...)) - require.Contains(t, r.Combined(), "--generation-model", - "the %s was suppressed by a flag that does not name it", tc.survives) - require.NotContains(t, strings.ToLower(r.Combined()), "generating ", - "the run must stop at the model check, before any job is submitted") - }) - } + r = requireFailure(t, runIn(t, dir, "dataset", "generate", "d", + "--target", "a", "--trace-days", "7")) + require.Contains(t, r.Combined(), "trace-days", + "--trace-days belongs to evaluator generate") } From 4199cd283dac3736f1f17f85e568cc54d39188e1 Mon Sep 17 00:00:00 2001 From: mohessie Date: Mon, 3 Aug 2026 17:03:22 +0300 Subject: [PATCH 079/320] Match the gate wording the scenario documents The pass rate reads as a percentage and the failure prints the two lines the CI scenario shows, so a pipeline log looks like the spec rather than close to it. Also records what the counts mean, which the contract left ambiguous: errored rows sit inside total. Verified on a run built to mix them - total=3 passed=2 errored=1. Had they been outside it, two passes and one error would have reported total=2 and scored a perfect rate, which is the broken evaluation a gate exists to catch. --- .../internal/cmd/gating.go | 20 +++++++++++-------- .../internal/cmd/gating_test.go | 12 +++++++++-- 2 files changed, 22 insertions(+), 10 deletions(-) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/gating.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/gating.go index c7754fa5064..0b6940a28d5 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/gating.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/gating.go @@ -62,11 +62,14 @@ func parseGate(spec string) (gate, error) { // breach reports why the run missed the threshold, or empty when it met it. // -// Errored and skipped rows count against the pass rate: they are in the total -// and they did not pass. A run that scored nothing at all breaches every -// threshold rather than dividing by zero — "no rows passed" is the honest -// reading of an empty result, and treating it as 100% would let a broken -// evaluation hold a gate open. +// Errored and skipped rows count against the pass rate, and they can: the +// service puts them inside `total`, verified live on a run that reported +// total=3 passed=2 errored=1. Were they outside it, a run with two passes and +// one error would report total=2 and score a perfect rate, which is precisely +// the broken evaluation a gate exists to catch. +// +// A run that scored nothing at all breaches every threshold rather than +// dividing by zero — "no rows passed" is the honest reading of an empty result. func (g gate) breach(counts *eval_api.EvalRunResultCounts) string { if !g.set { return "" @@ -86,8 +89,8 @@ func (g gate) breach(counts *eval_api.EvalRunResultCounts) string { } actual := float64(counts.Passed) / float64(counts.Total) if actual < g.passRate { - return fmt.Sprintf("pass rate %.3f is below the required %.3f (%d of %d passed)", - actual, g.passRate, counts.Passed, counts.Total) + return fmt.Sprintf("pass rate %.1f%% is below the required %.1f%%", + actual*100, g.passRate*100) } return "" } @@ -106,7 +109,8 @@ func applyGate(cmd *cobra.Command, g gate, run *eval_api.OpenAIEvalRun) { if reason == "" { return } - fmt.Fprintf(os.Stderr, "Evaluation gate failed: %s\n", reason) + fmt.Fprintf(os.Stderr, "(x) Failed: Evaluation gate: %s\n\n", reason) + fmt.Fprintln(os.Stderr, "ERROR: evaluation quality gate not met.") os.Exit(exitCodeGateBreached) } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/gating_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/gating_test.go index fb612cd6aca..5952e53d694 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/gating_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/gating_test.go @@ -51,8 +51,8 @@ func TestGateBreach(t *testing.T) { require.NotEmpty(t, anyFailure.breach(&eval_api.EvalRunResultCounts{Total: 2, Passed: 1, Failed: 1})) }) - // Errored rows are in the total and did not pass, so they count against the - // threshold the same way a failing row does. + // Errored rows are inside the total, verified live, so they count against + // the threshold the same way a failing row does. t.Run("errored rows count against the rate", func(t *testing.T) { counts := &eval_api.EvalRunResultCounts{Total: 10, Passed: 8, Errored: 2} require.Empty(t, eighty.breach(counts), "0.8 exactly meets a 0.8 threshold") @@ -61,6 +61,14 @@ func TestGateBreach(t *testing.T) { require.NotEmpty(t, eighty.breach(counts)) }) + // The wording is pinned because the hero scenario shows it verbatim. + t.Run("reads as a percentage", func(t *testing.T) { + counts := &eval_api.EvalRunResultCounts{Total: 1000, Passed: 764} + require.Equal(t, + "pass rate 76.4% is below the required 80.0%", + eighty.breach(counts)) + }) + // A run that scored nothing has no defensible pass rate, and treating it as // 100% would let a broken evaluation hold a gate open. t.Run("a run that scored nothing breaches", func(t *testing.T) { From a36717658e4c094b92436b7f37a69fb2ceca275e Mon Sep 17 00:00:00 2001 From: mohessie Date: Mon, 3 Aug 2026 17:18:44 +0300 Subject: [PATCH 080/320] Ask the direct read whether an evaluator exists create and update differ only in which starting state they accept, and the check was asking the version listing. That listing lags a publish by up to a second and a half, so an update issued straight after a create was told the evaluator it had just made did not exist - which the live suite caught, with show succeeding between the two because the direct read had already converged. The same lag is why reconciliation waits on both views before creating an eval that references a fresh evaluator. Here only one view is needed, and it is the faster one. --- .../internal/cmd/evaluator.go | 13 ++++++++++--- .../internal/pkg/evalcore/evaluator.go | 16 ++++++++++++++-- 2 files changed, 24 insertions(+), 5 deletions(-) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator.go index a4750af5a00..2407f4434ee 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator.go @@ -82,10 +82,17 @@ func newEvaluatorWriteCommand(verb, short string) *cobra.Command { } defer ec.Close() - latest := ec.evalClient.LatestEvaluatorVersionNumber( - ctx, name, ProjectEndpointAPIVersion, + // Asked of the direct read, not the version listing. The listing + // lags a publish by up to a second and a half, so an update + // issued straight after a create would be told the evaluator it + // just made does not exist. + _, readErr := ec.evalClient.GetEvaluatorRaw( + ctx, name, "", ProjectEndpointAPIVersion, ) - if err := checkAssetExistence(verb, "evaluator", name, latest > 0); err != nil { + if readErr != nil && !eval_api.IsNotFound(readErr) { + return fmt.Errorf("checking whether evaluator %q exists: %w", name, readErr) + } + if err := checkAssetExistence(verb, "evaluator", name, readErr == nil); err != nil { return err } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/evaluator.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/evaluator.go index f31584fd48a..88538768b37 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/evaluator.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/evaluator.go @@ -17,14 +17,26 @@ import ( const BuiltinPrefix = "builtin." // EvaluatorRef references an evaluator from an eval. It accepts either a -// bare string or a mapping carrying a pass threshold: +// bare string or a mapping carrying the rest of the declaration: // // evaluators: // - builtin.task_adherence -// - { name: support-quality, threshold: 4.0 } +// - name: support-quality +// source: ./evaluators/support-quality.json +// initialization_parameters: +// deployment_name: gpt-5.6-luna +// +// A built-in needs nothing but its name. One with a Source is the project's +// own, and is published before the eval that references it is created. type EvaluatorRef struct { Name string `yaml:"name" json:"name"` Version string `yaml:"version,omitempty" json:"version,omitempty"` + // Source names a local rubric JSON file. Empty means the evaluator is + // already registered, or is a built-in. + Source string `yaml:"source,omitempty" json:"source,omitempty"` + // InitializationParameters are passed through to the testing criterion, + // which is where a judge deployment is named. + InitializationParameters map[string]any `yaml:"initialization_parameters,omitempty" json:"initialization_parameters,omitempty"` // Threshold maps to testing_criteria[].initialization_parameters.threshold. Threshold *float64 `yaml:"threshold,omitempty" json:"threshold,omitempty"` } From 60239952f44a37658b6a698f572c10944e594d26 Mon Sep 17 00:00:00 2001 From: mohessie Date: Mon, 3 Aug 2026 19:26:49 +0300 Subject: [PATCH 081/320] One service per eval: restructure the configuration model The spec's configuration model is one `azure.ai.eval` service per eval, with the eval's body in `evals/.yaml` and the service key supplying its name. The extension implemented an older shape: a single `evals/azure.yaml` carrying parallel `datasets:`, `evaluators:` and `evals:` lists. That made the service key meaningless, gave azd no way to order one eval after the agent it evaluates, and forced a name to be repeated in three places. - `EvalConfig` is now one eval: `description`, `dataset`, `evaluators`, `target`, `options`. `Eval(name)` resolves it against the service key. - Evaluators are declared once, in the list the eval references. A built-in needs nothing; one carrying a `source` is the config's to publish. `CustomEvaluators()` is that subset. - `ResolveEvalConfigPath` resolves `--eval ` to a file, and names the candidates rather than guessing when there is more than one. `options.eval_model` is gone. A judge deployment is a testing criterion's `initialization_parameters.deployment_name`, which differs per evaluator, so it is declared on the evaluator reference; `--judge-model` goes with it. Verified against the live schemas: every built-in that judges declares `deployment_name` required, so `init` stamps the detected deployment onto each evaluator it writes rather than leaving behind an eval that cannot run. `EvaluatorList.MarshalJSON` dropped `source` and `initialization_parameters`. The eval fingerprint is taken over that encoding, so a judge-model change was invisible to reconciliation and would not have recreated the eval. Caught by the fingerprint test. `evals/generate.yaml` takes its documented shape: `generationModel` plus `dataset:`/`evaluator:` maps keyed by artifact name, so `dataset generate ` reads exactly the entry it was asked for. The resolution order -- flags, spec, then the eval's own target -- is settled in `resolvePlan` before the client is built, so a missing model or an out-of-range sample count costs no authentication round trip. `MergeArtifactRefs` is deleted. The spec is explicit that `generate` writes artifacts only and never edits `azure.yaml`, which is what keeps a generation run a reviewable data-file-only diff; the code was already unreachable from any command. `init` follows: `--name`, `--output-dir` (was `--out-dir`), `--generation-model`, a `-smoke` default eval name, the built-in plus generated rubric default evaluator set, and a service entry keyed by the eval name that `uses` both the Foundry project and the target agent. --- .../internal/cmd/agent_context_test.go | 99 +---- .../internal/cmd/build.go | 32 +- .../internal/cmd/build_live_test.go | 25 +- .../internal/cmd/build_test.go | 60 +-- .../internal/cmd/description_test.go | 7 +- .../internal/cmd/envkeys_test.go | 33 +- .../internal/cmd/generate.go | 214 +++------- .../internal/cmd/generate_commands.go | 185 +++++++-- .../internal/cmd/generate_defaults_test.go | 47 --- .../internal/cmd/generate_plan_test.go | 291 +++++++------- .../azure.ai.evaluations/internal/cmd/init.go | 379 +++++++++++------- .../internal/cmd/init_test.go | 132 ++++-- .../internal/cmd/init_wiring_test.go | 66 ++- .../internal/cmd/output.go | 8 + .../azure.ai.evaluations/internal/cmd/run.go | 50 ++- .../internal/cmd/run_output.go | 15 +- .../internal/cmd/schemas_live_test.go | 6 +- .../internal/cmd/trace_warning_test.go | 46 --- .../internal/pkg/evalcore/evaluator.go | 29 +- .../internal/project/eval_config.go | 248 ++++++------ .../internal/project/eval_config_test.go | 263 +++++++----- .../internal/project/generate_config.go | 180 +++++---- .../internal/project/generate_config_test.go | 74 ++-- .../internal/project/merge.go | 171 -------- .../internal/project/merge_test.go | 142 ------- .../internal/project/service_target_eval.go | 44 +- .../project/service_target_eval_test.go | 48 +-- .../tests/cli/evaluator_test.go | 2 +- .../tests/cli/generate_test.go | 52 ++- 29 files changed, 1408 insertions(+), 1540 deletions(-) delete mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_defaults_test.go delete mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/trace_warning_test.go delete mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/project/merge.go delete mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/project/merge_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/agent_context_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/agent_context_test.go index 7ba9e5ce23a..e6fa74be253 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/agent_context_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/agent_context_test.go @@ -4,7 +4,6 @@ package cmd import ( - "bytes" "encoding/json" "os" "path/filepath" @@ -20,7 +19,7 @@ import ( // The generation spec names an instructions file relative to itself, not to the // working directory, so `generate --config ` reads the same file the // author sees next to the spec. -func TestAgentContextInstructions_ResolvesRelativeToTheSpec(t *testing.T) { +func TestDeclaredInstructions_ResolvesRelativeToTheSpec(t *testing.T) { dir := t.TempDir() specDir := filepath.Join(dir, "evals") require.NoError(t, os.MkdirAll(filepath.Join(specDir, "agent"), 0o755)) @@ -29,28 +28,23 @@ func TestAgentContextInstructions_ResolvesRelativeToTheSpec(t *testing.T) { require.NoError(t, os.WriteFile( filepath.Join(specDir, "agent", "instructions.md"), []byte(" "+body+"\n"), 0o600)) - cfg := &project.GenerateConfig{} - cfg.Agent.Context.Instructions = "./agent/instructions.md" - - got, err := agentContextInstructions(cfg, filepath.Join(specDir, "eval_generate.yaml")) + got, err := declaredInstructions( + "./agent/instructions.md", filepath.Join(specDir, "generate.yaml")) require.NoError(t, err) assert.Equal(t, body, got, "the file's contents should be used, trimmed") } -// `init` writes the instructions path before that file exists. Treating the -// gap as an error would break the flow init itself scaffolds. -func TestAgentContextInstructions_MissingFileIsNotAnError(t *testing.T) { - dir := t.TempDir() - cfg := &project.GenerateConfig{} - cfg.Agent.Context.Instructions = "./agent/instructions.md" - - got, err := agentContextInstructions(cfg, filepath.Join(dir, "eval_generate.yaml")) +// A path can be declared before that file exists. Treating the gap as an error +// would break the flow `init` itself scaffolds. +func TestDeclaredInstructions_MissingFileIsNotAnError(t *testing.T) { + got, err := declaredInstructions( + "./agent/instructions.md", filepath.Join(t.TempDir(), "generate.yaml")) require.NoError(t, err) assert.Empty(t, got) } -func TestAgentContextInstructions_UnsetIsEmpty(t *testing.T) { - got, err := agentContextInstructions(&project.GenerateConfig{}, "eval_generate.yaml") +func TestDeclaredInstructions_UnsetIsEmpty(t *testing.T) { + got, err := declaredInstructions("", "generate.yaml") require.NoError(t, err) assert.Empty(t, got) } @@ -75,73 +69,10 @@ func TestAgentInstructions(t *testing.T) { assert.Empty(t, nilAgent.Instructions()) } -// Dataset generation has no model of its own; it runs against the judge model -// the spec declares. +// Dataset generation has no model of its own; both jobs run against the one +// generation model the spec declares. func TestGenerationModel(t *testing.T) { - cfg := &project.GenerateConfig{} - assert.Empty(t, generationModel(cfg), "no rubric means no model to borrow") - - cfg.Generate.Rubric = &project.RubricSpec{Model: "gpt-4.1-nano"} - assert.Equal(t, "gpt-4.1-nano", generationModel(cfg)) -} - -// Trace selection is accepted and ignored, so it has to be called out. -func TestWarnIgnoredFields_CoversTraceSelection(t *testing.T) { - cases := []struct { - name string - build func(*project.GenerateConfig) - want []string - quiet bool - }{ - { - name: "nothing set stays silent", - build: func(*project.GenerateConfig) {}, - quiet: true, - }, - { - name: "a source alone", - build: func(c *project.GenerateConfig) { - c.Agent.Context.Traces = &project.TraceSpec{Source: "app-insights"} - }, - want: []string{"agent.context.traces.source", "has no effect"}, - }, - { - name: "source and sample agree in number", - build: func(c *project.GenerateConfig) { - c.Agent.Context.Traces = &project.TraceSpec{Source: "app-insights", Sample: 100} - }, - want: []string{"agent.context.traces.source", "agent.context.traces.sample", "have no effect"}, - }, - { - name: "a window alone is honored, so no warning", - build: func(c *project.GenerateConfig) { - c.Agent.Context.Traces = &project.TraceSpec{Window: "7d"} - }, - quiet: true, - }, - } - - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - cfg := &project.GenerateConfig{} - tc.build(cfg) - - var buf bytes.Buffer - warnIgnoredTraceFields(cfg, &buf) - - if tc.quiet { - assert.Empty(t, buf.String()) - return - } - for _, want := range tc.want { - assert.Contains(t, buf.String(), want) - } - }) - } -} - -// init scaffolds only the context fields that are read. -func TestInitScaffold_KeepsInstructions(t *testing.T) { - cfg := buildGenerateScaffold("support-agent", "support-agent-quality", "gpt-4.1-nano") - assert.Equal(t, "./agent/instructions.md", cfg.Agent.Context.Instructions) + assert.Empty(t, generationModel(&project.GenerateConfig{})) + assert.Equal(t, "gpt-4.1-nano", + generationModel(&project.GenerateConfig{GenerationModel: "gpt-4.1-nano"})) } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build.go index d5008b57b9d..37f220dcab9 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build.go @@ -148,7 +148,6 @@ func planCriterion( schema *eval_api.EvaluatorSummary, targetBindings map[string]string, datasetColumns map[string]bool, - evalModel string, level string, ) (*criterionPlan, error) { accepted := legacyInputs @@ -214,13 +213,16 @@ func planCriterion( } // Evaluators disagree on what the judge model is called: built-ins declare - // deployment_name, custom rubrics declare model. Bind whichever the - // evaluator actually accepts rather than guessing one spelling. - if evalModel != "" { - for _, alias := range []string{"deployment_name", "model"} { - if accepts(alias) { - plan.initParams[alias] = evalModel - } + // deployment_name, custom rubrics declare model. The declaration names one + // of them; bind whichever the evaluator actually accepts rather than + // forwarding a spelling it will reject. + for name, value := range ref.InitializationParameters { + if accepts(name) { + plan.initParams[name] = value + continue + } + if alias, ok := judgeModelAliases[name]; ok && accepts(alias) { + plan.initParams[alias] = value } } if ref.Threshold != nil && accepts("threshold") { @@ -239,7 +241,8 @@ func planCriterion( } if len(missingInit) > 0 { return nil, fmt.Errorf( - "evaluator %q requires %s; set the judge model on the eval", + "evaluator %q requires %s; set it under the evaluator's "+ + "`initialization_parameters` in the eval config", ref.Name, quoteList(missingInit), ) } @@ -248,6 +251,13 @@ func planCriterion( return plan, nil } +// judgeModelAliases maps the two spellings of the judge deployment onto each +// other, so one declaration works whichever the evaluator publishes. +var judgeModelAliases = map[string]string{ + "deployment_name": "model", + "model": "deployment_name", +} + // buildEvalRequest converts an eval declaration into the create // request. Each evaluator becomes a testing criterion bound to its own // contract, and the item schema declares every dataset column those bindings @@ -279,10 +289,8 @@ func buildEvalRequest( metadata["azd_description"] = group.Description } - evalModel := "" level := "" if group.Options != nil { - evalModel = group.Options.EvalModel level = group.Options.EvaluationLevel } @@ -299,7 +307,7 @@ func buildEvalRequest( schema = &eval_api.EvaluatorSummary{Name: ref.Name} } - plan, err := planCriterion(ref, schema, targetBindings, datasetColumns, evalModel, level) + plan, err := planCriterion(ref, schema, targetBindings, datasetColumns, level) if err != nil { return nil, err } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build_live_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build_live_test.go index 1018e9cb4e4..64bcf50b2c2 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build_live_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build_live_test.go @@ -114,11 +114,14 @@ func TestLiveBuildAcceptedForEveryBuiltin(t *testing.T) { } group := &project.Eval{ - Name: fmt.Sprintf("azd-live-%d", time.Now().UTC().UnixNano()), - Dataset: "inline", - Target: &project.Target{Type: "agent", Name: "probe-agent"}, - Evaluators: []evalcore.EvaluatorRef{{Name: summary.Name}}, - Options: &project.Options{EvalModel: judge, EvaluationLevel: level}, + Name: fmt.Sprintf("azd-live-%d", time.Now().UTC().UnixNano()), + Dataset: "inline", + Target: &project.Target{Type: "agent", Name: "probe-agent"}, + Evaluators: []evalcore.EvaluatorRef{{ + Name: summary.Name, + InitializationParameters: map[string]any{"deployment_name": judge}, + }}, + Options: &project.Options{EvaluationLevel: level}, } req, err := buildEvalRequest(group, schemas, columns) @@ -156,11 +159,13 @@ func TestLiveBuildRejectsMissingColumnsLocally(t *testing.T) { "this test relies on ifeval declaring required inputs") group := &project.Eval{ - Name: "azd-live-negative", - Dataset: "inline", - Target: &project.Target{Type: "agent", Name: "probe-agent"}, - Evaluators: []evalcore.EvaluatorRef{{Name: "builtin.ifeval"}}, - Options: &project.Options{EvalModel: judge}, + Name: "azd-live-negative", + Dataset: "inline", + Target: &project.Target{Type: "agent", Name: "probe-agent"}, + Evaluators: []evalcore.EvaluatorRef{{ + Name: "builtin.ifeval", + InitializationParameters: map[string]any{"deployment_name": judge}, + }}, } // A dataset with only `query` cannot satisfy ifeval. diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build_test.go index f6c80e602a8..9a17598b680 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build_test.go @@ -45,6 +45,15 @@ func groupWith(evaluators []evalcore.EvaluatorRef, opts *project.Options) *proje } } +// withJudge declares the judge deployment where the service reads it from: an +// evaluator's initialization parameters, not a setting on the eval. +func withJudge(model string, refs ...evalcore.EvaluatorRef) []evalcore.EvaluatorRef { + for i := range refs { + refs[i].InitializationParameters = map[string]any{"deployment_name": model} + } + return refs +} + // An agent evaluator takes its response from the sample and its query from the // dataset. func TestBuildBindsAgentFieldsFromSample(t *testing.T) { @@ -55,8 +64,8 @@ func TestBuildBindsAgentFieldsFromSample(t *testing.T) { "turn"), } group := groupWith( - []evalcore.EvaluatorRef{{Name: "builtin.task_adherence"}}, - &project.Options{EvalModel: "gpt-4.1-nano"}, + withJudge("gpt-4.1-nano", evalcore.EvaluatorRef{Name: "builtin.task_adherence"}), + nil, ) req, err := buildEvalRequest(group, schemas, map[string]bool{"query": true}) @@ -126,10 +135,10 @@ func TestBuildOmitsUnacceptedInitParameters(t *testing.T) { nil, []string{"query", "response", "ground_truth"}, []string{"deployment_name"}, []string{"deployment_name", "threshold"}, "turn"), } - group := groupWith([]evalcore.EvaluatorRef{ - {Name: "builtin.ifeval", Threshold: &threshold}, - {Name: "builtin.similarity", Threshold: &threshold}, - }, &project.Options{EvalModel: "gpt-4.1-nano"}) + group := groupWith(withJudge("gpt-4.1-nano", + evalcore.EvaluatorRef{Name: "builtin.ifeval", Threshold: &threshold}, + evalcore.EvaluatorRef{Name: "builtin.similarity", Threshold: &threshold}, + ), nil) req, err := buildEvalRequest(group, schemas, map[string]bool{ "query": true, "ground_truth": true, @@ -158,10 +167,10 @@ func TestBuildPassesEvaluationLevelAsInitParameter(t *testing.T) { nil, []string{"query", "response"}, []string{"deployment_name"}, []string{"deployment_name", "threshold"}, "turn"), } - group := groupWith([]evalcore.EvaluatorRef{ - {Name: "builtin.task_completion"}, - {Name: "builtin.similarity"}, - }, &project.Options{EvalModel: "m", EvaluationLevel: "turn"}) + group := groupWith(withJudge("m", + evalcore.EvaluatorRef{Name: "builtin.task_completion"}, + evalcore.EvaluatorRef{Name: "builtin.similarity"}, + ), &project.Options{EvaluationLevel: "turn"}) req, err := buildEvalRequest(group, schemas, map[string]bool{"query": true}) require.NoError(t, err) @@ -178,8 +187,8 @@ func TestBuildRejectsUnsupportedLevel(t *testing.T) { nil, []string{"query", "response"}, []string{"deployment_name"}, []string{"deployment_name"}, "turn"), } - group := groupWith([]evalcore.EvaluatorRef{{Name: "builtin.similarity"}}, - &project.Options{EvalModel: "m", EvaluationLevel: "conversation"}) + group := groupWith(withJudge("m", evalcore.EvaluatorRef{Name: "builtin.similarity"}), + &project.Options{EvaluationLevel: "conversation"}) _, err := buildEvalRequest(group, schemas, map[string]bool{"query": true}) require.Error(t, err) @@ -204,8 +213,7 @@ func TestBuildRequiresJudgeModelWhenEvaluatorDoes(t *testing.T) { // An evaluator with no published contract keeps the historical agent-target // shape, so custom evaluators still deploy. func TestBuildFallsBackWithoutSchema(t *testing.T) { - group := groupWith([]evalcore.EvaluatorRef{{Name: "my-custom-evaluator"}}, - &project.Options{EvalModel: "m"}) + group := groupWith(withJudge("m", evalcore.EvaluatorRef{Name: "my-custom-evaluator"}), nil) req, err := buildEvalRequest(group, nil, nil) require.NoError(t, err) @@ -230,8 +238,8 @@ func TestBuildResolvesConversationTurnExclusivity(t *testing.T) { columns := map[string]bool{"query": true, "messages": true, "response": true} // Turn level keeps query/response and drops messages. - turn := groupWith([]evalcore.EvaluatorRef{{Name: "builtin.task_completion"}}, - &project.Options{EvalModel: "m", EvaluationLevel: "turn"}) + turn := groupWith(withJudge("m", evalcore.EvaluatorRef{Name: "builtin.task_completion"}), + &project.Options{EvaluationLevel: "turn"}) req, err := buildEvalRequest(turn, schemas, columns) require.NoError(t, err) mapping := req.TestingCriteria[0].DataMapping @@ -239,8 +247,8 @@ func TestBuildResolvesConversationTurnExclusivity(t *testing.T) { require.NotContains(t, mapping, "messages") // Conversation level keeps messages and drops query/response. - conv := groupWith([]evalcore.EvaluatorRef{{Name: "builtin.task_completion"}}, - &project.Options{EvalModel: "m", EvaluationLevel: "conversation"}) + conv := groupWith(withJudge("m", evalcore.EvaluatorRef{Name: "builtin.task_completion"}), + &project.Options{EvaluationLevel: "conversation"}) req, err = buildEvalRequest(conv, schemas, columns) require.NoError(t, err) mapping = req.TestingCriteria[0].DataMapping @@ -249,16 +257,16 @@ func TestBuildResolvesConversationTurnExclusivity(t *testing.T) { require.NotContains(t, mapping, "response") // An unset level behaves as turn, matching the service default. - dflt := groupWith([]evalcore.EvaluatorRef{{Name: "builtin.task_completion"}}, - &project.Options{EvalModel: "m"}) + dflt := groupWith(withJudge("m", evalcore.EvaluatorRef{Name: "builtin.task_completion"}), nil) req, err = buildEvalRequest(dflt, schemas, columns) require.NoError(t, err) require.NotContains(t, req.TestingCriteria[0].DataMapping, "messages") } // Evaluators disagree on what the judge model is called. Built-ins declare -// deployment_name; a custom rubric declares model, and rejects the group with -// "requires model" if only deployment_name is sent. +// deployment_name; a custom rubric declares model, and rejects the eval with +// "requires model" if only deployment_name is sent. One declaration binds +// whichever the evaluator actually accepts. func TestBuildBindsJudgeModelUnderTheDeclaredName(t *testing.T) { schemas := map[string]*eval_api.EvaluatorSummary{ "builtin.similarity": schema("builtin.similarity", @@ -268,10 +276,10 @@ func TestBuildBindsJudgeModelUnderTheDeclaredName(t *testing.T) { nil, []string{"query", "response"}, []string{"model"}, []string{"model"}, "turn"), } - group := groupWith([]evalcore.EvaluatorRef{ - {Name: "builtin.similarity"}, - {Name: "my-rubric"}, - }, &project.Options{EvalModel: "gpt-4.1-nano"}) + group := groupWith(withJudge("gpt-4.1-nano", + evalcore.EvaluatorRef{Name: "builtin.similarity"}, + evalcore.EvaluatorRef{Name: "my-rubric"}, + ), nil) req, err := buildEvalRequest(group, schemas, map[string]bool{"query": true}) require.NoError(t, err) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/description_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/description_test.go index 6d14b672856..a9fb31e194f 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/description_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/description_test.go @@ -8,7 +8,6 @@ import ( "azureaieval/internal/pkg/eval_api" "azureaieval/internal/pkg/evalcore" - "azureaieval/internal/project" "github.com/stretchr/testify/require" ) @@ -21,8 +20,7 @@ func TestBuildCarriesGroupDescriptionInMetadata(t *testing.T) { nil, []string{"query", "response"}, []string{"deployment_name"}, []string{"deployment_name"}, "turn"), } - group := groupWith([]evalcore.EvaluatorRef{{Name: "builtin.similarity"}}, - &project.Options{EvalModel: "m"}) + group := groupWith(withJudge("m", evalcore.EvaluatorRef{Name: "builtin.similarity"}), nil) group.Description = "Quality gate for the support agent" req, err := buildEvalRequest(group, schemas, map[string]bool{"query": true}) @@ -37,8 +35,7 @@ func TestBuildOmitsEmptyDescription(t *testing.T) { nil, []string{"query", "response"}, []string{"deployment_name"}, []string{"deployment_name"}, "turn"), } - group := groupWith([]evalcore.EvaluatorRef{{Name: "builtin.similarity"}}, - &project.Options{EvalModel: "m"}) + group := groupWith(withJudge("m", evalcore.EvaluatorRef{Name: "builtin.similarity"}), nil) req, err := buildEvalRequest(group, schemas, map[string]bool{"query": true}) require.NoError(t, err) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/envkeys_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/envkeys_test.go index e7c404211c7..b1df7509305 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/envkeys_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/envkeys_test.go @@ -4,9 +4,12 @@ package cmd import ( + "os" + "path/filepath" "testing" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) // Ids are per declaration. A shared key works only while a config has one @@ -36,18 +39,28 @@ func TestIDKey_DoesNotCollideWithVersionKey(t *testing.T) { assert.NotEqual(t, idKey("dataset", "golden"), versionKey("dataset", "golden")) } -// Setting EVAL_ID by hand is the documented way to point a config at a -// group that already exists. It is also the key the extension writes itself, -// which is what let a second group adopt the first one's id — so it stays -// readable only where it cannot be ambiguous. Fixing the aliasing dropped this -// fallback entirely once, silently breaking the documented behaviour. +// Setting EVAL_ID by hand is the documented way to point a config at an eval +// that already exists. It is also the key the extension writes itself, which is +// what let a second eval adopt the first one's id — so it stays readable only +// where it cannot be ambiguous. Fixing the aliasing dropped this fallback +// entirely once, silently breaking the documented behaviour. func TestGroupIDKeys_SharedKeyReadOnlyWhenUnambiguous(t *testing.T) { - sole := evalIDKeys("quality", true) + write := func(t *testing.T, names ...string) string { + t.Helper() + dir := t.TempDir() + for _, n := range names { + require.NoError(t, os.WriteFile(filepath.Join(dir, n+".yaml"), []byte("{}\n"), 0o600)) + } + return dir + } + + sole := evalIDKeys("quality", write(t, "quality")) assert.Equal(t, idKey("eval", "quality"), sole[0], - "a group's own entry is preferred over the shared one") + "an eval's own entry is preferred over the shared one") assert.Contains(t, sole, envKeyEvalID, - "a single-group config honours an id set by hand") + "a project with one eval honours an id set by hand") - assert.Equal(t, []string{idKey("eval", "quality")}, evalIDKeys("quality", false), - "with several groups the shared entry cannot say which group it means") + assert.Equal(t, []string{idKey("eval", "quality")}, + evalIDKeys("quality", write(t, "quality", "nightly")), + "with several evals the shared entry cannot say which one it means") } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go index 86cd9c7727c..823ed4e251b 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go @@ -11,7 +11,6 @@ import ( "io" "os" "path/filepath" - "strconv" "strings" "time" @@ -27,36 +26,35 @@ var generatePollBudget = eval_api.PollerOptions{ MaxAttempts: 720, // one hour } -// warnIgnoredTraceFields reports generation settings that are accepted but have -// no effect yet. -// -// The generation API takes a day window and nothing else, so `source` and -// `sample` are parsed and dropped. Silently discarding them is worse than not -// accepting them: the author believes they narrowed the trace selection when -// nothing changed. -func warnIgnoredTraceFields(cfg *project.GenerateConfig, out io.Writer) { - var fields []string - - if traces := cfg.Agent.Context.Traces; traces != nil { - if traces.Source != "" { - fields = append(fields, "agent.context.traces.source") - } - if traces.Sample > 0 { - fields = append(fields, "agent.context.traces.sample") - } - } - if len(fields) == 0 { - return - } +// generationPlan is everything one generation job needs, after the flags, the +// generation spec, and the eval's own target have been reconciled. +type generationPlan struct { + // Name of the artifact being generated — the positional argument. + Name string + // Agent whose context seeds generation. May be empty, in which case + // generation runs from the instruction alone. + Agent string + // Model deployment the generation job runs against. + Model string + // Instruction describing what the agent does and what to test. + Instruction string + // BaseDir is the directory OutputDir resolves against. + BaseDir string + // OutputDir is where the artifact is written. + OutputDir string + // SampleSize applies to dataset generation only. + SampleSize int + // TraceDays seeds generation from that many days of recent traces. + TraceDays int +} - verb := "has" - if len(fields) > 1 { - verb = "have" +// traceOptions converts the plan's trace window into the generation client's +// day count. Traces seed generation only; they are never a run's data source. +func (p generationPlan) traceOptions() *eval_api.TraceOptions { + if p.TraceDays <= 0 { + return nil } - fmt.Fprintf(out, - "warning: %s %s no effect yet; generation is seeded from the agent's "+ - "instructions and, when a window is set, its traces.\n", - strings.Join(fields, " and "), verb) + return &eval_api.TraceOptions{Days: p.TraceDays} } // resolveInstruction returns the generation instruction, reading it from a @@ -82,23 +80,19 @@ func resolveInstruction(inline, path string) (string, error) { // generationModel returns the deployment both generation jobs run against. // -// Dataset generation has no model of its own: the spec carries one judge model -// and both jobs use it. +// Dataset generation has no model of its own: the spec carries one generation +// model and both jobs use it. func generationModel(cfg *project.GenerateConfig) string { - if cfg.Generate.Rubric == nil { - return "" - } - return cfg.Generate.Rubric.Model + return cfg.GenerationModel } -// agentContextInstructions reads the instructions named by -// `agent.context.instructions`, relative to the spec that declared them. +// declaredInstructions reads the file named by a generation entry's +// `instructions`, relative to the spec that declared it. // -// A missing file is not an error. `init` writes the field pointing at a -// conventional path before that file exists, so treating its absence as a -// failure would break the flow it scaffolds. -func agentContextInstructions(cfg *project.GenerateConfig, configPath string) (string, error) { - named := cfg.Agent.Context.Instructions +// A missing file is not an error. The path can be written before the file +// exists, so treating its absence as a failure would break the flow `init` +// scaffolds. +func declaredInstructions(named, configPath string) (string, error) { if named == "" { return "", nil } @@ -112,7 +106,7 @@ func agentContextInstructions(cfg *project.GenerateConfig, configPath string) (s return "", nil } if err != nil { - return "", fmt.Errorf("reading agent.context.instructions %q: %w", named, err) + return "", fmt.Errorf("reading instructions %q: %w", named, err) } return strings.TrimSpace(string(raw)), nil } @@ -128,8 +122,7 @@ func agentContextInstructions(cfg *project.GenerateConfig, configPath string) (s // which is the flow `init` sets up. func (ec *evalContext) resolveGenerationInstruction( ctx context.Context, - cfg *project.GenerateConfig, - explicit, configPath string, + explicit, declared, configPath, agentName string, out io.Writer, quiet bool, ) (string, error) { @@ -137,7 +130,7 @@ func (ec *evalContext) resolveGenerationInstruction( return explicit, nil } - fromFile, err := agentContextInstructions(cfg, configPath) + fromFile, err := declaredInstructions(declared, configPath) if err != nil { return "", err } @@ -145,98 +138,39 @@ func (ec *evalContext) resolveGenerationInstruction( return fromFile, nil } - if cfg.Agent.Name == "" { + if agentName == "" { return "", nil } - agent, err := ec.evalClient.GetAgent(ctx, cfg.Agent.Name, ProjectEndpointAPIVersion) + agent, err := ec.evalClient.GetAgent(ctx, agentName, ProjectEndpointAPIVersion) if err != nil { // Generation can still proceed from the agent source alone, so a // failure to read the agent is reported without stopping. if !quiet { fmt.Fprintf(out, " warning: could not read agent %q for generation context: %v\n", - cfg.Agent.Name, err) + agentName, err) } return "", nil } instructions := agent.Instructions() if instructions != "" && !quiet { - fmt.Fprintf(out, " Seeding generation from the instructions of agent %q.\n", cfg.Agent.Name) + fmt.Fprintf(out, " Seeding generation from the instructions of agent %q.\n", agentName) } return instructions, nil } -// resolveGenerateConfig loads the spec when present, then layers flags on top. -// A missing file is not an error: flags alone are sufficient. -func resolveGenerateConfig( - path, target, evalModel, datasetFlag string, - maxSamples, traceDays int, -) (*project.GenerateConfig, error) { - cfg := &project.GenerateConfig{} - - if _, err := os.Stat(path); err == nil { - loaded, err := project.LoadGenerateConfig(path) - if err != nil { - return nil, err - } - cfg = loaded - } - - if target != "" { - cfg.Agent.Name = target - } - if cfg.Agent.Name == "" { - return nil, requireFlag("target") - } - - if cfg.Generate.Rubric == nil { - cfg.Generate.Rubric = &project.RubricSpec{ - Name: cfg.Agent.Name + "-quality", - LocalDir: "./" + project.DefaultEvaluatorsDir, - } - } - if cfg.Generate.Dataset == nil && datasetFlag == "" { - cfg.Generate.Dataset = &project.DatasetSpec{ - Name: cfg.Agent.Name + "-golden", - Strategy: project.StrategySynthetic, - SampleSize: project.DefaultSampleSize, - LocalDir: "./" + project.DefaultDatasetsDir, - } - } - - if evalModel != "" { - cfg.Generate.Rubric.Model = evalModel - } - if maxSamples > 0 && cfg.Generate.Dataset != nil { - cfg.Generate.Dataset.SampleSize = maxSamples - } - if cfg.Generate.Dataset != nil && cfg.Generate.Dataset.SampleSize == 0 { - cfg.Generate.Dataset.SampleSize = project.DefaultSampleSize - } - if traceDays > 0 { - if cfg.Agent.Context.Traces == nil { - cfg.Agent.Context.Traces = &project.TraceSpec{} - } - cfg.Agent.Context.Traces.Window = fmt.Sprintf("%dd", traceDays) - } - - return cfg, nil -} - // generateRubric submits the evaluator generation job and saves the rubric. func (ec *evalContext) generateRubric( ctx context.Context, - cfg *project.GenerateConfig, - instruction, baseDir string, + plan generationPlan, out io.Writer, noWait bool, ) (*project.ArtifactRef, error) { - spec := cfg.Generate.Rubric - fmt.Fprintf(out, "Generating rubric %s...\n", spec.Name) + fmt.Fprintf(out, "Generating rubric %s...\n", plan.Name) sources := eval_api.BuildGenerationSources( - "agent", cfg.Agent.Name, "", instruction, traceOptions(cfg), + "agent", plan.Agent, "", plan.Instruction, plan.traceOptions(), ) - req := eval_api.NewEvaluatorGenerationJobRequest(spec.Name, spec.Model, sources) + req := eval_api.NewEvaluatorGenerationJobRequest(plan.Name, plan.Model, sources) job, err := ec.evalClient.CreateEvaluatorGenerationJob(ctx, req, ProjectEndpointAPIVersion) if err != nil { @@ -253,31 +187,28 @@ func (ec *evalContext) generateRubric( return nil, fmt.Errorf("rubric generation: %w", err) } - path := project.ArtifactPath(baseDir, spec.LocalDir, spec.Name, ".json") + path := project.ArtifactPath(plan.BaseDir, plan.OutputDir, plan.Name, ".json") if err := writeRubric(path, completed.Result); err != nil { return nil, err } fmt.Fprintf(out, " wrote %s\n", path) - return &project.ArtifactRef{Name: spec.Name, Source: relativeSource(baseDir, path)}, nil + return &project.ArtifactRef{Name: plan.Name, Source: relativeSource(plan.BaseDir, path)}, nil } // generateDataset submits the data generation job and downloads the result. func (ec *evalContext) generateDataset( ctx context.Context, - cfg *project.GenerateConfig, - instruction, baseDir string, + plan generationPlan, out io.Writer, noWait bool, ) (*project.ArtifactRef, error) { - spec := cfg.Generate.Dataset - fmt.Fprintf(out, "Generating dataset %s (%d samples)...\n", spec.Name, spec.SampleSize) + fmt.Fprintf(out, "Generating dataset %s (%d samples)...\n", plan.Name, plan.SampleSize) sources := eval_api.BuildGenerationSources( - "agent", cfg.Agent.Name, "", instruction, traceOptions(cfg), + "agent", plan.Agent, "", plan.Instruction, plan.traceOptions(), ) - model := generationModel(cfg) - req := eval_api.NewDataGenerationJobRequest(spec.Name, model, spec.SampleSize, sources) + req := eval_api.NewDataGenerationJobRequest(plan.Name, plan.Model, plan.SampleSize, sources) job, err := ec.evalClient.CreateDataGenerationJob(ctx, req, DataGenerationAPIVersion) if err != nil { @@ -299,10 +230,10 @@ func (ec *evalContext) generateDataset( if eval_api.HasPromptSource(promptOnly) { fmt.Fprintf(out, " warning: generating from agent %q failed in the service; "+ - "retrying from the instruction alone.\n", cfg.Agent.Name) + "retrying from the instruction alone.\n", plan.Agent) req = eval_api.NewDataGenerationJobRequest( - spec.Name, model, spec.SampleSize, promptOnly) + plan.Name, plan.Model, plan.SampleSize, promptOnly) job, err = ec.evalClient.CreateDataGenerationJob(ctx, req, DataGenerationAPIVersion) if err != nil { return nil, fmt.Errorf("submitting the data generation job: %w", err) @@ -312,7 +243,7 @@ func (ec *evalContext) generateDataset( } } if err != nil { - return nil, fmt.Errorf("data generation: %w", explainDataGenerationFailure(err, cfg.Agent.Name)) + return nil, fmt.Errorf("data generation: %w", explainDataGenerationFailure(err, plan.Agent)) } name, version := completed.ResolvedNameVersion() @@ -332,7 +263,7 @@ func (ec *evalContext) generateDataset( return nil, fmt.Errorf("downloading the generated dataset %q: %w", name, err) } - path := project.ArtifactPath(baseDir, spec.LocalDir, spec.Name, ".jsonl") + path := project.ArtifactPath(plan.BaseDir, plan.OutputDir, plan.Name, ".jsonl") if err := os.MkdirAll(filepath.Dir(path), 0o750); err != nil { return nil, fmt.Errorf("creating %q: %w", filepath.Dir(path), err) } @@ -341,7 +272,7 @@ func (ec *evalContext) generateDataset( } fmt.Fprintf(out, " wrote %s\n", path) - return &project.ArtifactRef{Name: spec.Name, Source: relativeSource(baseDir, path)}, nil + return &project.ArtifactRef{Name: plan.Name, Source: relativeSource(plan.BaseDir, path)}, nil } // isAgentSeededGenerationFailure recognises the service-side failure that hits @@ -393,35 +324,6 @@ func (ec *evalContext) pollGeneration( return poller.Poll(ctx) } -// traceOptions converts the config's trace window into the generation client's -// day count. Traces seed rubric generation only; they are never a run's data -// source. -func traceOptions(cfg *project.GenerateConfig) *eval_api.TraceOptions { - t := cfg.Agent.Context.Traces - if t == nil { - return nil - } - days := parseWindowDays(t.Window) - if days <= 0 { - return nil - } - return &eval_api.TraceOptions{Days: days} -} - -// parseWindowDays reads a window such as "30d" or a bare day count. -func parseWindowDays(window string) int { - w := strings.TrimSpace(strings.ToLower(window)) - if w == "" { - return 0 - } - w = strings.TrimSuffix(w, "d") - days, err := strconv.Atoi(w) - if err != nil { - return 0 - } - return days -} - // writeRubric persists only the rubric dimensions so the developer can edit // weights and descriptions and publish a new version. func writeRubric(path string, result json.RawMessage) error { diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_commands.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_commands.go index b3c7b86c374..6a22d4fcbed 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_commands.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_commands.go @@ -5,6 +5,7 @@ package cmd import ( "fmt" + "path/filepath" "azureaieval/internal/project" @@ -22,6 +23,7 @@ import ( // generateFlags are the settings both generate commands share. type generateFlags struct { configPath string + evalName string target string instruction string instructionFile string @@ -34,6 +36,7 @@ type generateFlags struct { func addGenerateFlags(cmd *cobra.Command, f *generateFlags) { cmd.Flags().StringVar(&f.configPath, "config", project.DefaultGenerateConfig, "Path to the generation spec. Optional; flags alone are sufficient.") + addEvalFlag(cmd, &f.evalName) cmd.Flags().StringVar(&f.target, "target", "", "Agent whose context seeds generation.") cmd.Flags().StringVar(&f.instruction, "agent-instruction", "", "What the agent does and what to test.") @@ -42,8 +45,8 @@ func addGenerateFlags(cmd *cobra.Command, f *generateFlags) { cmd.MarkFlagsMutuallyExclusive("agent-instruction", "agent-instruction-file") cmd.Flags().StringVar(&f.model, "generation-model", "", "Model deployment that generates the artifact.") - cmd.Flags().StringVar(&f.outputDir, "output-dir", project.DefaultEvalDir, - "Directory the generated artifact is written under.") + cmd.Flags().StringVar(&f.outputDir, "output-dir", "", + "Directory the generated artifact is written to. Overrides the generation spec.") cmd.Flags().BoolVar(&f.noWait, "no-wait", false, "Submit the job and return its id without polling.") cmd.Flags().StringVar(&f.endpoint, "project-endpoint", "", "Foundry project endpoint.") @@ -57,45 +60,130 @@ func addGenerateFlags(cmd *cobra.Command, f *generateFlags) { func prepareGeneration( cmd *cobra.Command, f *generateFlags, - maxSamples, traceDays int, -) (*evalContext, *project.GenerateConfig, string, error) { + plan generationPlan, + declared genEntry, +) (*evalContext, generationPlan, error) { instruction, err := resolveInstruction(f.instruction, f.instructionFile) if err != nil { - return nil, nil, "", err + return nil, plan, err } - cfg, err := resolveGenerateConfig( - f.configPath, f.target, f.model, "", maxSamples, traceDays, - ) + ctx := cmd.Context() + ec, err := newEvalContext(ctx, f.endpoint) if err != nil { - return nil, nil, "", err + return nil, plan, err } - if err := cfg.Validate(); err != nil { - return nil, nil, "", err + + plan.Instruction, err = ec.resolveGenerationInstruction( + ctx, instruction, declared.instructions, f.configPath, plan.Agent, + cmd.OutOrStdout(), isJSON(cmd), + ) + if err != nil { + ec.Close() + return nil, plan, err } - if !isJSON(cmd) { - warnIgnoredTraceFields(cfg, cmd.OutOrStdout()) + return ec, plan, nil +} + +// resolvePlan settles every input that does not need the network. +// +// Resolution order is the one the spec fixes for every input: flags, then the +// generation spec, then what can be detected from the eval configuration. +// Doing it before the client is built means a missing model or an out-of-range +// sample count is refused without an authentication round trip. +func resolvePlan( + f *generateFlags, + cfg *project.GenerateConfig, + name string, + declared genEntry, +) (generationPlan, error) { + plan := generationPlan{ + Name: name, + Agent: firstNonEmpty(f.target, declared.deriveFrom, evalTarget(f)), + Model: firstNonEmpty(f.model, cfg.GenerationModel), + BaseDir: filepath.Dir(f.configPath), + OutputDir: firstNonEmpty(f.outputDir, declared.outputDir), + SampleSize: declared.sampleSize, + TraceDays: declared.traceDays, } - if generationModel(cfg) == "" { - return nil, nil, "", fmt.Errorf( + if plan.Model == "" { + return plan, fmt.Errorf( "a model deployment is required to generate: pass --generation-model, " + - "or set it in the generation spec") + "or set `generationModel` in the generation spec") } + return plan, nil +} - ctx := cmd.Context() - ec, err := newEvalContext(ctx, f.endpoint) +// genEntry is the subset of a generation spec entry both commands share, so +// resolvePlan does not need to know which one it is serving. +type genEntry struct { + outputDir string + deriveFrom string + instructions string + sampleSize int + traceDays int +} + +// evalTarget reads the agent from the eval configuration, which is where the +// target is already declared, so `generate` does not need it repeated. +// +// Best effort: generation runs from the instruction alone when there is no +// eval config to read, which is the case in a bare directory. +func evalTarget(f *generateFlags) string { + path, err := project.ResolveEvalConfigPath(filepath.Dir(f.configPath), f.evalName) if err != nil { - return nil, nil, "", err + return "" + } + cfg, err := project.LoadEvalConfig(path) + if err != nil || cfg.Target == nil { + return "" } + return cfg.Target.Name +} - instruction, err = ec.resolveGenerationInstruction( - ctx, cfg, instruction, f.configPath, cmd.OutOrStdout(), isJSON(cmd), - ) - if err != nil { - ec.Close() - return nil, nil, "", err +func firstNonEmpty(values ...string) string { + for _, v := range values { + if v != "" { + return v + } } - return ec, cfg, instruction, nil + return "" +} + +// datasetGenEntry reads one dataset's settings out of the generation spec, +// applying the flag override and the default row count. +func datasetGenEntry(cfg *project.GenerateConfig, name string, maxSamples int) genEntry { + spec, _ := cfg.DatasetSpec(name) + entry := genEntry{ + outputDir: firstNonEmpty(spec.OutputDir, "./"+project.DefaultDatasetsDir), + deriveFrom: spec.DeriveFrom, + instructions: spec.Instructions, + sampleSize: spec.SampleSize, + traceDays: spec.TraceDays, + } + if maxSamples > 0 { + entry.sampleSize = maxSamples + } + if entry.sampleSize == 0 { + entry.sampleSize = project.DefaultSampleSize + } + return entry +} + +// evaluatorGenEntry reads one evaluator's settings out of the generation spec, +// applying the flag override. +func evaluatorGenEntry(cfg *project.GenerateConfig, name string, traceDays int) genEntry { + spec, _ := cfg.EvaluatorSpec(name) + entry := genEntry{ + outputDir: firstNonEmpty(spec.OutputDir, "./"+project.DefaultEvaluatorsDir), + deriveFrom: spec.DeriveFrom, + instructions: spec.Instructions, + traceDays: spec.TraceDays, + } + if traceDays > 0 { + entry.traceDays = traceDays + } + return entry } func newDatasetGenerateCommand() *cobra.Command { @@ -111,19 +199,31 @@ func newDatasetGenerateCommand() *cobra.Command { RunE: func(cmd *cobra.Command, args []string) error { name := args[0] - ec, cfg, instruction, err := prepareGeneration(cmd, &flags, maxSamples, 0) + if err := project.ValidateSampleSize(maxSamples); err != nil { + return err + } + + cfg, err := project.LoadGenerateConfig(flags.configPath) if err != nil { return err } - defer ec.Close() + declared := datasetGenEntry(cfg, name, maxSamples) + + plan, err := resolvePlan(&flags, cfg, name, declared) + if err != nil { + return err + } + if err := project.ValidateSampleSize(plan.SampleSize); err != nil { + return err + } - if cfg.Generate.Dataset == nil { - return fmt.Errorf("the generation spec declares no dataset to generate") + ec, plan, err := prepareGeneration(cmd, &flags, plan, declared) + if err != nil { + return err } - cfg.Generate.Dataset.Name = name + defer ec.Close() - ref, err := ec.generateDataset( - cmd.Context(), cfg, instruction, flags.outputDir, cmd.OutOrStdout(), flags.noWait) + ref, err := ec.generateDataset(cmd.Context(), plan, cmd.OutOrStdout(), flags.noWait) if err != nil { return err } @@ -150,19 +250,24 @@ func newEvaluatorGenerateCommand() *cobra.Command { RunE: func(cmd *cobra.Command, args []string) error { name := args[0] - ec, cfg, instruction, err := prepareGeneration(cmd, &flags, 0, traceDays) + cfg, err := project.LoadGenerateConfig(flags.configPath) if err != nil { return err } - defer ec.Close() + declared := evaluatorGenEntry(cfg, name, traceDays) - if cfg.Generate.Rubric == nil { - return fmt.Errorf("the generation spec declares no rubric to generate") + plan, err := resolvePlan(&flags, cfg, name, declared) + if err != nil { + return err + } + + ec, plan, err := prepareGeneration(cmd, &flags, plan, declared) + if err != nil { + return err } - cfg.Generate.Rubric.Name = name + defer ec.Close() - ref, err := ec.generateRubric( - cmd.Context(), cfg, instruction, flags.outputDir, cmd.OutOrStdout(), flags.noWait) + ref, err := ec.generateRubric(cmd.Context(), plan, cmd.OutOrStdout(), flags.noWait) if err != nil { return err } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_defaults_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_defaults_test.go deleted file mode 100644 index 97f141a2f53..00000000000 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_defaults_test.go +++ /dev/null @@ -1,47 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package cmd - -import ( - "path/filepath" - "testing" - - "github.com/stretchr/testify/require" -) - -// --dataset is documented as taking a path or the name of a registered -// dataset, and means "use this one instead of generating". Only a local path -// used to suppress generation, so passing a registered name still submitted a -// generation job. -func TestGenerateScaffoldSkipsDatasetWhenSupplied(t *testing.T) { - cases := []struct { - name string - datasetFlag string - wantSpec bool - }{ - {"registered name", "prod-sample", false}, - {"relative path", "./data/golden.jsonl", false}, - {"bare filename", "golden.jsonl", false}, - {"not supplied", "", true}, - } - - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - // A path that does not exist means flags alone drive the config. - cfg, err := resolveGenerateConfig( - filepath.Join(t.TempDir(), "absent.yaml"), - "my-agent", "gpt-4.1-nano", tc.datasetFlag, 0, 0) - require.NoError(t, err) - - if tc.wantSpec { - require.NotNil(t, cfg.Generate.Dataset, - "a dataset spec is needed when none was supplied") - } else { - require.Nil(t, cfg.Generate.Dataset, - "a supplied dataset must not produce a generation spec") - } - require.NotNil(t, cfg.Generate.Rubric, "the rubric spec is independent") - }) - } -} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_plan_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_plan_test.go index 584a19e7547..c018ddda8ac 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_plan_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_plan_test.go @@ -6,7 +6,6 @@ package cmd import ( "os" "path/filepath" - "strings" "testing" "azureaieval/internal/project" @@ -15,133 +14,157 @@ import ( ) // `generate` decides what to submit before it touches the network, so the plan -// it builds — which artifacts, from what instruction, at what sample size — is -// checkable without paying for a generation job. These are the parts that -// cannot be observed afterwards: once the jobs are submitted, a wrong default -// is indistinguishable from an intended one. +// it builds — which agent, what model, where the artifact lands, at what sample +// size — is checkable without paying for a generation job. These are the parts +// that cannot be observed afterwards: once the job is submitted, a wrong +// default is indistinguishable from an intended one. -func writeConfig(t *testing.T, body string) string { +// evalsDir writes a generation spec and returns the flags pointing at it. +func evalsDir(t *testing.T, generateBody string, files map[string]string) *generateFlags { t.Helper() - path := filepath.Join(t.TempDir(), "eval_generate.yaml") - require.NoError(t, os.WriteFile(path, []byte(body), 0o600)) - return path + dir := t.TempDir() + configPath := filepath.Join(dir, "generate.yaml") + if generateBody != "" { + require.NoError(t, os.WriteFile(configPath, []byte(generateBody), 0o600)) + } + for name, body := range files { + require.NoError(t, os.WriteFile(filepath.Join(dir, name), []byte(body), 0o600)) + } + return &generateFlags{configPath: configPath} +} + +func loadSpec(t *testing.T, f *generateFlags) *project.GenerateConfig { + t.Helper() + cfg, err := project.LoadGenerateConfig(f.configPath) + require.NoError(t, err) + return cfg } -// A spec is optional, so the defaults are what most callers actually run with. -func TestResolveGenerateConfigDefaultsFromFlagsAlone(t *testing.T) { - absent := filepath.Join(t.TempDir(), "absent.yaml") +// A spec is optional, so flags alone are what most callers actually run with. +func TestResolvePlan_FromFlagsAlone(t *testing.T) { + f := evalsDir(t, "", nil) + f.target = "shop-agent" + f.model = "gpt-4o-mini" - cfg, err := resolveGenerateConfig(absent, "shop-agent", "gpt-4o-mini", "", 0, 0) + plan, err := resolvePlan(f, loadSpec(t, f), "shop-golden", + datasetGenEntry(loadSpec(t, f), "shop-golden", 0)) require.NoError(t, err) - require.NoError(t, cfg.Validate()) - - require.Equal(t, "shop-agent", cfg.Agent.Name) - require.NotNil(t, cfg.Generate.Rubric) - require.Equal(t, "shop-agent-quality", cfg.Generate.Rubric.Name) - require.Equal(t, "gpt-4o-mini", cfg.Generate.Rubric.Model) - require.Equal(t, "./"+project.DefaultEvaluatorsDir, cfg.Generate.Rubric.LocalDir) - - require.NotNil(t, cfg.Generate.Dataset) - require.Equal(t, "shop-agent-golden", cfg.Generate.Dataset.Name) - require.Equal(t, project.StrategySynthetic, cfg.Generate.Dataset.Strategy) - require.Equal(t, project.DefaultSampleSize, cfg.Generate.Dataset.SampleSize) - require.Equal(t, "./"+project.DefaultDatasetsDir, cfg.Generate.Dataset.LocalDir) + + require.Equal(t, "shop-golden", plan.Name) + require.Equal(t, "shop-agent", plan.Agent) + require.Equal(t, "gpt-4o-mini", plan.Model) + require.Equal(t, "./"+project.DefaultDatasetsDir, plan.OutputDir) + require.Equal(t, project.DefaultSampleSize, plan.SampleSize) } -// Without a target there is nothing to generate from, and the refusal has to -// name the flag rather than a config field the caller may not have. -func TestResolveGenerateConfigRequiresATarget(t *testing.T) { - _, err := resolveGenerateConfig( - filepath.Join(t.TempDir(), "absent.yaml"), "", "gpt-4o-mini", "", 0, 0) +// Without a model there is nothing to bill the job against, and the refusal has +// to name both ways of supplying one. +func TestResolvePlan_RequiresAGenerationModel(t *testing.T) { + f := evalsDir(t, "", nil) + f.target = "shop-agent" + + _, err := resolvePlan(f, loadSpec(t, f), "d", genEntry{}) require.Error(t, err) - require.Contains(t, err.Error(), "--target") + require.Contains(t, err.Error(), "--generation-model") + require.Contains(t, err.Error(), "generationModel") } -func TestResolveGenerateConfigReadsTheSpec(t *testing.T) { - path := writeConfig(t, ` -agent: - name: from-spec - context: - instructions: ./instructions.md - traces: - window: 7d - source: ignored-today - sample: 5 -generate: - rubric: - name: spec-rubric - model: gpt-4o - local_dir: ./custom-evaluators - dataset: - name: spec-dataset - strategy: synthetic +// The spec is read per artifact name, so generating one artifact never picks up +// the other's settings. +func TestResolvePlan_ReadsTheNamedSpecEntry(t *testing.T) { + f := evalsDir(t, ` +generationModel: gpt-4o +dataset: + spec-dataset: sampleSize: 200 - local_dir: ./custom-datasets -`) + outputDir: ./custom-datasets + deriveFrom: from-spec +evaluator: + spec-rubric: + outputDir: ./custom-evaluators + deriveFrom: rubric-agent +`, nil) + + cfg := loadSpec(t, f) + + ds, err := resolvePlan(f, cfg, "spec-dataset", datasetGenEntry(cfg, "spec-dataset", 0)) + require.NoError(t, err) + require.Equal(t, "gpt-4o", ds.Model) + require.Equal(t, 200, ds.SampleSize) + require.Equal(t, "./custom-datasets", ds.OutputDir) + require.Equal(t, "from-spec", ds.Agent) - cfg, err := resolveGenerateConfig(path, "", "", "", 0, 0) + ev, err := resolvePlan(f, cfg, "spec-rubric", evaluatorGenEntry(cfg, "spec-rubric", 0)) require.NoError(t, err) - require.NoError(t, cfg.Validate()) - - require.Equal(t, "from-spec", cfg.Agent.Name) - require.Equal(t, "./instructions.md", cfg.Agent.Context.Instructions) - require.Equal(t, "spec-rubric", cfg.Generate.Rubric.Name) - require.Equal(t, "gpt-4o", cfg.Generate.Rubric.Model) - require.Equal(t, "./custom-evaluators", cfg.Generate.Rubric.LocalDir) - require.Equal(t, "spec-dataset", cfg.Generate.Dataset.Name) - require.Equal(t, 200, cfg.Generate.Dataset.SampleSize) - - require.NotNil(t, cfg.Agent.Context.Traces) - require.Equal(t, "7d", cfg.Agent.Context.Traces.Window) - require.Equal(t, 5, cfg.Agent.Context.Traces.Sample) + require.Equal(t, "./custom-evaluators", ev.OutputDir) + require.Equal(t, "rubric-agent", ev.Agent) + + // An artifact the spec says nothing about still generates, on the defaults. + other, err := resolvePlan(f, cfg, "unlisted", datasetGenEntry(cfg, "unlisted", 0)) + require.NoError(t, err) + require.Equal(t, project.DefaultSampleSize, other.SampleSize) + require.Equal(t, "./"+project.DefaultDatasetsDir, other.OutputDir) } // Flags win over the spec, which is what makes a one-off run possible without // editing a file that is checked in. -func TestResolveGenerateConfigLayersFlagsOverTheSpec(t *testing.T) { - path := writeConfig(t, ` -agent: - name: from-spec -generate: - rubric: - name: spec-rubric - model: gpt-4o - dataset: - name: spec-dataset +func TestResolvePlan_LayersFlagsOverTheSpec(t *testing.T) { + f := evalsDir(t, ` +generationModel: gpt-4o +dataset: + spec-dataset: sampleSize: 200 -`) + outputDir: ./custom-datasets + deriveFrom: from-spec +`, nil) + f.target = "from-flag" + f.model = "gpt-4o-mini" + f.outputDir = "./from-flag-dir" + + cfg := loadSpec(t, f) + plan, err := resolvePlan(f, cfg, "spec-dataset", datasetGenEntry(cfg, "spec-dataset", 500)) + require.NoError(t, err) + + require.Equal(t, "from-flag", plan.Agent) + require.Equal(t, "gpt-4o-mini", plan.Model) + require.Equal(t, "./from-flag-dir", plan.OutputDir) + require.Equal(t, 500, plan.SampleSize) +} + +// The target is already declared on the eval, so `generate` does not need it +// repeated on every invocation. +func TestResolvePlan_FallsBackToTheEvalTarget(t *testing.T) { + f := evalsDir(t, "generationModel: gpt-4o\n", map[string]string{ + "support-agent-smoke.yaml": "evaluators: [builtin.relevance]\n" + + "target:\n type: agent\n name: support-agent\n", + }) - cfg, err := resolveGenerateConfig(path, "from-flag", "gpt-4o-mini", "", 500, 14) + cfg := loadSpec(t, f) + plan, err := resolvePlan(f, cfg, "support-agent-smoke", + datasetGenEntry(cfg, "support-agent-smoke", 0)) require.NoError(t, err) - require.NoError(t, cfg.Validate()) - - require.Equal(t, "from-flag", cfg.Agent.Name) - require.Equal(t, "gpt-4o-mini", cfg.Generate.Rubric.Model) - require.Equal(t, 500, cfg.Generate.Dataset.SampleSize) - require.Equal(t, "14d", cfg.Agent.Context.Traces.Window, - "--trace-days must reach the spec as a window, since that is the only "+ - "trace field the generation API takes") - - // The rubric name is not derived when the spec named one, so a --target - // override must not silently rename an artifact the spec author declared. - require.Equal(t, "spec-rubric", cfg.Generate.Rubric.Name) + require.Equal(t, "support-agent", plan.Agent, + "the eval's declared target is the agent to generate from") } -// A spec that declares a dataset without a size still has to submit a legal -// job, so the default is applied rather than left at zero. -func TestResolveGenerateConfigFillsAMissingSampleSize(t *testing.T) { - path := writeConfig(t, ` -agent: - name: sized -generate: - dataset: - name: no-size -`) - - cfg, err := resolveGenerateConfig(path, "", "", "", 0, 0) +// With more than one eval the target is ambiguous, so nothing is guessed: +// generation falls back to the instruction alone rather than picking one. +func TestResolvePlan_AmbiguousEvalTargetIsNotGuessed(t *testing.T) { + f := evalsDir(t, "generationModel: gpt-4o\n", map[string]string{ + "a.yaml": "target:\n type: agent\n name: agent-a\n", + "b.yaml": "target:\n type: agent\n name: agent-b\n", + }) + + cfg := loadSpec(t, f) + plan, err := resolvePlan(f, cfg, "d", datasetGenEntry(cfg, "d", 0)) + require.NoError(t, err) + require.Empty(t, plan.Agent) + + // Naming one resolves it. + f.evalName = "b" + plan, err = resolvePlan(f, cfg, "d", datasetGenEntry(cfg, "d", 0)) require.NoError(t, err) - require.Equal(t, project.DefaultSampleSize, cfg.Generate.Dataset.SampleSize) - require.NoError(t, cfg.Validate()) + require.Equal(t, "agent-b", plan.Agent) } // The bounds are the service's, and the boundaries themselves have to be @@ -158,43 +181,24 @@ func TestGenerateSampleSizeBounds(t *testing.T) { {project.MaxSampleSize, true}, {project.MaxSampleSize + 1, false}, } { - cfg, err := resolveGenerateConfig( - filepath.Join(t.TempDir(), "absent.yaml"), - "bounded", "gpt-4o-mini", "", tc.size, 0) - require.NoError(t, err) - require.Equal(t, tc.size, cfg.Generate.Dataset.SampleSize) - - err = cfg.Validate() + err := project.ValidateSampleSize(tc.size) if tc.allowed { require.NoErrorf(t, err, "%d is inside the service's range", tc.size) continue } require.Errorf(t, err, "%d is outside the service's range", tc.size) - require.Contains(t, err.Error(), "sampleSize") + require.Contains(t, err.Error(), "must be between") } } -// --dataset and --evaluator both mean "use this one". Only the dataset side is -// resolved here; the evaluator side is decided in the command body, so it is -// covered by the CLI test that watches for the skip message. -func TestResolveGenerateConfigSkipsTheDatasetWhenOneIsSupplied(t *testing.T) { - cfg, err := resolveGenerateConfig( - filepath.Join(t.TempDir(), "absent.yaml"), - "supplied", "gpt-4o-mini", "prod-sample", 0, 0) - require.NoError(t, err) - require.Nil(t, cfg.Generate.Dataset, "a supplied dataset must not be generated") - require.NotNil(t, cfg.Generate.Rubric) - require.NoError(t, cfg.Validate()) -} - -// Both jobs bill against one deployment, so a spec with no rubric has no model -// to run either of them. -func TestGenerationModelComesFromTheRubricSpec(t *testing.T) { - require.Equal(t, "", generationModel(&project.GenerateConfig{})) - - cfg := &project.GenerateConfig{} - cfg.Generate.Rubric = &project.RubricSpec{Name: "r", Model: "gpt-4o"} - require.Equal(t, "gpt-4o", generationModel(cfg)) +// Trace days come from the spec, and the flag overrides them. +func TestEvaluatorGenEntry_TraceDays(t *testing.T) { + cfg := &project.GenerateConfig{ + Evaluator: map[string]project.EvaluatorGenSpec{"r": {TraceDays: 7}}, + } + require.Equal(t, 7, evaluatorGenEntry(cfg, "r", 0).traceDays) + require.Equal(t, 30, evaluatorGenEntry(cfg, "r", 30).traceDays) + require.Zero(t, evaluatorGenEntry(cfg, "absent", 0).traceDays) } func TestResolveInstruction(t *testing.T) { @@ -230,24 +234,3 @@ func TestResolveInstruction(t *testing.T) { require.Contains(t, err.Error(), "--agent-instruction-file") }) } - -// The generation API takes a day window and nothing else, so the two fields it -// drops are reported rather than silently discarded. -func TestWarnIgnoredTraceFields(t *testing.T) { - cfg := &project.GenerateConfig{} - cfg.Agent.Context.Traces = &project.TraceSpec{Window: "7d", Source: "some-source", Sample: 5} - - var out strings.Builder - warnIgnoredTraceFields(cfg, &out) - require.Contains(t, out.String(), "agent.context.traces.source") - require.Contains(t, out.String(), "agent.context.traces.sample") - require.NotContains(t, out.String(), "agent.context.traces.window", - "the window is the one trace field the API takes, so it is not a no-op") - - // A window on its own is fully supported and must not produce a warning. - quiet := &project.GenerateConfig{} - quiet.Agent.Context.Traces = &project.TraceSpec{Window: "7d"} - out.Reset() - warnIgnoredTraceFields(quiet, &out) - require.Empty(t, out.String()) -} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go index fba0d533682..f9cbbbddd86 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go @@ -23,11 +23,12 @@ import ( // all, so it works offline and unauthenticated. func newInitCommand() *cobra.Command { var ( + evalName string target string dataset string evaluators []string - evalModel string - outDir string + genModel string + outputDir string force bool ) @@ -40,14 +41,17 @@ func newInitCommand() *cobra.Command { if target == "" { return requireFlag("target") } - if outDir == "" { - outDir = project.DefaultEvalDir + if outputDir == "" { + outputDir = project.DefaultEvalDir + } + if evalName == "" { + evalName = target + "-smoke" } - genPath := filepath.Join(outDir, "eval_generate.yaml") - depPath := filepath.Join(outDir, "azure.yaml") + evalPath := project.EvalConfigPath(outputDir, evalName) + genPath := filepath.Join(outputDir, "generate.yaml") - for _, p := range []string{genPath, depPath} { + for _, p := range []string{evalPath, genPath} { if _, err := os.Stat(p); err == nil && !force { return fmt.Errorf("%s already exists; pass --force to overwrite", p) } @@ -56,26 +60,29 @@ func newInitCommand() *cobra.Command { // Asked before anything is written: the project is the one thing init // cannot supply for itself, and failing after creating directories // leaves a half-scaffolded tree behind for the user to clean up. - if err := confirmAzdProject(cmd.Context()); err != nil { + azdProject, err := readAzdProject(cmd.Context()) + if err != nil { return err } + if genModel == "" { + genModel = detectModelDeployment(azdProject) + } - if err := os.MkdirAll(filepath.Join(outDir, project.DefaultDatasetsDir), 0o750); err != nil { + if err := os.MkdirAll(filepath.Join(outputDir, project.DefaultDatasetsDir), 0o750); err != nil { return fmt.Errorf("creating the datasets directory: %w", err) } - if err := os.MkdirAll(filepath.Join(outDir, project.DefaultEvaluatorsDir), 0o750); err != nil { + if err := os.MkdirAll(filepath.Join(outputDir, project.DefaultEvaluatorsDir), 0o750); err != nil { return fmt.Errorf("creating the evaluators directory: %w", err) } - rubricName := fmt.Sprintf("%s-quality", target) + rubricName := target + "-quality" + + plan := planScaffold(evalName, target, rubricName, dataset, evaluators, genModel, outputDir) - genCfg := buildGenerateScaffold(target, rubricName, evalModel) - if err := writeYAML(genPath, genCfg); err != nil { + if err := writeYAML(evalPath, plan.eval); err != nil { return err } - - depCfg := buildDeployScaffold(target, rubricName, dataset, evaluators, evalModel, outDir) - if err := writeYAML(depPath, depCfg); err != nil { + if err := writeYAML(genPath, plan.generate); err != nil { return err } @@ -85,44 +92,59 @@ func newInitCommand() *cobra.Command { // Printing the block and leaving the edit to the reader was enough // to make the documented flow stop working between `init` and // `azd up`. - rootWiring, err := ensureRootEvalService(cmd.Context(), depPath) + rootWiring, err := ensureRootEvalService(cmd.Context(), evalName, target, evalPath) if err != nil { return err } if isJSON(cmd) { return emitJSON(out, map[string]any{ - "generateConfig": genPath, - "deployConfig": depPath, - "datasetsDir": filepath.Join(outDir, project.DefaultDatasetsDir), - "evaluatorsDir": filepath.Join(outDir, project.DefaultEvaluatorsDir), - "rootConfig": rootWiring, + "eval": evalName, + "evalConfig": evalPath, + "generateConfig": genPath, + "datasetsDir": filepath.Join(outputDir, project.DefaultDatasetsDir), + "evaluatorsDir": filepath.Join(outputDir, project.DefaultEvaluatorsDir), + "rootConfig": rootWiring, + "target": target, + "generationModel": genModel, + "evaluators": plan.evaluatorNames(), }) } - fmt.Fprintf(out, "Wrote %s\n", genPath) - fmt.Fprintf(out, "Wrote %s\n", depPath) + fmt.Fprintf(out, "%s Detected agent target: %s\n", doneMark, target) + if genModel != "" { + fmt.Fprintf(out, "%s Detected model deployment: %s\n", doneMark, genModel) + } + fmt.Fprintf(out, "%s Planned evaluators: %s\n", doneMark, plan.evaluatorSummary()) + + fmt.Fprintln(out, "\nCreated") + fmt.Fprintf(out, " %-33s eval definition\n", filepath.ToSlash(evalPath)) + fmt.Fprintf(out, " %-33s generation settings (%d samples, %d rubric)\n", + filepath.ToSlash(genPath), project.DefaultSampleSize, plan.rubricCount()) switch rootWiring { case wiringAdded: - fmt.Fprintf(out, "Added the evals service to %s\n", rootConfigName) + fmt.Fprintf(out, " %-33s added service '%s'\n", rootConfigName, evalName) case wiringPresent: - fmt.Fprintf(out, "%s already declares an eval service\n", rootConfigName) + fmt.Fprintf(out, " %-33s already declares service '%s'\n", rootConfigName, evalName) } - fmt.Fprintln(out, "\nNext:") - fmt.Fprintln(out, " 1. azd ai eval generate (or supply your own dataset)") - fmt.Fprintln(out, " 2. azd up") - fmt.Fprintln(out, " 3. azd ai eval run") + fmt.Fprintf(out, "\nNext: azd ai eval dataset generate %s\n", plan.datasetName) + if plan.rubricCount() > 0 { + fmt.Fprintf(out, " azd ai eval evaluator generate %s\n", rubricName) + } return nil }, } + cmd.Flags().StringVar(&evalName, "name", "", "Name of the eval. Defaults to -smoke.") cmd.Flags().StringVar(&target, "target", "", "Name of the agent to evaluate.") cmd.Flags().StringVar(&dataset, "dataset", "", "Path to a local .jsonl, or the name of a registered dataset.") cmd.Flags().StringArrayVar(&evaluators, "evaluator", nil, - "Evaluator reference, repeatable. Use builtin. for a built-in.") - cmd.Flags().StringVar(&evalModel, "judge-model", "", "Model deployment that scores the results.") - cmd.Flags().StringVar(&outDir, "out-dir", project.DefaultEvalDir, + "Evaluator reference, repeatable. Use builtin. for a built-in. "+ + "Passing this replaces the defaults, so it also opts out of rubric generation.") + cmd.Flags().StringVar(&genModel, "generation-model", "", + "Model deployment that generates and judges. Detected from the project when omitted.") + cmd.Flags().StringVar(&outputDir, "output-dir", project.DefaultEvalDir, "Directory to write the config into. Used verbatim, never re-rooted.") cmd.Flags().BoolVar(&force, "force", false, "Overwrite existing files.") return cmd @@ -145,19 +167,51 @@ const ( const noAzdProject = "no azd project found in this directory. Run `azd init` first, " + "or run this from the root of an existing one; the eval service is " -// confirmAzdProject reports whether a project exists, without changing it. -func confirmAzdProject(ctx context.Context) error { +// readAzdProject returns the project, without changing it. +// +// It is read before anything is written: the project is the one thing init +// cannot supply for itself, and it also carries the agent and model detection +// that `init` reports. +func readAzdProject(ctx context.Context) (*azdext.ProjectConfig, error) { azdClient, err := azdext.NewAzdClient() if err != nil { - return fmt.Errorf("%sadded to its azure.yaml", noAzdProject) + return nil, fmt.Errorf("%sadded to its azure.yaml", noAzdProject) } defer azdClient.Close() resp, err := azdClient.Project().Get(ctx, &azdext.EmptyRequest{}) if err != nil || resp.GetProject() == nil { - return fmt.Errorf("%sadded to its azure.yaml", noAzdProject) + return nil, fmt.Errorf("%sadded to its azure.yaml", noAzdProject) } - return nil + return resp.GetProject(), nil +} + +// aiModelHost is the model-deployment service the sibling Foundry extensions +// declare, which is where a judge deployment can be read without a service +// call. +const aiModelHost = "azure.ai.model" + +// detectModelDeployment finds the deployment generation and judging run +// against, from what the project already declares. +// +// `init` makes no service calls, so detection is limited to the project file. +// Coming back empty is not a failure: --generation-model supplies it, and the +// generate commands say so when it is missing. +func detectModelDeployment(proj *azdext.ProjectConfig) string { + for name, svc := range proj.GetServices() { + if svc.GetHost() != aiModelHost { + continue + } + if props := svc.GetAdditionalProperties().AsMap(); props != nil { + for _, key := range []string{"deployment", "deploymentName", "name", "model"} { + if v, ok := props[key].(string); ok && v != "" { + return v + } + } + } + return name + } + return "" } // ensureRootEvalService declares the eval service in azd's project file. @@ -167,10 +221,11 @@ func confirmAzdProject(ctx context.Context) error { // the agents extension uses, so azd owns the edit and the project file keeps // whatever shape azd gives it. // -// The eval config itself stays in evals/azure.yaml and is referenced with -// `$ref`. azd carries unknown keys through AdditionalProperties untouched, -// which is how the extension gets it back at deploy time. -func ensureRootEvalService(ctx context.Context, depPath string) (string, error) { +// The service key is the eval's name — one `azure.ai.eval` service per eval — +// and the eval body stays in evals/.yaml, referenced with `$ref`. +// azd carries unknown keys through AdditionalProperties untouched, which is how +// the extension gets it back at deploy time. +func ensureRootEvalService(ctx context.Context, evalName, target, evalPath string) (string, error) { azdClient, err := azdext.NewAzdClient() if err != nil { return "", fmt.Errorf("connecting to azd: %w", err) @@ -187,16 +242,15 @@ func ensureRootEvalService(ctx context.Context, depPath string) (string, error) "added to its %s", rootConfigName) } - // A service already pointing at an eval config is left alone, whatever it - // is called: a second one would deploy the same evals twice. - for _, svc := range resp.GetProject().GetServices() { - if svc.GetHost() == project.EvalHost { - return wiringPresent, nil - } + // A service already declaring this eval is left alone: re-adding it would + // deploy the same eval twice. A differently-named eval service is not a + // conflict, because one service is one eval. + if svc, ok := resp.GetProject().GetServices()[evalName]; ok && svc.GetHost() == project.EvalHost { + return wiringPresent, nil } props, err := structpb.NewStruct(map[string]any{ - "$ref": "./" + filepath.ToSlash(depPath), + "$ref": "./" + filepath.ToSlash(evalPath), }) if err != nil { return "", fmt.Errorf("building the eval service entry: %w", err) @@ -204,9 +258,9 @@ func ensureRootEvalService(ctx context.Context, depPath string) (string, error) _, err = azdClient.Project().AddService(ctx, &azdext.AddServiceRequest{ Service: &azdext.ServiceConfig{ - Name: evalServiceName(resp.GetProject()), + Name: evalName, Host: project.EvalHost, - Uses: projectServiceUses(resp.GetProject()), + Uses: evalServiceUses(resp.GetProject(), target), AdditionalProperties: props, }, }) @@ -216,74 +270,76 @@ func ensureRootEvalService(ctx context.Context, depPath string) (string, error) return wiringAdded, nil } -// projectServiceUses points the eval service at the Foundry project service -// when the repo declares one, so azd provisions it first. +// evalServiceUses orders the eval after the things it reads. // // It is conditional for the same reason the agents extension makes it // conditional: naming a service the project does not declare is a broken // reference, and an eval config can perfectly well sit in a repo that reaches -// an existing Foundry project by endpoint instead. -func projectServiceUses(proj *azdext.ProjectConfig) []string { +// an existing Foundry project by endpoint and an agent that is deployed +// elsewhere. +func evalServiceUses(proj *azdext.ProjectConfig, target string) []string { + var uses []string for name, svc := range proj.GetServices() { if svc.GetHost() == aiProjectHost { - return []string{name} + uses = append(uses, name) + break } } - return nil + if _, ok := proj.GetServices()[target]; ok { + uses = append(uses, target) + } + return uses +} + +// scaffold is what `init` writes: one eval body and the generation settings +// that fill in the artifacts it references. +type scaffold struct { + eval *project.EvalConfig + generate *project.GenerateConfig + datasetName string + rubricName string +} + +// evaluatorNames lists the evaluators the eval will run, in declaration order. +func (s scaffold) evaluatorNames() []string { + names := make([]string, 0, len(s.eval.Evaluators)) + for _, ref := range s.eval.Evaluators { + names = append(names, ref.Name) + } + return names } -// evalServiceName avoids colliding with a service the project already has. -// azd keys services by name, so the map key is the name to avoid. -func evalServiceName(proj *azdext.ProjectConfig) string { - taken := proj.GetServices() - if _, exists := taken["evals"]; !exists { - return "evals" - } - for i := 2; ; i++ { - candidate := fmt.Sprintf("evals%d", i) - if _, exists := taken[candidate]; !exists { - return candidate +// evaluatorSummary is the one-line form `init` reports, marking the evaluator +// that still has to be generated. +func (s scaffold) evaluatorSummary() string { + parts := make([]string, 0, len(s.eval.Evaluators)) + for _, ref := range s.eval.Evaluators { + if ref.Name == s.rubricName && s.rubricCount() > 0 { + parts = append(parts, ref.Name+" (rubric)") + continue } + parts = append(parts, ref.Name) } + return strings.Join(parts, ", ") } -func buildGenerateScaffold(target, rubricName, evalModel string) *project.GenerateConfig { - return &project.GenerateConfig{ - Agent: project.AgentSpec{ - Name: target, - Context: project.AgentContext{ - // Scaffolded even though the file does not exist yet: writing - // it overrides the agent's published instructions, which is the - // usual way to narrow what gets generated. `tools` is left out - // because nothing reads it yet. - Instructions: "./agent/instructions.md", - }, - }, - Generate: project.GenerateSpec{ - Rubric: &project.RubricSpec{ - Name: rubricName, - Model: evalModel, - LocalDir: "./" + project.DefaultEvaluatorsDir, - }, - Dataset: &project.DatasetSpec{ - Name: fmt.Sprintf("%s-golden", target), - Strategy: project.StrategySynthetic, - SampleSize: project.DefaultSampleSize, - LocalDir: "./" + project.DefaultDatasetsDir, - }, - }, +// rubricCount is the number of evaluators `init` expects to be generated. +func (s scaffold) rubricCount() int { + if s.generate == nil { + return 0 } + return len(s.generate.Evaluator) } // relativeToConfig rewrites a path given relative to the working directory so -// it resolves from the directory holding the deploy spec. +// it resolves from the directory holding the eval config. // -// `--dataset ./tests/golden.jsonl` means "relative to where I am", but the -// deploy spec's `source:` is resolved relative to that file, so writing the -// path through unchanged sends the deploy looking inside evals/. An absolute -// path is left alone, and forward slashes are kept so the config reads the same -// on every platform. -func relativeToConfig(path, outDir string) string { +// `--dataset ./tests/golden.jsonl` means "relative to where I am", but +// `source:` is resolved relative to the config file, so writing the path +// through unchanged sends the deploy looking inside evals/. An absolute path is +// left alone, and forward slashes are kept so the config reads the same on +// every platform. +func relativeToConfig(path, outputDir string) string { if filepath.IsAbs(path) { return path } @@ -292,7 +348,7 @@ func relativeToConfig(path, outDir string) string { if err != nil { return path } - absOut, err := filepath.Abs(outDir) + absOut, err := filepath.Abs(outputDir) if err != nil { return path } @@ -309,73 +365,120 @@ func relativeToConfig(path, outDir string) string { return rel } -func buildDeployScaffold( - target, rubricName, dataset string, +// planScaffold builds both files `init` writes. +// +// The default evaluator set is a built-in plus a generated rubric: the built-in +// alone would be generic, and the rubric is what makes the baseline about this +// agent. Passing --evaluator replaces both, which is how a caller opts out of +// rubric generation. +func planScaffold( + evalName, target, rubricName, dataset string, evaluators []string, - evalModel string, - outDir string, -) *project.EvalConfig { - cfg := &project.EvalConfig{} + genModel string, + outputDir string, +) scaffold { + cfg := &project.EvalConfig{ + Description: fmt.Sprintf("Basic quality evaluation for %s", target), + } - datasetName := fmt.Sprintf("%s-golden", target) + datasetName := evalName datasetSource := "" + generateDataset := true if dataset != "" { if looksLikeLocalDataset(dataset) { // --dataset is given relative to where the user is standing, but - // source: is resolved relative to the deploy spec, so the path has - // to be rebased or the deploy looks for it inside evals/. - datasetSource = relativeToConfig(dataset, outDir) + // source: is resolved relative to the config, so the path has to be + // rebased or the deploy looks for it inside evals/. + datasetSource = relativeToConfig(dataset, outputDir) datasetName = strings.TrimSuffix(filepath.Base(dataset), filepath.Ext(dataset)) } else { // A bare name references an already-registered dataset. datasetName = dataset } + generateDataset = false } else { datasetSource = fmt.Sprintf("./%s/%s.jsonl", project.DefaultDatasetsDir, datasetName) } - cfg.Datasets = append(cfg.Datasets, project.DatasetDecl{ + cfg.Dataset = &project.DatasetDecl{ Name: datasetName, Source: datasetSource, - }) + } + + // Every evaluator carries the judge deployment, because that is where the + // service reads it from: built-ins declare `deployment_name` as required, + // so an eval that leaves it off is rejected before it runs. + initParams := map[string]any{} + if genModel != "" { + initParams["deployment_name"] = genModel + } + withModel := func(ref evalcore.EvaluatorRef) evalcore.EvaluatorRef { + if len(initParams) == 0 { + return ref + } + params := make(map[string]any, len(initParams)) + for k, v := range initParams { + params[k] = v + } + ref.InitializationParameters = params + return ref + } - // Evaluators supplied on the command line win; otherwise scaffold the - // generated rubric so `generate` has somewhere to write its reference. refs := evalcore.EvaluatorList{} + generateRubric := false if len(evaluators) == 0 { - cfg.Evaluators = append(cfg.Evaluators, project.EvaluatorDecl{ - Name: rubricName, - Source: fmt.Sprintf("./%s/%s.json", project.DefaultEvaluatorsDir, rubricName), - }) - refs = append(refs, evalcore.EvaluatorRef{Name: rubricName}) + refs = append(refs, + withModel(evalcore.EvaluatorRef{Name: evalcore.BuiltinPrefix + "task_adherence"}), + withModel(evalcore.EvaluatorRef{ + Name: rubricName, + Source: fmt.Sprintf("./%s/%s.json", project.DefaultEvaluatorsDir, rubricName), + }), + ) + generateRubric = true } else { for _, e := range evaluators { ref := evalcore.EvaluatorRef{Name: e} if !ref.IsBuiltin() { - cfg.Evaluators = append(cfg.Evaluators, project.EvaluatorDecl{ - Name: e, - Source: fmt.Sprintf("./%s/%s.json", project.DefaultEvaluatorsDir, e), - }) + ref.Source = fmt.Sprintf("./%s/%s.json", project.DefaultEvaluatorsDir, e) } - refs = append(refs, ref) + refs = append(refs, withModel(ref)) } } + cfg.Evaluators = refs - group := project.Eval{ - Name: fmt.Sprintf("%s-quality", target), - Description: fmt.Sprintf("Quality gate for %s", target), - Dataset: datasetName, - Evaluators: refs, - Target: &project.Target{ - Type: project.TargetTypeAgent, - Name: target, - }, + cfg.Target = &project.Target{ + Type: project.TargetTypeAgent, + Name: target, } - if evalModel != "" { - group.Options = &project.Options{EvalModel: evalModel} + cfg.Options = &project.Options{ + MaxSamples: project.DefaultSampleSize, + EvaluationLevel: project.EvaluationLevelTurn, } - cfg.Evals = append(cfg.Evals, group) - return cfg + gen := &project.GenerateConfig{GenerationModel: genModel} + if generateDataset { + gen.Dataset = map[string]project.DatasetGenSpec{ + datasetName: { + SampleSize: project.DefaultSampleSize, + OutputDir: "./" + project.DefaultDatasetsDir, + DeriveFrom: target, + }, + } + } + if generateRubric { + gen.Evaluator = map[string]project.EvaluatorGenSpec{ + rubricName: { + OutputDir: "./" + project.DefaultEvaluatorsDir, + DeriveFrom: target, + }, + } + } + + return scaffold{ + eval: cfg, + generate: gen, + datasetName: datasetName, + rubricName: rubricName, + } } // looksLikeLocalDataset distinguishes a path from a registered dataset name. diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_test.go index a4e22c85f87..40c92cf0855 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_test.go @@ -17,54 +17,93 @@ import ( // config the tool itself produced. func TestScaffold_RoundTripsAndValidates(t *testing.T) { dir := t.TempDir() - depPath := filepath.Join(dir, "azure.yaml") + evalPath := filepath.Join(dir, "support-agent-smoke.yaml") - cfg := buildDeployScaffold("support-agent", "support-agent-quality", "", nil, "gpt-4.1-nano", project.DefaultEvalDir) - require.NoError(t, writeYAML(depPath, cfg)) + plan := planScaffold("support-agent-smoke", "support-agent", "support-agent-quality", + "", nil, "gpt-4.1-nano", project.DefaultEvalDir) + require.NoError(t, writeYAML(evalPath, plan.eval)) - loaded, err := project.LoadEvalConfig(depPath) + loaded, err := project.LoadEvalConfig(evalPath) require.NoError(t, err) require.NoError(t, loaded.Validate(), "the generated scaffold must be valid") - g, err := loaded.ResolveGroup("") - require.NoError(t, err) - require.Equal(t, project.TargetTypeAgent, g.Target.Type) - require.Equal(t, "support-agent", g.Target.Name) - require.Equal(t, "gpt-4.1-nano", g.Options.EvalModel) - require.Len(t, g.Evaluators, 1) + require.Equal(t, project.TargetTypeAgent, loaded.Target.Type) + require.Equal(t, "support-agent", loaded.Target.Name) + require.Equal(t, project.DefaultSampleSize, loaded.Options.MaxSamples) + + // The eval takes its name from the file, which is the azure.yaml service key. + require.Equal(t, "support-agent-smoke", loaded.Eval("support-agent-smoke").Name) +} + +// The default set is a built-in plus a generated rubric: the built-in alone +// would be generic, and the rubric is what makes the baseline about this agent. +func TestScaffold_DefaultEvaluators(t *testing.T) { + plan := planScaffold("support-agent-smoke", "support-agent", "support-agent-quality", + "", nil, "gpt-5.6-luna", project.DefaultEvalDir) + + require.Equal(t, + []string{"builtin.task_adherence", "support-agent-quality"}, + plan.evaluatorNames()) + require.Contains(t, plan.evaluatorSummary(), "support-agent-quality (rubric)") + + // Every evaluator carries the judge deployment, because built-ins declare + // deployment_name as required and an eval that leaves it off is rejected. + for _, ref := range plan.eval.Evaluators { + require.Equal(t, "gpt-5.6-luna", ref.InitializationParameters["deployment_name"], + "%s must name a judge deployment", ref.Name) + } +} + +// Passing --evaluator replaces the defaults, which is how a caller opts out of +// rubric generation. +func TestScaffold_ExplicitEvaluatorsOptOutOfGeneration(t *testing.T) { + plan := planScaffold("smoke", "support-agent", "support-agent-quality", "", + []string{"builtin.task_adherence"}, "m", project.DefaultEvalDir) + + require.Equal(t, []string{"builtin.task_adherence"}, plan.evaluatorNames()) + require.Zero(t, plan.rubricCount(), "no rubric is generated when evaluators are given") + require.Empty(t, plan.generate.Evaluator) } func TestGenerateScaffold_RoundTripsAndValidates(t *testing.T) { dir := t.TempDir() - genPath := filepath.Join(dir, "eval_generate.yaml") + genPath := filepath.Join(dir, "generate.yaml") - cfg := buildGenerateScaffold("support-agent", "support-agent-quality", "gpt-4.1-nano") - require.NoError(t, writeYAML(genPath, cfg)) + plan := planScaffold("support-agent-smoke", "support-agent", "support-agent-quality", + "", nil, "gpt-4.1-nano", project.DefaultEvalDir) + require.NoError(t, writeYAML(genPath, plan.generate)) loaded, err := project.LoadGenerateConfig(genPath) require.NoError(t, err) - require.NoError(t, loaded.Validate()) - require.Equal(t, "support-agent", loaded.Agent.Name) - require.Equal(t, project.StrategySynthetic, loaded.Generate.Dataset.Strategy) - require.Equal(t, project.DefaultSampleSize, loaded.Generate.Dataset.SampleSize) + require.Equal(t, "gpt-4.1-nano", loaded.GenerationModel) + + ds, ok := loaded.DatasetSpec("support-agent-smoke") + require.True(t, ok, "the generation spec is keyed by artifact name") + require.Equal(t, project.DefaultSampleSize, ds.SampleSize) + require.Equal(t, "support-agent", ds.DeriveFrom) + + ev, ok := loaded.EvaluatorSpec("support-agent-quality") + require.True(t, ok) + require.Equal(t, "./"+project.DefaultEvaluatorsDir, ev.OutputDir) } -// Built-ins are referenced from the group but never declared as custom -// evaluators; declaring one is a validation error. -func TestScaffold_BuiltinEvaluatorsAreNotDeclared(t *testing.T) { - cfg := buildDeployScaffold( - "support-agent", "unused", "", - []string{"builtin.task_adherence", "my-custom"}, "", project.DefaultEvalDir, - ) +// Built-ins are referenced but never published, so the scaffold must not give +// one a local source to upload. +func TestScaffold_BuiltinEvaluatorsHaveNoSource(t *testing.T) { + plan := planScaffold("smoke", "support-agent", "unused", "", + []string{"builtin.task_adherence", "my-custom"}, "", project.DefaultEvalDir) + cfg := plan.eval - require.Len(t, cfg.Evaluators, 1, "only the custom evaluator should be declared") - require.Equal(t, "my-custom", cfg.Evaluators[0].Name) + require.Len(t, cfg.Evaluators, 2) + require.True(t, cfg.Evaluators[0].IsBuiltin()) + require.Empty(t, cfg.Evaluators[0].Source) + require.False(t, cfg.Evaluators[1].IsBuiltin()) + require.NotEmpty(t, cfg.Evaluators[1].Source) - require.Len(t, cfg.Evals[0].Evaluators, 2) - require.True(t, cfg.Evals[0].Evaluators[0].IsBuiltin()) - require.False(t, cfg.Evals[0].Evaluators[1].IsBuiltin()) + require.Len(t, cfg.CustomEvaluators(), 1, + "only the custom evaluator is this config's to publish") - path := filepath.Join(t.TempDir(), "azure.yaml") + path := filepath.Join(t.TempDir(), "smoke.yaml") require.NoError(t, writeYAML(path, cfg)) loaded, err := project.LoadEvalConfig(path) require.NoError(t, err) @@ -72,26 +111,35 @@ func TestScaffold_BuiltinEvaluatorsAreNotDeclared(t *testing.T) { } // A bare name means an already-registered dataset; a path means a local file. +// Either way the dataset was supplied, so nothing is scheduled to generate it — +// only a missing --dataset produces a generation entry. func TestScaffold_DatasetReferenceForms(t *testing.T) { t.Run("local path becomes a source", func(t *testing.T) { // --dataset is relative to the working directory, but source: is - // resolved relative to the deploy spec, so it has to be rebased. - cfg := buildDeployScaffold("a", "r", "./tests/golden.jsonl", nil, "", "evals") - require.Equal(t, "../tests/golden.jsonl", cfg.Datasets[0].Source, + // resolved relative to the eval config, so it has to be rebased. + plan := planScaffold("smoke", "a", "r", "./tests/golden.jsonl", nil, "", "evals") + require.Equal(t, "../tests/golden.jsonl", plan.eval.Dataset.Source, "a dataset outside the eval dir must be reached with ..") - require.Equal(t, "golden", cfg.Datasets[0].Name) + require.Equal(t, "golden", plan.eval.Dataset.Name) + require.Empty(t, plan.generate.Dataset, + "a supplied dataset must not be scheduled for generation") }) t.Run("bare name references a registered dataset", func(t *testing.T) { - cfg := buildDeployScaffold("a", "r", "prod-sample", nil, "", project.DefaultEvalDir) - require.Equal(t, "prod-sample", cfg.Datasets[0].Name) - require.Empty(t, cfg.Datasets[0].Source, + plan := planScaffold("smoke", "a", "r", "prod-sample", nil, "", project.DefaultEvalDir) + require.Equal(t, "prod-sample", plan.eval.Dataset.Name) + require.Empty(t, plan.eval.Dataset.Source, "a registered dataset must not get a local source") + require.Empty(t, plan.generate.Dataset) }) - t.Run("no dataset flag scaffolds a local path", func(t *testing.T) { - cfg := buildDeployScaffold("support-agent", "r", "", nil, "", project.DefaultEvalDir) - require.Contains(t, cfg.Datasets[0].Source, "support-agent-golden.jsonl") + t.Run("no dataset flag scaffolds a local path and a generation entry", func(t *testing.T) { + plan := planScaffold("support-agent-smoke", "support-agent", "r", "", + nil, "", project.DefaultEvalDir) + require.Equal(t, "support-agent-smoke", plan.eval.Dataset.Name, + "the dataset is named after the eval") + require.Contains(t, plan.eval.Dataset.Source, "support-agent-smoke.jsonl") + require.Contains(t, plan.generate.Dataset, "support-agent-smoke") }) } @@ -106,13 +154,13 @@ func TestLooksLikeLocalDataset(t *testing.T) { // in the agent-scoped command must not reappear. func TestWriteYAML_UsesPathVerbatim(t *testing.T) { dir := t.TempDir() - nested := filepath.Join(dir, "evals", "azure.yaml") + nested := filepath.Join(dir, "evals", "smoke.yaml") require.NoError(t, writeYAML(nested, &project.EvalConfig{})) _, err := os.Stat(nested) require.NoError(t, err, "the file must land exactly at the requested path") - doubled := filepath.Join(dir, "evals", "evals", "azure.yaml") + doubled := filepath.Join(dir, "evals", "evals", "smoke.yaml") _, err = os.Stat(doubled) require.Error(t, err, "the path must not be re-rooted under itself") } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_wiring_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_wiring_test.go index 18b37e8e969..03745d01e75 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_wiring_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_wiring_test.go @@ -8,6 +8,8 @@ import ( "github.com/azure/azure-dev/cli/azd/pkg/azdext" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/types/known/structpb" ) func projectWith(names ...string) *azdext.ProjectConfig { @@ -18,31 +20,51 @@ func projectWith(names ...string) *azdext.ProjectConfig { return proj } -// The eval service is added through azd's own AddService, so it has to pick a -// name azd will accept. Reusing one already in the project would overwrite -// somebody else's service. -func TestEvalServiceName_PrefersEvals(t *testing.T) { - assert.Equal(t, "evals", evalServiceName(projectWith())) - assert.Equal(t, "evals", evalServiceName(projectWith("api", "web"))) -} +// The eval service is ordered after everything it reads, but only names +// services the project actually declares. Naming one it does not have is a +// broken reference, and an eval config can sit in a repo that reaches an +// existing Foundry project by endpoint and an agent deployed elsewhere. +func TestEvalServiceUses_OnlyWhatTheProjectDeclares(t *testing.T) { + assert.Nil(t, evalServiceUses(projectWith("api", "web"), "support-agent"), + "neither the project service nor the agent is declared, so there is nothing to order after") + + withProject := projectWith("api", "support-agent") + withProject.Services["ai-project"] = &azdext.ServiceConfig{ + Name: "ai-project", Host: aiProjectHost, + } + assert.Equal(t, []string{"ai-project", "support-agent"}, + evalServiceUses(withProject, "support-agent"), + "the eval runs after the project it evaluates against and the agent it evaluates") -func TestEvalServiceName_StepsAsideForAnExistingName(t *testing.T) { - assert.Equal(t, "evals2", evalServiceName(projectWith("evals"))) - assert.Equal(t, "evals3", evalServiceName(projectWith("evals", "evals2"))) + assert.Equal(t, []string{"support-agent"}, + evalServiceUses(projectWith("support-agent"), "support-agent"), + "an agent alone is still worth ordering after") } -// The agents extension wires uses: only to services the project actually -// declares. Naming one it does not have is a broken reference, and an eval -// config can sit in a repo that reaches an existing Foundry project by -// endpoint instead of declaring one. -func TestProjectServiceUses_OnlyWhenTheProjectDeclaresOne(t *testing.T) { - assert.Nil(t, projectServiceUses(projectWith("api", "web")), - "no Foundry project service means no uses entry") +// `init` detects the judge deployment from the project, because it makes no +// service calls and this is the only place it can read one. +func TestDetectModelDeployment(t *testing.T) { + assert.Empty(t, detectModelDeployment(projectWith("api", "web"))) - withProject := projectWith("api") - withProject.Services["ai-project"] = &azdext.ServiceConfig{ - Name: "ai-project", Host: aiProjectHost, + proj := projectWith("api") + proj.Services["chat"] = &azdext.ServiceConfig{Name: "chat", Host: aiModelHost} + assert.Equal(t, "chat", detectModelDeployment(proj), + "the service name is the deployment name when nothing more specific is declared") + + named := projectWith() + named.Services["chat"] = &azdext.ServiceConfig{ + Name: "chat", + Host: aiModelHost, + AdditionalProperties: mustStruct(t, map[string]any{ + "deployment": "gpt-5.6-luna", + }), } - assert.Equal(t, []string{"ai-project"}, projectServiceUses(withProject), - "the eval service should be ordered after the project it evaluates against") + assert.Equal(t, "gpt-5.6-luna", detectModelDeployment(named)) +} + +func mustStruct(t *testing.T, m map[string]any) *structpb.Struct { + t.Helper() + s, err := structpb.NewStruct(m) + require.NoError(t, err) + return s } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/output.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/output.go index 319ed2565f0..62f9c2eade1 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/output.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/output.go @@ -15,6 +15,14 @@ import ( const outputJSON = "json" +// Progress markers from the azd style guide, so the extension's lines sit +// alongside core's without a second vocabulary. +const ( + doneMark = "(✓) Done:" // finished successfully + skippedMark = "(-) Skipped:" // intentionally not done, not a failure + failedMark = "(x) Failed:" // the step did not complete +) + // outputFormat reads the inherited -o/--output flag. func outputFormat(cmd *cobra.Command) string { if cmd == nil { diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go index 7c2ab84e458..af31fb489c1 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go @@ -53,7 +53,6 @@ func newRunCommand() *cobra.Command { // buildRunCommand builds `run start`. func buildRunCommand(use, short string) *cobra.Command { var ( - configPath string groupName string evalID string runName string @@ -86,18 +85,21 @@ func buildRunCommand(use, short string) *cobra.Command { // --eval-id bypasses the config entirely. var group *project.Eval + configPath := "" if evalID == "" { - cfg, err := project.LoadEvalConfig(configPath) + configPath, err = project.ResolveEvalConfigPath(project.DefaultEvalDir, groupName) if err != nil { return err } - if err := cfg.Validate(); err != nil { + cfg, err := project.LoadEvalConfig(configPath) + if err != nil { return err } - group, err = cfg.ResolveGroup(groupName) - if err != nil { + if err := cfg.Validate(); err != nil { return err } + resolved := cfg.Eval(evalNameFromPath(configPath)) + group = &resolved if err := ec.checkDatasetRegistered(ctx, cfg, group, configPath); err != nil { return err @@ -105,7 +107,7 @@ func buildRunCommand(use, short string) *cobra.Command { evalID, err = ec.resolveEvalIDFromConfig( ctx, group, configPath, resolveLevel(level, group), - len(cfg.Evals) == 1, out, isJSON(cmd)) + out, isJSON(cmd)) if err != nil { return err } @@ -190,10 +192,8 @@ func buildRunCommand(use, short string) *cobra.Command { }, } - cmd.Flags().StringVar(&configPath, "config", project.DefaultDeployConfig, - "Path to the eval deployment config.") cmd.Flags().StringVar(&groupName, "eval", "", - "Which evals entry to run. Defaults to the only one.") + "Name of the eval to run. Defaults to the only one declared.") cmd.Flags().StringVar(&evalID, "eval-id", "", "Run against an existing eval by id, ignoring the config.") cmd.Flags().StringVar(&runName, "name", "", "Name for this run. Defaults to the eval name plus a timestamp.") @@ -225,7 +225,6 @@ func (ec *evalContext) resolveEvalIDFromConfig( group *project.Eval, configPath string, level string, - soleGroup bool, out interface{ Write([]byte) (int, error) }, jsonMode bool, ) (string, error) { @@ -233,7 +232,7 @@ func (ec *evalContext) resolveEvalIDFromConfig( return group.ID, nil } - for _, key := range evalIDKeys(group.Name, soleGroup) { + for _, key := range evalIDKeys(group.Name, filepath.Dir(configPath)) { cached := ec.getEnvValue(ctx, key) if cached == "" { continue @@ -279,18 +278,18 @@ func (ec *evalContext) resolveEvalIDFromConfig( return created.ID, nil } -// evalIDKeys lists the env entries that may hold this group's id, most +// evalIDKeys lists the env entries that may hold this eval's id, most // specific first. // // The per-name entry is what the extension writes. EVAL_ID is also the -// documented way to point a config at a group that already exists, created in +// documented way to point a config at an eval that already exists, created in // the portal or by another tool, so it stays readable — but only when the -// config declares a single group. With more than one there is no way to tell -// which group a shared entry refers to, and reading it anyway is what let a -// second group adopt the first one's id. -func evalIDKeys(name string, soleGroup bool) []string { +// project declares a single eval. With more than one there is no way to tell +// which eval a shared entry refers to, and reading it anyway is what let a +// second eval adopt the first one's id. +func evalIDKeys(name, evalDir string) []string { keys := []string{idKey("eval", name)} - if soleGroup { + if names, err := project.EvalNamesIn(evalDir); err == nil && len(names) == 1 { keys = append(keys, envKeyEvalID) } return keys @@ -318,8 +317,8 @@ func (ec *evalContext) checkDatasetRegistered( return nil } - decl, ok := cfg.Dataset(group.Dataset) - if !ok { + decl := cfg.Dataset + if decl == nil { return nil } @@ -496,8 +495,8 @@ func localDatasetPath(configPath string, group *project.Eval) string { if err != nil { return "" } - decl, ok := cfg.Dataset(group.Dataset) - if !ok || decl.Source == "" { + decl := cfg.Dataset + if decl == nil || decl.Source == "" { return "" } if filepath.IsAbs(decl.Source) { @@ -506,6 +505,13 @@ func localDatasetPath(configPath string, group *project.Eval) string { return filepath.Join(filepath.Dir(configPath), decl.Source) } +// evalNameFromPath is the eval's name: one config file is one eval, and the +// file is named after it, matching the azure.yaml service key that $refs it. +func evalNameFromPath(configPath string) string { + base := filepath.Base(configPath) + return strings.TrimSuffix(base, filepath.Ext(base)) +} + // readJSONL reads newline-delimited JSON, optionally truncating to limit rows. func readJSONL(path string, limit int) ([]map[string]any, error) { f, err := os.Open(path) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output.go index 4a30bcd200d..0f7545fffd6 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output.go @@ -269,10 +269,14 @@ func resolveEvalID( envKeyEvalID) } -// addEvalFlag registers the flag that names a group from the config, so -// every command taking an eval-id can reach a group by the name its author -// used. -// addEvalFlags registers the two ways to say which group a command acts +// addEvalFlag registers the flag that names an eval from the config, for +// commands that never take a raw service id. +func addEvalFlag(cmd *cobra.Command, target *string) { + cmd.Flags().StringVar(target, "eval", "", + "Name of the eval declared in azure.yaml.") +} + +// addEvalFlags registers the two ways to say which eval a command acts // on: --eval names one from the config, --eval-id gives its service id. // // The id is also accepted as a positional argument. The flag exists because @@ -280,8 +284,7 @@ func resolveEvalID( // it there should not have to find out that the sibling commands take only a // positional. func addEvalFlags(cmd *cobra.Command, target *string) { - cmd.Flags().StringVar(target, "eval", "", - "Name an evals entry from the config instead of passing its id.") + addEvalFlag(cmd, target) cmd.Flags().String("eval-id", "", "Id of the eval. Same as passing the id as an argument.") } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/schemas_live_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/schemas_live_test.go index dd852e0a778..55329762df9 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/schemas_live_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/schemas_live_test.go @@ -82,11 +82,13 @@ func TestLiveConversationEvaluatorBindsMessages(t *testing.T) { } plan, err := planCriterion( - evalcore.EvaluatorRef{Name: name}, + evalcore.EvaluatorRef{ + Name: name, + InitializationParameters: map[string]any{"deployment_name": judge}, + }, schemas[name], nil, // no target: the dataset holds both sides of the exchange map[string]bool{conversationField: true}, - judge, "conversation", ) require.NoError(t, err) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/trace_warning_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/trace_warning_test.go deleted file mode 100644 index 6c952569aa4..00000000000 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/trace_warning_test.go +++ /dev/null @@ -1,46 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package cmd - -import ( - "bytes" - "testing" - - "azureaieval/internal/project" - - "github.com/stretchr/testify/require" -) - -func warnFor(t *testing.T, traces *project.TraceSpec) string { - t.Helper() - cfg := &project.GenerateConfig{} - cfg.Agent.Context.Traces = traces - - var buf bytes.Buffer - warnIgnoredTraceFields(cfg, &buf) - return buf.String() -} - -// source and sample are accepted by the config model but the generation API -// takes only a day window, so they are dropped. Dropping them silently lets an -// author believe they narrowed the trace selection when nothing changed. -func TestWarnsAboutTraceFieldsWithNoEffect(t *testing.T) { - out := warnFor(t, &project.TraceSpec{Source: "production", Window: "30d", Sample: 500}) - require.Contains(t, out, "agent.context.traces.source") - require.Contains(t, out, "agent.context.traces.sample") - require.Contains(t, out, "window") - require.Contains(t, out, "have no effect", "two fields take a plural verb") - - out = warnFor(t, &project.TraceSpec{Source: "production", Window: "30d"}) - require.Contains(t, out, "agent.context.traces.source") - require.NotContains(t, out, "sample") - require.Contains(t, out, "has no effect", "one field takes a singular verb") -} - -// The field that does work draws no warning, and neither does an absent block. -func TestNoWarningWhenOnlyWindowIsSet(t *testing.T) { - require.Empty(t, warnFor(t, &project.TraceSpec{Window: "30d"})) - require.Empty(t, warnFor(t, nil)) - require.Empty(t, warnFor(t, &project.TraceSpec{})) -} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/evaluator.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/evaluator.go index 88538768b37..df6aa791b7d 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/evaluator.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/evaluator.go @@ -93,7 +93,7 @@ func (el *EvaluatorList) UnmarshalYAML(value *yaml.Node) error { func (el EvaluatorList) MarshalYAML() (any, error) { out := make([]any, 0, len(el)) for _, ref := range el { - if ref.Threshold == nil && ref.Version == "" { + if ref.isBareName() { out = append(out, ref.Name) continue } @@ -141,18 +141,29 @@ func (el *EvaluatorList) UnmarshalJSON(data []byte) error { } // MarshalJSON mirrors MarshalYAML's compact form. +// +// Everything the reference carries has to survive the round trip, including +// the source and the initialization parameters: the eval fingerprint is taken +// over this encoding, so a field dropped here is a change the reconciler +// cannot see. func (el EvaluatorList) MarshalJSON() ([]byte, error) { + // Aliased so the element encoder does not recurse through this method. + type ref = EvaluatorRef + out := make([]any, 0, len(el)) - for _, ref := range el { - if ref.Threshold == nil && ref.Version == "" { - out = append(out, ref.Name) + for _, r := range el { + if r.isBareName() { + out = append(out, r.Name) continue } - out = append(out, struct { - Name string `json:"name"` - Version string `json:"version,omitempty"` - Threshold *float64 `json:"threshold,omitempty"` - }{ref.Name, ref.Version, ref.Threshold}) + out = append(out, ref(r)) } return json.Marshal(out) } + +// isBareName reports whether the reference carries nothing but its name, in +// which case both encoders emit the compact string form. +func (e EvaluatorRef) isBareName() bool { + return e.Threshold == nil && e.Version == "" && e.Source == "" && + len(e.InitializationParameters) == 0 +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go index 0c75f38fdf0..e2ff2e74772 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go @@ -8,6 +8,8 @@ package project import ( "fmt" "os" + "path/filepath" + "sort" "strings" "azureaieval/internal/pkg/evalcore" @@ -15,12 +17,18 @@ import ( "go.yaml.in/yaml/v3" ) -// EvalConfig is the deployment spec — the body of the azure.ai.eval service -// entry, normally kept in evals/azure.yaml and pulled in with $ref. +// EvalConfig is one eval — the body of a single `azure.ai.eval` service entry, +// kept in evals/.yaml and pulled in with $ref. +// +// The eval's name is the service key in azure.yaml and is not repeated here. +// One service per eval is what lets azd's own dependency graph order an eval +// after the agent it evaluates. type EvalConfig struct { - Evaluators []EvaluatorDecl `yaml:"evaluators,omitempty" json:"evaluators,omitempty"` - Datasets []DatasetDecl `yaml:"datasets,omitempty" json:"datasets,omitempty"` - Evals []Eval `yaml:"evals,omitempty" json:"evals,omitempty"` + Description string `yaml:"description,omitempty" json:"description,omitempty"` + Dataset *DatasetDecl `yaml:"dataset,omitempty" json:"dataset,omitempty"` + Evaluators evalcore.EvaluatorList `yaml:"evaluators,omitempty" json:"evaluators,omitempty"` + Target *Target `yaml:"target,omitempty" json:"target,omitempty"` + Options *Options `yaml:"options,omitempty" json:"options,omitempty"` } // DatasetDecl declares a dataset. A local Source is uploaded on deploy; without @@ -61,9 +69,12 @@ type Target struct { const TargetTypeAgent = "agent" -// Options are run settings carried on the group. +// Options are run settings carried on the eval. +// +// There is deliberately no judge-model option. A judge deployment is a testing +// criterion's `initialization_parameters.deployment_name`, which differs per +// evaluator, so it is declared on the evaluator reference instead. type Options struct { - EvalModel string `yaml:"eval_model,omitempty" json:"eval_model,omitempty"` MaxSamples int `yaml:"max_samples,omitempty" json:"max_samples,omitempty"` EvaluationLevel string `yaml:"evaluation_level,omitempty" json:"evaluation_level,omitempty"` } @@ -74,7 +85,7 @@ const ( EvaluationLevelConversation = "conversation" ) -// LoadEvalConfig reads a deployment spec from disk. The path is used verbatim, +// LoadEvalConfig reads an eval body from disk. The path is used verbatim, // relative to the process working directory — never re-rooted. func LoadEvalConfig(path string) (*EvalConfig, error) { data, err := os.ReadFile(path) @@ -89,18 +100,77 @@ func LoadEvalConfig(path string) (*EvalConfig, error) { return &cfg, nil } +// EvalNamesIn lists the evals declared under evalDir, in sorted order. +// +// One file is one eval, named after it. The generation spec shares the +// directory and is not one, so it is excluded by name. +func EvalNamesIn(evalDir string) ([]string, error) { + entries, err := os.ReadDir(evalDir) + if err != nil { + return nil, err + } + + var names []string + for _, e := range entries { + if e.IsDir() { + continue + } + ext := strings.ToLower(filepath.Ext(e.Name())) + if ext != ".yaml" && ext != ".yml" { + continue + } + name := strings.TrimSuffix(e.Name(), filepath.Ext(e.Name())) + if name == generateConfigBase { + continue + } + names = append(names, name) + } + sort.Strings(names) + return names, nil +} + +// generateConfigBase is the reserved file name in the evals directory. +const generateConfigBase = "generate" + +// ResolveEvalConfigPath finds the config file holding one eval's body. +// +// A named eval is evals/.yaml. With no name the directory must hold +// exactly one eval, and anything else names the candidates rather than +// picking one, because guessing which eval a command meant is the kind of +// mistake that is only noticed after it has run. +func ResolveEvalConfigPath(evalDir, evalName string) (string, error) { + if evalName != "" { + path := EvalConfigPath(evalDir, evalName) + if _, err := os.Stat(path); err != nil { + return "", fmt.Errorf("eval %q is not declared in %s", evalName, evalDir) + } + return path, nil + } + + names, err := EvalNamesIn(evalDir) + if err != nil { + return "", fmt.Errorf("reading %s: %w", evalDir, err) + } + switch len(names) { + case 0: + return "", fmt.Errorf("no evals are declared in %s", evalDir) + case 1: + return EvalConfigPath(evalDir, names[0]), nil + default: + return "", fmt.Errorf( + "%s declares %d evals (%s); choose one with --eval", + evalDir, len(names), strings.Join(names, ", ")) + } +} + // Validate checks the invariants the provider relies on before it calls the // service, so failures surface as config errors rather than opaque 4xx. func (c *EvalConfig) Validate() error { - datasets := map[string]bool{} - for i, d := range c.Datasets { - if d.Name == "" { - return fmt.Errorf("datasets[%d]: 'name' is required", i) - } - if datasets[d.Name] { - return fmt.Errorf("datasets[%d]: duplicate dataset name %q", i, d.Name) - } - datasets[d.Name] = true + if c.Dataset != nil && c.Dataset.Name == "" { + return fmt.Errorf("dataset: 'name' is required") + } + if len(c.Evaluators) == 0 { + return fmt.Errorf("at least one evaluator is required") } evaluators := map[string]bool{} @@ -108,17 +178,22 @@ func (c *EvalConfig) Validate() error { if e.Name == "" { return fmt.Errorf("evaluators[%d]: 'name' is required", i) } - if strings.HasPrefix(e.Name, evalcore.BuiltinPrefix) { - return fmt.Errorf( - "evaluators[%d]: built-in evaluator %q must not be declared; "+ - "reference it directly from an eval", i, e.Name) - } if evaluators[e.Name] { return fmt.Errorf("evaluators[%d]: duplicate evaluator name %q", i, e.Name) } + evaluators[e.Name] = true + + if e.IsBuiltin() { + if e.Source != "" { + return fmt.Errorf( + "evaluators[%d] (%s): a built-in evaluator has no source to publish", + i, e.Name) + } + continue + } // The service assigns an evaluator's version on publish, so a declared // one cannot be honoured alongside a source: the upload lands on - // whatever comes next and the group binds that, leaving the pin + // whatever comes next and the eval binds that, leaving the pin // describing a version nothing uses. if e.Source != "" && e.Version != "" { return fmt.Errorf( @@ -127,110 +202,55 @@ func (c *EvalConfig) Validate() error { "publish this file, or drop `source` to reference a version already "+ "on the project", i, e.Name) } - evaluators[e.Name] = true } - groups := map[string]bool{} - for i, g := range c.Evals { - if g.Name == "" { - return fmt.Errorf("evals[%d]: 'name' is required", i) - } - if groups[g.Name] { - return fmt.Errorf("evals[%d]: duplicate eval name %q", i, g.Name) - } - groups[g.Name] = true - - if g.Dataset != "" && !datasets[g.Dataset] { - return fmt.Errorf( - "evals[%d] (%s): dataset %q is not declared in datasets", - i, g.Name, g.Dataset) - } - if len(g.Evaluators) == 0 { - return fmt.Errorf("evals[%d] (%s): at least one evaluator is required", i, g.Name) - } - for _, ref := range g.Evaluators { - if ref.IsBuiltin() { - continue - } - if !evaluators[ref.Name] { - return fmt.Errorf( - "evals[%d] (%s): evaluator %q is not declared in evaluators "+ - "(built-ins need the %q prefix)", - i, g.Name, ref.Name, evalcore.BuiltinPrefix) - } - } - if g.Target != nil && g.Target.Type != "" && g.Target.Type != TargetTypeAgent { + if c.Target != nil && c.Target.Type != "" && c.Target.Type != TargetTypeAgent { + return fmt.Errorf( + "target.type %q is not supported; use %q", c.Target.Type, TargetTypeAgent) + } + if c.Options != nil { + switch c.Options.EvaluationLevel { + case "", EvaluationLevelTurn, EvaluationLevelConversation: + default: return fmt.Errorf( - "evals[%d] (%s): target.type %q is not supported; use %q", - i, g.Name, g.Target.Type, TargetTypeAgent) - } - if g.Options != nil { - switch g.Options.EvaluationLevel { - case "", EvaluationLevelTurn, EvaluationLevelConversation: - default: - return fmt.Errorf( - "evals[%d] (%s): evaluation_level %q is invalid; expected %q or %q", - i, g.Name, g.Options.EvaluationLevel, - EvaluationLevelTurn, EvaluationLevelConversation) - } + "options.evaluation_level %q is invalid; expected %q or %q", + c.Options.EvaluationLevel, EvaluationLevelTurn, EvaluationLevelConversation) } } return nil } -// Dataset returns the declaration with the given name. -func (c *EvalConfig) Dataset(name string) (*DatasetDecl, bool) { - for i := range c.Datasets { - if c.Datasets[i].Name == name { - return &c.Datasets[i], true - } - } - return nil, false -} - -// Evaluator returns the declaration with the given name. -func (c *EvalConfig) Evaluator(name string) (*EvaluatorDecl, bool) { - for i := range c.Evaluators { - if c.Evaluators[i].Name == name { - return &c.Evaluators[i], true - } +// Eval resolves the config into the eval the reconciler publishes, taking its +// name from the service entry that pulled the file in. +func (c *EvalConfig) Eval(name string) Eval { + resolved := Eval{ + Name: name, + Description: c.Description, + Evaluators: c.Evaluators, + Target: c.Target, + Options: c.Options, } - return nil, false -} - -// Group returns the eval with the given name. -func (c *EvalConfig) Group(name string) (*Eval, bool) { - for i := range c.Evals { - if c.Evals[i].Name == name { - return &c.Evals[i], true - } + if c.Dataset != nil { + resolved.Dataset = c.Dataset.Name } - return nil, false + return resolved } -// ResolveGroup picks the group to act on: the named one, or the only one when -// the config declares exactly one. -func (c *EvalConfig) ResolveGroup(name string) (*Eval, error) { - if name != "" { - g, ok := c.Group(name) - if !ok { - return nil, fmt.Errorf("eval %q is not declared in the config", name) - } - return g, nil - } - switch len(c.Evals) { - case 0: - return nil, fmt.Errorf("no evals are declared in the config") - case 1: - return &c.Evals[0], nil - default: - names := make([]string, 0, len(c.Evals)) - for _, g := range c.Evals { - names = append(names, g.Name) +// CustomEvaluators are the evaluators this config owns — the referenced ones +// carrying a local source, which are published before the eval that names them. +// A built-in needs nothing, and one without a source is already registered. +func (c *EvalConfig) CustomEvaluators() []EvaluatorDecl { + var owned []EvaluatorDecl + for _, ref := range c.Evaluators { + if ref.IsBuiltin() || ref.Source == "" { + continue } - return nil, fmt.Errorf( - "the config declares %d evals (%s); choose one with --eval", - len(c.Evals), strings.Join(names, ", ")) + owned = append(owned, EvaluatorDecl{ + Name: ref.Name, + Source: ref.Source, + Version: ref.Version, + }) } + return owned } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_test.go index 76a5a0cc96b..03e58c5cf35 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_test.go @@ -14,38 +14,36 @@ import ( "go.yaml.in/yaml/v3" ) -const sampleDeployConfig = ` +// sampleEvalConfig is the shape the spec documents for evals/.yaml. +const sampleEvalConfig = ` +description: Quality gate for the support agent + +dataset: + name: support-golden + source: ./datasets/support-golden.jsonl + version: "1" + evaluators: + - builtin.task_adherence - name: support-quality - source: ./evaluators/support-quality/rubric_dimensions.json - - name: safety-check - source: ./evaluators/safety-check.json - -datasets: - - name: support-golden - source: ./datasets/support-golden.jsonl - version: "1" - -evals: - - name: pr-gate - description: Quality gate for the support agent - dataset: support-golden - evaluators: - - builtin.task_adherence - - { name: support-quality, threshold: 4.0 } - - safety-check - target: - type: agent - name: support-agent - options: - eval_model: gpt-4.1-nano - max_samples: 100 - evaluation_level: conversation + source: ./evaluators/support-quality.json + threshold: 4.0 + initialization_parameters: + deployment_name: gpt-4.1-nano + - safety-check + +target: + type: agent + name: support-agent + +options: + max_samples: 100 + evaluation_level: conversation ` func loadFromString(t *testing.T, body string) *EvalConfig { t.Helper() - path := filepath.Join(t.TempDir(), "azure.yaml") + path := filepath.Join(t.TempDir(), "support-agent-smoke.yaml") require.NoError(t, os.WriteFile(path, []byte(body), 0o600)) cfg, err := LoadEvalConfig(path) require.NoError(t, err) @@ -53,45 +51,66 @@ func loadFromString(t *testing.T, body string) *EvalConfig { } func TestLoadEvalConfig_ParsesAllSections(t *testing.T) { - cfg := loadFromString(t, sampleDeployConfig) - - require.Len(t, cfg.Evaluators, 2) - require.Len(t, cfg.Datasets, 1) - require.Len(t, cfg.Evals, 1) - - ds, ok := cfg.Dataset("support-golden") - require.True(t, ok) - require.Equal(t, "./datasets/support-golden.jsonl", ds.Source) - require.Equal(t, "1", ds.Version) - - g, ok := cfg.Group("pr-gate") - require.True(t, ok) - require.Equal(t, "support-golden", g.Dataset) - require.Equal(t, TargetTypeAgent, g.Target.Type) - require.Equal(t, "support-agent", g.Target.Name) - require.Equal(t, EvaluationLevelConversation, g.Options.EvaluationLevel) + cfg := loadFromString(t, sampleEvalConfig) + + require.NotNil(t, cfg.Dataset) + require.Equal(t, "support-golden", cfg.Dataset.Name) + require.Equal(t, "./datasets/support-golden.jsonl", cfg.Dataset.Source) + require.Equal(t, "1", cfg.Dataset.Version) + + require.Len(t, cfg.Evaluators, 3) + require.Equal(t, TargetTypeAgent, cfg.Target.Type) + require.Equal(t, "support-agent", cfg.Target.Name) + require.Equal(t, EvaluationLevelConversation, cfg.Options.EvaluationLevel) + require.Equal(t, 100, cfg.Options.MaxSamples) +} + +// The eval takes its name from the service entry that pulled the file in, so +// the body never repeats it. +func TestEval_TakesNameFromTheService(t *testing.T) { + cfg := loadFromString(t, sampleEvalConfig) + + eval := cfg.Eval("support-agent-smoke") + require.Equal(t, "support-agent-smoke", eval.Name) + require.Equal(t, "support-golden", eval.Dataset) + require.Equal(t, "Quality gate for the support agent", eval.Description) + require.Len(t, eval.Evaluators, 3) + require.Same(t, cfg.Target, eval.Target) } -// Evaluator entries accept a bare string or a mapping carrying a threshold. +// Only the referenced evaluators carrying a local source are this config's to +// publish. A built-in needs nothing, and one without a source already exists. +func TestCustomEvaluators_OnlyOwnsLocalSources(t *testing.T) { + cfg := loadFromString(t, sampleEvalConfig) + + owned := cfg.CustomEvaluators() + require.Len(t, owned, 1) + require.Equal(t, "support-quality", owned[0].Name) + require.Equal(t, "./evaluators/support-quality.json", owned[0].Source) +} + +// Evaluator entries accept a bare string or a mapping carrying the rest of the +// declaration. func TestEvaluatorList_MixedForms(t *testing.T) { - cfg := loadFromString(t, sampleDeployConfig) - g, ok := cfg.Group("pr-gate") - require.True(t, ok) - require.Len(t, g.Evaluators, 3) - - require.Equal(t, "builtin.task_adherence", g.Evaluators[0].Name) - require.True(t, g.Evaluators[0].IsBuiltin()) - require.Equal(t, "task_adherence", g.Evaluators[0].APIName(), + cfg := loadFromString(t, sampleEvalConfig) + require.Len(t, cfg.Evaluators, 3) + + require.Equal(t, "builtin.task_adherence", cfg.Evaluators[0].Name) + require.True(t, cfg.Evaluators[0].IsBuiltin()) + require.Equal(t, "task_adherence", cfg.Evaluators[0].APIName(), "the builtin prefix must be stripped before it reaches the service") - require.Nil(t, g.Evaluators[0].Threshold) + require.Nil(t, cfg.Evaluators[0].Threshold) - require.Equal(t, "support-quality", g.Evaluators[1].Name) - require.False(t, g.Evaluators[1].IsBuiltin()) - require.NotNil(t, g.Evaluators[1].Threshold) - require.InDelta(t, 4.0, *g.Evaluators[1].Threshold, 0.0001) + require.Equal(t, "support-quality", cfg.Evaluators[1].Name) + require.False(t, cfg.Evaluators[1].IsBuiltin()) + require.NotNil(t, cfg.Evaluators[1].Threshold) + require.InDelta(t, 4.0, *cfg.Evaluators[1].Threshold, 0.0001) + require.Equal(t, "gpt-4.1-nano", + cfg.Evaluators[1].InitializationParameters["deployment_name"], + "the judge deployment is declared per evaluator, not once per eval") - require.Equal(t, "safety-check", g.Evaluators[2].Name) - require.Nil(t, g.Evaluators[2].Threshold) + require.Equal(t, "safety-check", cfg.Evaluators[2].Name) + require.Nil(t, cfg.Evaluators[2].Threshold) } // Round-tripping must not rewrite bare names into mappings. @@ -115,8 +134,27 @@ func TestEvaluatorList_RoundTripKeepsCompactForm(t *testing.T) { "an evaluator with only a name should stay a plain string") } +// An evaluator carrying a source must not be flattened to its name, or the +// declaration that says what to publish is lost on the next write. +func TestEvaluatorList_RoundTripKeepsSource(t *testing.T) { + list := evalcore.EvaluatorList{ + {Name: "support-quality", Source: "./evaluators/support-quality.json"}, + {Name: "builtin.task_adherence", + InitializationParameters: map[string]any{"deployment_name": "gpt-4.1-nano"}}, + } + + out, err := yaml.Marshal(list) + require.NoError(t, err) + + var back evalcore.EvaluatorList + require.NoError(t, yaml.Unmarshal(out, &back)) + require.Len(t, back, 2) + require.Equal(t, "./evaluators/support-quality.json", back[0].Source) + require.Equal(t, "gpt-4.1-nano", back[1].InitializationParameters["deployment_name"]) +} + func TestValidate_Accepts(t *testing.T) { - require.NoError(t, loadFromString(t, sampleDeployConfig).Validate()) + require.NoError(t, loadFromString(t, sampleEvalConfig).Validate()) } func TestValidate_Rejects(t *testing.T) { @@ -126,42 +164,41 @@ func TestValidate_Rejects(t *testing.T) { wantErr string }{ { - name: "dataset referenced but not declared", - body: "evals:\n - name: g\n dataset: missing\n evaluators: [builtin.relevance]\n", - wantErr: "is not declared in datasets", + name: "dataset without a name", + body: "dataset:\n source: ./d.jsonl\nevaluators: [builtin.relevance]\n", + wantErr: "'name' is required", }, { - name: "custom evaluator referenced but not declared", - body: "datasets:\n - name: d\n" + - "evals:\n - name: g\n dataset: d\n evaluators: [not-declared]\n", - wantErr: "is not declared in evaluators", + name: "no evaluators", + body: "evaluators: []\n", + wantErr: "at least one evaluator is required", }, { - name: "built-in declared as a custom evaluator", - body: "evaluators:\n - name: builtin.relevance\n", - wantErr: "must not be declared", + name: "built-in with a source to publish", + body: "evaluators:\n - name: builtin.relevance\n source: ./x.json\n", + wantErr: "has no source to publish", }, { - name: "group without evaluators", - body: "evals:\n - name: g\n evaluators: []\n", - wantErr: "at least one evaluator is required", + name: "duplicate evaluator", + body: "evaluators: [builtin.relevance, builtin.relevance]\n", + wantErr: "duplicate evaluator name", + }, + { + name: "version pinned alongside a source", + body: "evaluators:\n - name: q\n source: ./q.json\n version: \"3\"\n", + wantErr: "cannot be set with `source`", }, { name: "unsupported target type", - body: "evals:\n - name: g\n evaluators: [builtin.relevance]\n target:\n type: prompt\n", + body: "evaluators: [builtin.relevance]\ntarget:\n type: prompt\n", wantErr: "is not supported", }, { name: "invalid evaluation level", - body: "evals:\n - name: g\n evaluators: [builtin.relevance]\n" + - " options:\n evaluation_level: sentence\n", + body: "evaluators: [builtin.relevance]\n" + + "options:\n evaluation_level: sentence\n", wantErr: "evaluation_level", }, - { - name: "duplicate dataset", - body: "datasets:\n - name: d\n - name: d\n", - wantErr: "duplicate dataset name", - }, } for _, tc := range cases { @@ -173,55 +210,73 @@ func TestValidate_Rejects(t *testing.T) { } } -func TestResolveGroup(t *testing.T) { - single := loadFromString(t, sampleDeployConfig) +// One file is one eval, named after the file, so the directory listing is the +// list of evals a project declares. +func TestResolveEvalConfigPath(t *testing.T) { + write := func(t *testing.T, dir string, names ...string) { + t.Helper() + for _, n := range names { + require.NoError(t, os.WriteFile( + filepath.Join(dir, n), []byte("evaluators: [builtin.relevance]\n"), 0o600)) + } + } + + t.Run("the only eval is used when unnamed", func(t *testing.T) { + dir := t.TempDir() + write(t, dir, "pr-gate.yaml", "generate.yaml") - t.Run("only group is used when unnamed", func(t *testing.T) { - g, err := single.ResolveGroup("") + path, err := ResolveEvalConfigPath(dir, "") require.NoError(t, err) - require.Equal(t, "pr-gate", g.Name) + require.Equal(t, filepath.Join(dir, "pr-gate.yaml"), path, + "the generation spec shares the directory and is not an eval") }) - t.Run("named group", func(t *testing.T) { - g, err := single.ResolveGroup("pr-gate") + t.Run("named eval", func(t *testing.T) { + dir := t.TempDir() + write(t, dir, "pr-gate.yaml", "nightly.yaml") + + path, err := ResolveEvalConfigPath(dir, "nightly") require.NoError(t, err) - require.Equal(t, "pr-gate", g.Name) + require.Equal(t, filepath.Join(dir, "nightly.yaml"), path) }) t.Run("unknown name is an error", func(t *testing.T) { - _, err := single.ResolveGroup("nope") + dir := t.TempDir() + write(t, dir, "pr-gate.yaml") + + _, err := ResolveEvalConfigPath(dir, "nope") require.ErrorContains(t, err, "is not declared") }) t.Run("ambiguous without a name", func(t *testing.T) { - multi := loadFromString(t, - "evals:\n - name: pr-gate\n evaluators: [builtin.relevance]\n"+ - " - name: nightly\n evaluators: [builtin.relevance]\n") - _, err := multi.ResolveGroup("") + dir := t.TempDir() + write(t, dir, "pr-gate.yaml", "nightly.yaml") + + _, err := ResolveEvalConfigPath(dir, "") require.ErrorContains(t, err, "--eval") require.ErrorContains(t, err, "nightly") }) - t.Run("empty config", func(t *testing.T) { - _, err := (&EvalConfig{}).ResolveGroup("") + t.Run("empty directory", func(t *testing.T) { + _, err := ResolveEvalConfigPath(t.TempDir(), "") require.ErrorContains(t, err, "no evals") }) } -// local_dir accepts a directory or an explicit file path. +// outputDir accepts a directory or an explicit file path. func TestArtifactPath(t *testing.T) { cases := []struct { - name string - localDir string - resource string - ext string - want string + name string + outputDir string + resource string + ext string + want string }{ {"directory derives the file name", "datasets", "support-golden", ".jsonl", filepath.Join("base", "datasets", "support-golden.jsonl")}, {"explicit file path is used as-is", "generated/datasets/support-golden.jsonl", "ignored", ".jsonl", filepath.Join("base", "generated", "datasets", "support-golden.jsonl")}, - {"empty local_dir falls back to the base", "", "support-quality", ".json", + {"empty outputDir falls back to the base", "", "support-quality", ".json", filepath.Join("base", "support-quality.json")}, {"yaml rubric file path", "generated/rubrics/quality.yaml", "ignored", ".json", filepath.Join("base", "generated", "rubrics", "quality.yaml")}, @@ -229,7 +284,7 @@ func TestArtifactPath(t *testing.T) { for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { - require.Equal(t, tc.want, ArtifactPath("base", tc.localDir, tc.resource, tc.ext)) + require.Equal(t, tc.want, ArtifactPath("base", tc.outputDir, tc.resource, tc.ext)) }) } } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/generate_config.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/generate_config.go index 1c41c92f9cf..7ab28e5d7ac 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/generate_config.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/generate_config.go @@ -12,70 +12,82 @@ import ( "go.yaml.in/yaml/v3" ) -// Conventional locations. Both are relative to the working directory and are +// Conventional locations. All are relative to the working directory and are // used verbatim — never re-rooted under the agent or project directory. const ( DefaultEvalDir = "evals" - DefaultGenerateConfig = "evals/eval_generate.yaml" - DefaultDeployConfig = "evals/azure.yaml" + DefaultGenerateConfig = "evals/generate.yaml" DefaultDatasetsDir = "datasets" DefaultEvaluatorsDir = "evaluators" ) -// GenerateConfig is the generation spec — input to `azd ai eval generate`. It -// is never deployed. -type GenerateConfig struct { - Agent AgentSpec `yaml:"agent" json:"agent"` - Generate GenerateSpec `yaml:"generate" json:"generate"` -} - -// AgentSpec identifies the agent and the context the generator reads. -type AgentSpec struct { - Name string `yaml:"name" json:"name"` - Context AgentContext `yaml:"context,omitempty" json:"context,omitempty"` +// EvalConfigPath is where the body of the eval named by a service entry lives. +func EvalConfigPath(evalDir, evalName string) string { + return filepath.Join(evalDir, evalName+".yaml") } -// AgentContext points at the material used to synthesize a rubric and dataset. -type AgentContext struct { - Instructions string `yaml:"instructions,omitempty" json:"instructions,omitempty"` - Traces *TraceSpec `yaml:"traces,omitempty" json:"traces,omitempty"` -} - -// TraceSpec seeds rubric generation from recent traces. Traces are a generation -// input only; they cannot be a run's data source. -type TraceSpec struct { - Source string `yaml:"source,omitempty" json:"source,omitempty"` - Window string `yaml:"window,omitempty" json:"window,omitempty"` - Sample int `yaml:"sample,omitempty" json:"sample,omitempty"` +// GenerateConfig says how the local dataset and evaluator artifacts referenced +// by an eval are produced. It is never deployed. +// +// generationModel: gpt-5.6-luna +// dataset: +// support-agent-smoke: +// sampleSize: 15 +// outputDir: ./datasets +// evaluator: +// support-quality: +// outputDir: ./evaluators +// deriveFrom: support-agent +// +// The maps are keyed by artifact name so `dataset generate ` and +// `evaluator generate ` each look up exactly the entry they were asked +// for, and generating one artifact never reads the other's settings. +type GenerateConfig struct { + GenerationModel string `yaml:"generationModel,omitempty" json:"generationModel,omitempty"` + Dataset map[string]DatasetGenSpec `yaml:"dataset,omitempty" json:"dataset,omitempty"` + Evaluator map[string]EvaluatorGenSpec `yaml:"evaluator,omitempty" json:"evaluator,omitempty"` } -// GenerateSpec configures what gets produced. -type GenerateSpec struct { - Rubric *RubricSpec `yaml:"rubric,omitempty" json:"rubric,omitempty"` - Dataset *DatasetSpec `yaml:"dataset,omitempty" json:"dataset,omitempty"` +// DatasetGenSpec configures synthetic dataset generation for one dataset. +type DatasetGenSpec struct { + SampleSize int `yaml:"sampleSize,omitempty" json:"sampleSize,omitempty"` + OutputDir string `yaml:"outputDir,omitempty" json:"outputDir,omitempty"` + // DeriveFrom names the agent whose context seeds generation. Optional: the + // eval's target supplies it, and --target overrides both. + DeriveFrom string `yaml:"deriveFrom,omitempty" json:"deriveFrom,omitempty"` + // Instructions points at a local file whose contents stand in for the + // agent's published instructions for this generation only. + Instructions string `yaml:"instructions,omitempty" json:"instructions,omitempty"` + // TraceDays seeds generation from that many days of recent traces. Zero + // disables it. Traces are a generation input only; they cannot be a run's + // data source. + TraceDays int `yaml:"traceDays,omitempty" json:"traceDays,omitempty"` } -// RubricSpec configures rubric (LLM-graded evaluator) generation. -type RubricSpec struct { - Name string `yaml:"name" json:"name"` - Model string `yaml:"model,omitempty" json:"model,omitempty"` - LocalDir string `yaml:"local_dir,omitempty" json:"local_dir,omitempty"` +// EvaluatorGenSpec configures rubric generation for one evaluator. +type EvaluatorGenSpec struct { + OutputDir string `yaml:"outputDir,omitempty" json:"outputDir,omitempty"` + // DeriveFrom names the agent the rubric is written against. + DeriveFrom string `yaml:"deriveFrom,omitempty" json:"deriveFrom,omitempty"` + // Instructions points at a local file whose contents stand in for the + // agent's published instructions for this generation only. + Instructions string `yaml:"instructions,omitempty" json:"instructions,omitempty"` + // TraceDays seeds generation from that many days of recent traces. Zero + // disables it. + TraceDays int `yaml:"traceDays,omitempty" json:"traceDays,omitempty"` } -// DatasetSpec configures synthetic dataset generation. -type DatasetSpec struct { - Name string `yaml:"name" json:"name"` - Strategy string `yaml:"strategy,omitempty" json:"strategy,omitempty"` - SampleSize int `yaml:"sampleSize,omitempty" json:"sampleSize,omitempty"` - LocalDir string `yaml:"local_dir,omitempty" json:"local_dir,omitempty"` +// ArtifactRef is the name/source pair a generation run produces, so the +// command can tell the developer how to reference it. +// +// Generation writes artifacts only and never edits azure.yaml or the eval +// config: `init` declares the paths and `generate` fills them in, which is what +// keeps a generation run a data-file-only diff. +type ArtifactRef struct { + Name string `json:"name"` + Source string `json:"source"` } -// Generation strategies. -const ( - StrategySynthetic = "synthetic" - StrategyFromTraces = "from-traces" -) - // Sample-count bounds enforced by the generation service. const ( MinSampleSize = 15 @@ -84,9 +96,16 @@ const ( ) // LoadGenerateConfig reads a generation spec from disk. +// +// A missing file is not an error. Generation is optional — a developer with +// hand-authored data and evaluators never writes one — and every setting it +// carries can be given on the command line instead. func LoadGenerateConfig(path string) (*GenerateConfig, error) { data, err := os.ReadFile(path) if err != nil { + if os.IsNotExist(err) { + return &GenerateConfig{}, nil + } return nil, fmt.Errorf("reading generation config %q: %w", path, err) } @@ -97,58 +116,43 @@ func LoadGenerateConfig(path string) (*GenerateConfig, error) { return &cfg, nil } -// Validate reports configuration errors before any generation job is submitted. -func (c *GenerateConfig) Validate() error { - if c.Agent.Name == "" { - return fmt.Errorf("agent.name is required") - } - if c.Generate.Rubric == nil && c.Generate.Dataset == nil { - return fmt.Errorf("generate must declare a rubric, a dataset, or both") - } - if r := c.Generate.Rubric; r != nil && r.Name == "" { - return fmt.Errorf("generate.rubric.name is required") - } - if d := c.Generate.Dataset; d != nil { - if d.Name == "" { - return fmt.Errorf("generate.dataset.name is required") - } - switch d.Strategy { - case "", StrategySynthetic: - case StrategyFromTraces: - // Accepting this and generating synthetic rows anyway would hand back - // data that looks nothing like what was asked for. The generation API - // takes one dataset strategy today; traces seed generation through - // the agent's context instead. - return fmt.Errorf( - "generate.dataset.strategy %q is not supported yet; "+ - "use %q, and set agent.context.traces.window to seed generation from traces", - StrategyFromTraces, StrategySynthetic) - default: - return fmt.Errorf( - "generate.dataset.strategy %q is invalid; expected %q", - d.Strategy, StrategySynthetic) - } - if d.SampleSize != 0 && (d.SampleSize < MinSampleSize || d.SampleSize > MaxSampleSize) { - return fmt.Errorf( - "generate.dataset.sampleSize must be between %d and %d, got %d", - MinSampleSize, MaxSampleSize, d.SampleSize) - } +// DatasetSpec returns the settings for one dataset, and whether the config +// declared them. +func (c *GenerateConfig) DatasetSpec(name string) (DatasetGenSpec, bool) { + spec, ok := c.Dataset[name] + return spec, ok +} + +// EvaluatorSpec returns the settings for one evaluator, and whether the config +// declared them. +func (c *GenerateConfig) EvaluatorSpec(name string) (EvaluatorGenSpec, bool) { + spec, ok := c.Evaluator[name] + return spec, ok +} + +// ValidateSampleSize rejects a row count the service would reject, before a +// generation job is submitted and billed. +func ValidateSampleSize(n int) error { + if n != 0 && (n < MinSampleSize || n > MaxSampleSize) { + return fmt.Errorf( + "sample size must be between %d and %d, got %d", + MinSampleSize, MaxSampleSize, n) } return nil } -// ArtifactPath resolves a local_dir value against baseDir. The value may be a +// ArtifactPath resolves an outputDir value against baseDir. The value may be a // directory, in which case the file name is derived from resourceName and ext, // or an explicit file path, which is used as-is. -func ArtifactPath(baseDir, localDir, resourceName, ext string) string { - if localDir == "" { +func ArtifactPath(baseDir, outputDir, resourceName, ext string) string { + if outputDir == "" { return filepath.Join(baseDir, resourceName+ext) } - candidate := localDir + candidate := outputDir if !filepath.IsAbs(candidate) { candidate = filepath.Join(baseDir, candidate) } - if looksLikeFile(localDir, ext) { + if looksLikeFile(outputDir, ext) { return candidate } return filepath.Join(candidate, resourceName+ext) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/generate_config_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/generate_config_test.go index 227674b0fc4..5746b2286b6 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/generate_config_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/generate_config_test.go @@ -4,37 +4,65 @@ package project import ( + "os" + "path/filepath" "testing" "github.com/stretchr/testify/require" ) -func generateCfgWithStrategy(strategy string) *GenerateConfig { - cfg := &GenerateConfig{} - cfg.Agent.Name = "my-agent" - cfg.Generate.Rubric = &RubricSpec{Name: "r"} - cfg.Generate.Dataset = &DatasetSpec{Name: "d", Strategy: strategy} - return cfg -} +// The generation spec is keyed by artifact name, which is what makes +// `dataset generate ` and `evaluator generate ` able to look up +// exactly the entry they were asked for. +func TestLoadGenerateConfig_ParsesTheDocumentedShape(t *testing.T) { + body := ` +generationModel: gpt-5.6-luna +dataset: + support-agent-smoke: + sampleSize: 15 + outputDir: ./datasets +evaluator: + support-quality: + outputDir: ./evaluators + deriveFrom: support-agent +` + path := filepath.Join(t.TempDir(), "generate.yaml") + require.NoError(t, os.WriteFile(path, []byte(body), 0o600)) + + cfg, err := LoadGenerateConfig(path) + require.NoError(t, err) + require.Equal(t, "gpt-5.6-luna", cfg.GenerationModel) + + ds, ok := cfg.DatasetSpec("support-agent-smoke") + require.True(t, ok) + require.Equal(t, 15, ds.SampleSize) + require.Equal(t, "./datasets", ds.OutputDir) -// from-traces used to pass validation and then generate synthetic rows anyway, -// handing back data that looked nothing like what was asked for. Rejecting it -// is better than answering the wrong question. -func TestValidateRejectsUnsupportedDatasetStrategy(t *testing.T) { - err := generateCfgWithStrategy(StrategyFromTraces).Validate() - require.Error(t, err) - require.Contains(t, err.Error(), "not supported yet") - require.Contains(t, err.Error(), "agent.context.traces.window", - "the error should point at the way traces are actually used") + ev, ok := cfg.EvaluatorSpec("support-quality") + require.True(t, ok) + require.Equal(t, "./evaluators", ev.OutputDir) + require.Equal(t, "support-agent", ev.DeriveFrom) } -func TestValidateAcceptsSupportedDatasetStrategies(t *testing.T) { - require.NoError(t, generateCfgWithStrategy("").Validate()) - require.NoError(t, generateCfgWithStrategy(StrategySynthetic).Validate()) +// Generation is optional: a developer with hand-authored data and evaluators +// never writes a spec, and the generate commands still run from flags alone. +func TestLoadGenerateConfig_MissingFileIsNotAnError(t *testing.T) { + cfg, err := LoadGenerateConfig(filepath.Join(t.TempDir(), "generate.yaml")) + require.NoError(t, err) + require.Empty(t, cfg.GenerationModel) + require.Empty(t, cfg.Dataset) + + _, ok := cfg.DatasetSpec("anything") + require.False(t, ok) } -func TestValidateRejectsUnknownDatasetStrategy(t *testing.T) { - err := generateCfgWithStrategy("made-up").Validate() - require.Error(t, err) - require.Contains(t, err.Error(), "invalid") +// A row count the service would reject costs a billed job to find out about, +// so it is refused at the flag that carried it. +func TestValidateSampleSize(t *testing.T) { + require.NoError(t, ValidateSampleSize(0), "unset means the default applies") + require.NoError(t, ValidateSampleSize(MinSampleSize)) + require.NoError(t, ValidateSampleSize(MaxSampleSize)) + + require.ErrorContains(t, ValidateSampleSize(MinSampleSize-1), "must be between") + require.ErrorContains(t, ValidateSampleSize(MaxSampleSize+1), "must be between") } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/merge.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/merge.go deleted file mode 100644 index c8329356c23..00000000000 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/merge.go +++ /dev/null @@ -1,171 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package project - -import ( - "fmt" - "os" - - "go.yaml.in/yaml/v3" -) - -// ArtifactRef is a name/source pair written back into the deployment spec after -// generation. -type ArtifactRef struct { - Name string - Source string -} - -// MergeArtifactRefs writes `source:` references for generated artifacts into -// the deployment spec, matching entries by name and appending when absent. -// -// It edits the document through the yaml Node API rather than round-tripping -// through structs, so comments, key order, and formatting survive. Only the -// `source` key of a matched entry is touched; anything the developer hand-edited -// is left alone. -func MergeArtifactRefs(path string, datasets, evaluators []ArtifactRef) error { - data, err := os.ReadFile(path) - if err != nil { - return fmt.Errorf("reading %q: %w", path, err) - } - - var doc yaml.Node - if err := yaml.Unmarshal(data, &doc); err != nil { - return fmt.Errorf("parsing %q: %w", path, err) - } - - root := documentRoot(&doc) - if root == nil { - return fmt.Errorf("%q is not a YAML mapping", path) - } - - if err := mergeSection(root, "datasets", datasets); err != nil { - return fmt.Errorf("%q: %w", path, err) - } - if err := mergeSection(root, "evaluators", evaluators); err != nil { - return fmt.Errorf("%q: %w", path, err) - } - - out, err := yaml.Marshal(&doc) - if err != nil { - return fmt.Errorf("serializing %q: %w", path, err) - } - if err := os.WriteFile(path, out, 0o600); err != nil { - return fmt.Errorf("writing %q: %w", path, err) - } - return nil -} - -// documentRoot unwraps the document node to the top-level mapping. -func documentRoot(doc *yaml.Node) *yaml.Node { - if doc.Kind == yaml.DocumentNode && len(doc.Content) > 0 { - doc = doc.Content[0] - } - if doc.Kind != yaml.MappingNode { - return nil - } - return doc -} - -// mergeSection updates or appends entries in a top-level sequence. -func mergeSection(root *yaml.Node, key string, refs []ArtifactRef) error { - if len(refs) == 0 { - return nil - } - - seq := findOrCreateSequence(root, key) - if seq == nil { - return fmt.Errorf("%q is present but is not a sequence", key) - } - - for _, ref := range refs { - if entry := findEntryByName(seq, ref.Name); entry != nil { - setMappingValue(entry, "source", ref.Source) - continue - } - seq.Content = append(seq.Content, newArtifactNode(ref)) - } - return nil -} - -// findOrCreateSequence returns the sequence node for key, creating an empty one -// when the key is absent. -func findOrCreateSequence(root *yaml.Node, key string) *yaml.Node { - for i := 0; i+1 < len(root.Content); i += 2 { - if root.Content[i].Value != key { - continue - } - value := root.Content[i+1] - if value.Kind == yaml.SequenceNode { - return value - } - // An explicit null is treated as an empty sequence. - if value.Tag == "!!null" { - value.Kind = yaml.SequenceNode - value.Tag = "!!seq" - value.Value = "" - return value - } - return nil - } - - keyNode := &yaml.Node{Kind: yaml.ScalarNode, Tag: "!!str", Value: key} - seqNode := &yaml.Node{Kind: yaml.SequenceNode, Tag: "!!seq"} - root.Content = append(root.Content, keyNode, seqNode) - return seqNode -} - -// findEntryByName locates a mapping entry whose `name` matches. -func findEntryByName(seq *yaml.Node, name string) *yaml.Node { - for _, item := range seq.Content { - if item.Kind != yaml.MappingNode { - continue - } - if mappingValue(item, "name") == name { - return item - } - } - return nil -} - -// mappingValue reads a scalar value from a mapping node. -func mappingValue(node *yaml.Node, key string) string { - for i := 0; i+1 < len(node.Content); i += 2 { - if node.Content[i].Value == key { - return node.Content[i+1].Value - } - } - return "" -} - -// setMappingValue updates a scalar in place, or appends it when absent. Only -// the targeted key is touched. -func setMappingValue(node *yaml.Node, key, value string) { - for i := 0; i+1 < len(node.Content); i += 2 { - if node.Content[i].Value == key { - node.Content[i+1].Kind = yaml.ScalarNode - node.Content[i+1].Tag = "!!str" - node.Content[i+1].Value = value - return - } - } - node.Content = append(node.Content, - &yaml.Node{Kind: yaml.ScalarNode, Tag: "!!str", Value: key}, - &yaml.Node{Kind: yaml.ScalarNode, Tag: "!!str", Value: value}, - ) -} - -// newArtifactNode builds a fresh `{name, source}` entry. -func newArtifactNode(ref ArtifactRef) *yaml.Node { - return &yaml.Node{ - Kind: yaml.MappingNode, - Tag: "!!map", - Content: []*yaml.Node{ - {Kind: yaml.ScalarNode, Tag: "!!str", Value: "name"}, - {Kind: yaml.ScalarNode, Tag: "!!str", Value: ref.Name}, - {Kind: yaml.ScalarNode, Tag: "!!str", Value: "source"}, - {Kind: yaml.ScalarNode, Tag: "!!str", Value: ref.Source}, - }, - } -} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/merge_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/merge_test.go deleted file mode 100644 index 56f159102dd..00000000000 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/merge_test.go +++ /dev/null @@ -1,142 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package project - -import ( - "os" - "path/filepath" - "testing" - - "github.com/stretchr/testify/require" -) - -const handAuthored = `# Eval deployment spec -# Edited by hand - comments must survive generate. -evaluators: - - name: safety-check # hand-authored - source: ./evaluators/safety-check.json - -datasets: - - name: support-golden - source: ./datasets/old.jsonl - version: "3" - -evals: - - name: pr-gate - dataset: support-golden - evaluators: - - builtin.task_adherence - - { name: safety-check, threshold: 4.0 } -` - -func writeTemp(t *testing.T, body string) string { - t.Helper() - path := filepath.Join(t.TempDir(), "azure.yaml") - require.NoError(t, os.WriteFile(path, []byte(body), 0o600)) - return path -} - -// Regenerating must not destroy a hand-edited file. -func TestMergeArtifactRefs_PreservesCommentsAndSiblings(t *testing.T) { - path := writeTemp(t, handAuthored) - - require.NoError(t, MergeArtifactRefs(path, - []ArtifactRef{{Name: "support-golden", Source: "./datasets/new.jsonl"}}, - []ArtifactRef{{Name: "support-quality", Source: "./evaluators/support-quality.json"}}, - )) - - out, err := os.ReadFile(path) - require.NoError(t, err) - text := string(out) - - require.Contains(t, text, "# Eval deployment spec", "leading comments must survive") - require.Contains(t, text, "# hand-authored", "inline comments must survive") - require.Contains(t, text, "./datasets/new.jsonl", "the matched source must be updated") - require.NotContains(t, text, "./datasets/old.jsonl", "the old source must be replaced") - require.Contains(t, text, "support-quality", "a new evaluator must be appended") - require.Contains(t, text, "safety-check", "existing entries must be kept") - - cfg, err := LoadEvalConfig(path) - require.NoError(t, err) - require.NoError(t, cfg.Validate()) - - ds, ok := cfg.Dataset("support-golden") - require.True(t, ok) - require.Equal(t, "./datasets/new.jsonl", ds.Source) - require.Equal(t, "3", ds.Version, "sibling keys must not be disturbed") - require.Len(t, cfg.Evaluators, 2) -} - -// The eval's evaluator list must be left exactly as written. -func TestMergeArtifactRefs_DoesNotTouchEvals(t *testing.T) { - path := writeTemp(t, handAuthored) - require.NoError(t, MergeArtifactRefs(path, nil, - []ArtifactRef{{Name: "support-quality", Source: "./evaluators/q.json"}})) - - cfg, err := LoadEvalConfig(path) - require.NoError(t, err) - g, ok := cfg.Group("pr-gate") - require.True(t, ok) - require.Len(t, g.Evaluators, 2) - require.Equal(t, "builtin.task_adherence", g.Evaluators[0].Name) - require.NotNil(t, g.Evaluators[1].Threshold) -} - -// Sections absent from the file are created rather than erroring. -func TestMergeArtifactRefs_CreatesMissingSections(t *testing.T) { - path := writeTemp(t, "evals:\n - name: pr-gate\n evaluators: [builtin.relevance]\n") - - require.NoError(t, MergeArtifactRefs(path, - []ArtifactRef{{Name: "d1", Source: "./datasets/d1.jsonl"}}, - []ArtifactRef{{Name: "e1", Source: "./evaluators/e1.json"}}, - )) - - cfg, err := LoadEvalConfig(path) - require.NoError(t, err) - require.Len(t, cfg.Datasets, 1) - require.Len(t, cfg.Evaluators, 1) - require.Equal(t, "d1", cfg.Datasets[0].Name) -} - -// Running generate twice must be idempotent. -func TestMergeArtifactRefs_IsIdempotent(t *testing.T) { - path := writeTemp(t, handAuthored) - refs := []ArtifactRef{{Name: "support-golden", Source: "./datasets/new.jsonl"}} - - require.NoError(t, MergeArtifactRefs(path, refs, nil)) - first, err := os.ReadFile(path) - require.NoError(t, err) - - require.NoError(t, MergeArtifactRefs(path, refs, nil)) - second, err := os.ReadFile(path) - require.NoError(t, err) - - require.Equal(t, string(first), string(second), - "merging the same references twice must not change the file") -} - -func TestFingerprint_DetectsChange(t *testing.T) { - dir := t.TempDir() - path := filepath.Join(dir, "data.jsonl") - - require.NoError(t, os.WriteFile(path, []byte(`{"query":"a"}`), 0o600)) - first, err := Fingerprint(path) - require.NoError(t, err) - - again, err := Fingerprint(path) - require.NoError(t, err) - require.Equal(t, first, again, "unchanged content must hash the same") - - require.NoError(t, os.WriteFile(path, []byte(`{"query":"b"}`), 0o600)) - changed, err := Fingerprint(path) - require.NoError(t, err) - require.NotEqual(t, first, changed, "changed content must hash differently") -} - -func TestFingerprintKey_IsEnvSafe(t *testing.T) { - require.Equal(t, "EVAL_FINGERPRINT_DATASET_SUPPORT_GOLDEN", - FingerprintKey("dataset", "support-golden")) - require.Equal(t, "EVAL_FINGERPRINT_EVALUATOR_MY_EVAL_1", - FingerprintKey("evaluator", "my.eval-1")) -} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go index f4902f236a3..6fd01d06680 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go @@ -145,21 +145,27 @@ func (p *EvalServiceTargetProvider) Deploy( baseDir := serviceRelativeDir(serviceConfig) - // 1. Datasets. + // The eval takes its name from the service entry that pulled this config + // in, which is what makes one service per eval work. + eval := cfg.Eval(serviceConfig.Name) + + // 1. Dataset. anyChanged := false - for _, decl := range cfg.Datasets { - report(progress, fmt.Sprintf("Reconciling dataset %s", decl.Name)) - localPath := resolveSource(baseDir, decl.Source) - version, changed, err := reconciler.EnsureDataset(ctx, decl, localPath) + datasetPath := "" + if cfg.Dataset != nil { + report(progress, fmt.Sprintf("Reconciling dataset %s", cfg.Dataset.Name)) + datasetPath = resolveSource(baseDir, cfg.Dataset.Source) + version, changed, err := reconciler.EnsureDataset(ctx, *cfg.Dataset, datasetPath) if err != nil { - return nil, fmt.Errorf("dataset %q: %w", decl.Name, err) + return nil, fmt.Errorf("dataset %q: %w", cfg.Dataset.Name, err) } anyChanged = anyChanged || changed - report(progress, describeResult("dataset", decl.Name, version, changed)) + report(progress, describeResult("dataset", cfg.Dataset.Name, version, changed)) } - // 2. Evaluators. - for _, decl := range cfg.Evaluators { + // 2. Evaluators this config owns. Built-ins and already-registered ones + // need no publish. + for _, decl := range cfg.CustomEvaluators() { report(progress, fmt.Sprintf("Reconciling evaluator %s", decl.Name)) localPath := resolveSource(baseDir, decl.Source) version, changed, err := reconciler.EnsureEvaluator(ctx, decl, localPath) @@ -170,20 +176,14 @@ func (p *EvalServiceTargetProvider) Deploy( report(progress, describeResult("evaluator", decl.Name, version, changed)) } - // 3. Evals. Groups are immutable, so a change upstream means a new - // group must be created and the stored id replaced. - for _, group := range cfg.Evals { - report(progress, fmt.Sprintf("Reconciling eval %s", group.Name)) - datasetPath := "" - if decl, ok := cfg.Dataset(group.Dataset); ok { - datasetPath = resolveSource(baseDir, decl.Source) - } - id, err := reconciler.EnsureEval(ctx, group, datasetPath, anyChanged) - if err != nil { - return nil, fmt.Errorf("eval %q: %w", group.Name, err) - } - report(progress, fmt.Sprintf("Eval %s is %s", group.Name, id)) + // 3. The eval. Evals are immutable, so a change upstream means a new one + // must be created and the stored id replaced. + report(progress, fmt.Sprintf("Reconciling eval %s", eval.Name)) + id, err := reconciler.EnsureEval(ctx, eval, datasetPath, anyChanged) + if err != nil { + return nil, fmt.Errorf("eval %q: %w", eval.Name, err) } + report(progress, fmt.Sprintf("Eval %s is %s", eval.Name, id)) return &azdext.ServiceDeployResult{}, nil } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval_test.go index bd24e6c0030..987b2815c44 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval_test.go @@ -68,29 +68,25 @@ func TestServiceRelativeDirDefaultsToProjectRoot(t *testing.T) { // includes against. func TestEvalConfigFromServiceReadsInlineConfig(t *testing.T) { svc := &azdext.ServiceConfig{ - Name: "evals", + Name: "support-agent-smoke", AdditionalProperties: propsFrom(t, map[string]any{ - "datasets": []any{ - map[string]any{"name": "golden", "source": "./datasets/golden.jsonl"}, - }, - "evals": []any{ - map[string]any{ - "name": "quality", - "dataset": "golden", - "evaluators": []any{"builtin.task_adherence"}, - "target": map[string]any{"type": "agent", "name": "my-agent"}, - }, - }, + "dataset": map[string]any{"name": "golden", "source": "./datasets/golden.jsonl"}, + "evaluators": []any{"builtin.task_adherence"}, + "target": map[string]any{"type": "agent", "name": "my-agent"}, }), } cfg, err := EvalConfigFromService(svc, "") require.NoError(t, err) - require.Len(t, cfg.Datasets, 1) - require.Equal(t, "golden", cfg.Datasets[0].Name) - require.Len(t, cfg.Evals, 1) - require.Len(t, cfg.Evals[0].Evaluators, 1) - require.Equal(t, "builtin.task_adherence", cfg.Evals[0].Evaluators[0].Name) + require.NotNil(t, cfg.Dataset) + require.Equal(t, "golden", cfg.Dataset.Name) + require.Len(t, cfg.Evaluators, 1) + require.Equal(t, "builtin.task_adherence", cfg.Evaluators[0].Name) + require.Equal(t, "my-agent", cfg.Target.Name) + + // The eval's name is the service key, which is what makes one service per + // eval work without the body repeating it. + require.Equal(t, "support-agent-smoke", cfg.Eval(svc.Name).Name) } func TestEvalConfigFromServiceRejectsEmptyService(t *testing.T) { @@ -99,16 +95,16 @@ func TestEvalConfigFromServiceRejectsEmptyService(t *testing.T) { require.Contains(t, err.Error(), "no eval configuration") } -// Groups are immutable, so a change to the group's own declaration has to be -// detectable. Upstream artifact fingerprints do not cover it: retargeting a -// group at a different agent leaves the dataset and evaluators untouched. +// Evals are immutable, so a change to the eval's own declaration has to be +// detectable. Upstream artifact fingerprints do not cover it: retargeting an +// eval at a different agent leaves the dataset and evaluators untouched. func TestFingerprintGroupTracksMeaningfulChanges(t *testing.T) { base := Eval{ Name: "quality", Dataset: "golden", Evaluators: evalcore.EvaluatorList{{Name: "builtin.task_adherence"}}, Target: &Target{Type: "agent", Name: "agent-a"}, - Options: &Options{EvalModel: "gpt-4.1-nano"}, + Options: &Options{EvaluationLevel: EvaluationLevelTurn}, } original, err := FingerprintGroup(base) @@ -116,14 +112,20 @@ func TestFingerprintGroupTracksMeaningfulChanges(t *testing.T) { same, err := FingerprintGroup(base) require.NoError(t, err) - require.Equal(t, original, same, "an unchanged group must keep its fingerprint") + require.Equal(t, original, same, "an unchanged eval must keep its fingerprint") cases := map[string]func(g *Eval){ "target": func(g *Eval) { g.Target = &Target{Type: "agent", Name: "agent-b"} }, "evaluators": func(g *Eval) { g.Evaluators = append(g.Evaluators, evalcore.EvaluatorRef{Name: "builtin.similarity"}) }, - "options": func(g *Eval) { g.Options = &Options{EvalModel: "gpt-4o-mini"} }, + "judge deployment": func(g *Eval) { + g.Evaluators = evalcore.EvaluatorList{{ + Name: "builtin.task_adherence", + InitializationParameters: map[string]any{"deployment_name": "gpt-4o-mini"}, + }} + }, + "options": func(g *Eval) { g.Options = &Options{EvaluationLevel: EvaluationLevelConversation} }, "dataset": func(g *Eval) { g.Dataset = "other" }, } for name, mutate := range cases { diff --git a/cli/azd/extensions/azure.ai.evaluations/tests/cli/evaluator_test.go b/cli/azd/extensions/azure.ai.evaluations/tests/cli/evaluator_test.go index 19112c2c95c..e6c97dffd9e 100644 --- a/cli/azd/extensions/azure.ai.evaluations/tests/cli/evaluator_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/tests/cli/evaluator_test.go @@ -84,7 +84,7 @@ func TestCLIInitNeedsAnAzdProject(t *testing.T) { r := requireFailure(t, runIn(t, dir, "init", "--target", "probe-agent", - "--judge-model", "gpt-4o-mini", + "--generation-model", "gpt-4o-mini", "--no-prompt")) require.Contains(t, r.Combined(), "azd init", diff --git a/cli/azd/extensions/azure.ai.evaluations/tests/cli/generate_test.go b/cli/azd/extensions/azure.ai.evaluations/tests/cli/generate_test.go index cc4eed3edac..71466a9b203 100644 --- a/cli/azd/extensions/azure.ai.evaluations/tests/cli/generate_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/tests/cli/generate_test.go @@ -59,7 +59,6 @@ func TestCLIGenerateRefusesBadFlagCombinations(t *testing.T) { args: []string{"evaluator", "generate", "e", "--target", "a", "--agent-instruction", "inline"}, want: "--generation-model", }} - for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { r := requireFailure(t, runIn(t, dir, tc.args...)) @@ -80,37 +79,48 @@ func TestCLIGenerateNamesTheArtifact(t *testing.T) { } } -// TestCLIGenerateNoPromptNamesWhatIsMissing is the CI case: with no target and -// nothing to prompt with, the process has to end saying which flag to pass. +// TestCLIGenerateNoPromptNamesWhatIsMissing is the CI case: with nothing to +// prompt with, the process has to end saying which flag to pass. +// +// The target is no longer among them — it is read from the eval's declaration — +// but the generation model has no other source, so it is the one input a bare +// directory cannot supply. func TestCLIGenerateNoPromptNamesWhatIsMissing(t *testing.T) { r := requireFailure(t, runIn(t, t.TempDir(), "dataset", "generate", "d", "--no-prompt")) - require.Contains(t, r.Combined(), "--target is required") - require.Contains(t, r.Combined(), "--no-prompt", - "the message must say why it could not be resolved") + require.Contains(t, r.Combined(), "--generation-model") + require.Contains(t, r.Combined(), "generationModel", + "the message must name both ways of supplying it") } -// TestCLIGenerateReadsTheSpec proves the config file is loaded and validated -// rather than only the flags. +// TestCLIGenerateReadsTheSpec proves the config file is loaded and that its +// entries are looked up by artifact name, rather than only the flags being read. // -// The strategy is the clearest evidence: `from-traces` is a value the spec -// accepts syntactically and the generation API cannot honour, so the refusal -// can only come from having parsed the file. +// The sample size is the clearest evidence: a value the service would reject is +// refused before any job is submitted, and it can only have come from the file. func TestCLIGenerateReadsTheSpec(t *testing.T) { dir := t.TempDir() - spec := filepath.Join(dir, "gen.yaml") + spec := filepath.Join(dir, "generate.yaml") require.NoError(t, os.WriteFile(spec, []byte(` -agent: - name: from-spec -generate: - dataset: - name: spec-dataset - strategy: from-traces +generationModel: gpt-4o-mini +dataset: + spec-dataset: + sampleSize: 5 `), 0o600)) + r := requireFailure(t, runIn(t, dir, "dataset", "generate", "spec-dataset", "--config", spec)) + require.Contains(t, r.Combined(), "between 15 and 1000", + "the spec's sampleSize must be validated before a job is billed") +} + +// A spec that cannot be parsed has to name itself, or the caller is left +// guessing which of several YAML files the command choked on. +func TestCLIGenerateReportsAnUnparseableSpec(t *testing.T) { + dir := t.TempDir() + spec := filepath.Join(dir, "generate.yaml") + require.NoError(t, os.WriteFile(spec, []byte("dataset: [not-a-mapping\n"), 0o600)) + r := requireFailure(t, runIn(t, dir, "dataset", "generate", "d", "--config", spec)) - require.Contains(t, r.Combined(), "from-traces") - require.Contains(t, r.Combined(), "agent.context.traces.window", - "the refusal must point at the field that does seed generation from traces") + require.Contains(t, r.Combined(), "generate.yaml") } // TestCLIGenerateFlagsAreScopedToTheirArtifact asserts the two commands do not From 0aa5ec5ca01a7be172d91fb1874b75a701923d53 Mon Sep 17 00:00:00 2001 From: mohessie Date: Mon, 3 Aug 2026 19:40:43 +0300 Subject: [PATCH 082/320] Refuse a wrong flag before a missing one `dataset generate d --agent-instruction-file ` answered "a model deployment is required", naming a flag the caller had not touched and saying nothing about the one they had. Splitting the offline resolution out of `prepareGeneration` had moved the instruction-file read behind the model check, and both checks are local, so the only thing deciding which error the user sees is the order they run in. The instruction file is read in `resolvePlan` now. An input the caller named and got wrong is a typo they can act on; a missing generation model has a documented default path and is the weaker complaint. Caught by the live CLI suite. Pinned offline now too, since the ordering is exactly the kind of thing a refactor moves without noticing and a 500-second live run is a slow way to find out. --- .../internal/cmd/generate_commands.go | 39 +++++++++++-------- .../internal/cmd/generate_plan_test.go | 15 +++++++ 2 files changed, 37 insertions(+), 17 deletions(-) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_commands.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_commands.go index 6a22d4fcbed..9e8c5a33dac 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_commands.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_commands.go @@ -52,22 +52,18 @@ func addGenerateFlags(cmd *cobra.Command, f *generateFlags) { cmd.Flags().StringVar(&f.endpoint, "project-endpoint", "", "Foundry project endpoint.") } -// prepareGeneration resolves everything both commands need before they diverge. +// prepareGeneration builds the client and settles the one input that needs it. // -// The model check happens here rather than at the service, because a generation -// job is billed against a deployment and a rejection partway through the -// command says less than a refusal at the flag that caused it. +// Everything decidable offline is already on the plan by this point, so a +// mistake in the flags has been reported without an authentication round trip. +// What is left is the generation instruction's last fallback: the agent's +// published instructions, which only the service can supply. func prepareGeneration( cmd *cobra.Command, f *generateFlags, plan generationPlan, declared genEntry, ) (*evalContext, generationPlan, error) { - instruction, err := resolveInstruction(f.instruction, f.instructionFile) - if err != nil { - return nil, plan, err - } - ctx := cmd.Context() ec, err := newEvalContext(ctx, f.endpoint) if err != nil { @@ -75,7 +71,7 @@ func prepareGeneration( } plan.Instruction, err = ec.resolveGenerationInstruction( - ctx, instruction, declared.instructions, f.configPath, plan.Agent, + ctx, plan.Instruction, declared.instructions, f.configPath, plan.Agent, cmd.OutOrStdout(), isJSON(cmd), ) if err != nil { @@ -91,20 +87,29 @@ func prepareGeneration( // generation spec, then what can be detected from the eval configuration. // Doing it before the client is built means a missing model or an out-of-range // sample count is refused without an authentication round trip. +// +// The instruction file is read here rather than later so that an input the +// caller named and got wrong is reported ahead of one they simply left out. func resolvePlan( f *generateFlags, cfg *project.GenerateConfig, name string, declared genEntry, ) (generationPlan, error) { + instruction, err := resolveInstruction(f.instruction, f.instructionFile) + if err != nil { + return generationPlan{}, err + } + plan := generationPlan{ - Name: name, - Agent: firstNonEmpty(f.target, declared.deriveFrom, evalTarget(f)), - Model: firstNonEmpty(f.model, cfg.GenerationModel), - BaseDir: filepath.Dir(f.configPath), - OutputDir: firstNonEmpty(f.outputDir, declared.outputDir), - SampleSize: declared.sampleSize, - TraceDays: declared.traceDays, + Name: name, + Agent: firstNonEmpty(f.target, declared.deriveFrom, evalTarget(f)), + Model: firstNonEmpty(f.model, cfg.GenerationModel), + Instruction: instruction, + BaseDir: filepath.Dir(f.configPath), + OutputDir: firstNonEmpty(f.outputDir, declared.outputDir), + SampleSize: declared.sampleSize, + TraceDays: declared.traceDays, } if plan.Model == "" { return plan, fmt.Errorf( diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_plan_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_plan_test.go index c018ddda8ac..9e4f9bd3e5d 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_plan_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_plan_test.go @@ -69,6 +69,21 @@ func TestResolvePlan_RequiresAGenerationModel(t *testing.T) { require.Contains(t, err.Error(), "generationModel") } +// An input the caller named and got wrong is reported ahead of one they simply +// left out. Both checks are local, so the only thing deciding which the user +// sees is the order they run in — and a missing instruction file is a typo the +// caller can act on, while the model has a documented default path. +func TestResolvePlan_ReportsABadExplicitInputFirst(t *testing.T) { + f := evalsDir(t, "", nil) + f.target = "shop-agent" + f.instructionFile = filepath.Join(t.TempDir(), "absent.md") + + _, err := resolvePlan(f, loadSpec(t, f), "d", genEntry{}) + require.Error(t, err) + require.Contains(t, err.Error(), "--agent-instruction-file", + "the flag the caller got wrong must win over the one they omitted") +} + // The spec is read per artifact name, so generating one artifact never picks up // the other's settings. func TestResolvePlan_ReadsTheNamedSpecEntry(t *testing.T) { From 9bf1505e4b1829eec957f77aef0495a8bcb32abc Mon Sep 17 00:00:00 2001 From: mohessie Date: Mon, 3 Aug 2026 20:02:04 +0300 Subject: [PATCH 083/320] Drive the hero scenarios through real azd `init` is the first command in Scenario 1 and the one that produces the local diff every later step depends on, and nothing was testing it end to end. It resolves the project and edits azure.yaml over azd's gRPC channel, so the CLI suite -- which runs the extension binary directly -- cannot reach it: with no azd on the other end the command refuses before it does anything. What covered it instead was a unit test calling the scaffold function, which cannot see the service entry azd writes, the detection that reads the project, or the terminal output the spec pins line for line. tests/hero runs the extension the way a user installs it. Scenario 1's output is compared whole rather than by keyword: every line is a promise the spec makes to a reader deciding whether to adopt this, and a keyword assertion would pass while their terminal said something else. It found one: with `--dataset prod-golden` -- a dataset that already exists -- `init` still closed with "Next: azd ai eval dataset generate prod-golden", sending the reader to submit a billed generation job for an artifact they had just supplied. Next steps are now built from what was actually scheduled, and when everything is already in place they point at `azd up` instead. Two properties are asserted that only this harness can see: that `init` makes no service calls, by running it behind a proxy pointing nowhere; and that a second `init` refuses without having edited the project, then under --force leaves exactly one eval service rather than a duplicate that would deploy the same eval twice. TestMain fails rather than skips, and rejects a stale install. A suite that silently reports on a binary other than the one under test is worse than no suite, and `azd x pack` leaves the local registry's checksum alone when the version has not changed, so a stale install is the easy mistake to make. The guard builds the working tree and compares help output, which is the cheapest fingerprint that actually moves with the code. Verified by making it fail. --- .../azure.ai.evaluations/internal/cmd/init.go | 31 +- .../internal/cmd/init_test.go | 29 ++ .../tests/hero/init_test.go | 348 ++++++++++++++++++ 3 files changed, 405 insertions(+), 3 deletions(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/tests/hero/init_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go index f9cbbbddd86..6e17b5f2806 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go @@ -128,9 +128,13 @@ func newInitCommand() *cobra.Command { fmt.Fprintf(out, " %-33s already declares service '%s'\n", rootConfigName, evalName) } - fmt.Fprintf(out, "\nNext: azd ai eval dataset generate %s\n", plan.datasetName) - if plan.rubricCount() > 0 { - fmt.Fprintf(out, " azd ai eval evaluator generate %s\n", rubricName) + // Only what was actually scheduled is offered. Suggesting + // `dataset generate` for a dataset the caller supplied sends them + // to submit a billed job for an artifact they already have. + next := plan.nextSteps() + fmt.Fprintf(out, "\nNext: %s\n", next[0]) + for _, step := range next[1:] { + fmt.Fprintf(out, " %s\n", step) } return nil }, @@ -331,6 +335,27 @@ func (s scaffold) rubricCount() int { return len(s.generate.Evaluator) } +// nextSteps are the commands to run after `init`, and only the ones that have +// something to do. +// +// A caller who supplied both a dataset and their evaluators has nothing left to +// generate, and pointing them at a generation command would submit a billed job +// for an artifact they already have. With everything in place the next step is +// to deploy it. +func (s scaffold) nextSteps() []string { + var steps []string + if s.generate != nil && len(s.generate.Dataset) > 0 { + steps = append(steps, "azd ai eval dataset generate "+s.datasetName) + } + if s.rubricCount() > 0 { + steps = append(steps, "azd ai eval evaluator generate "+s.rubricName) + } + if len(steps) == 0 { + steps = append(steps, "azd up", "azd ai eval run start") + } + return steps +} + // relativeToConfig rewrites a path given relative to the working directory so // it resolves from the directory holding the eval config. // diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_test.go index 40c92cf0855..b391dd908c0 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_test.go @@ -65,6 +65,35 @@ func TestScaffold_ExplicitEvaluatorsOptOutOfGeneration(t *testing.T) { require.Empty(t, plan.generate.Evaluator) } +// `init` closes by naming what to run next, and only what has something to do. +// Pointing a caller who supplied their own artifacts at a generation command +// would submit a billed job for something they already have. +func TestScaffold_NextStepsOfferOnlyWhatIsScheduled(t *testing.T) { + t.Run("nothing supplied", func(t *testing.T) { + plan := planScaffold("support-agent-smoke", "support-agent", "support-agent-quality", + "", nil, "m", project.DefaultEvalDir) + require.Equal(t, []string{ + "azd ai eval dataset generate support-agent-smoke", + "azd ai eval evaluator generate support-agent-quality", + }, plan.nextSteps()) + }) + + t.Run("dataset supplied", func(t *testing.T) { + plan := planScaffold("smoke", "support-agent", "support-agent-quality", + "prod-golden", nil, "m", project.DefaultEvalDir) + require.Equal(t, + []string{"azd ai eval evaluator generate support-agent-quality"}, + plan.nextSteps()) + }) + + t.Run("everything supplied", func(t *testing.T) { + plan := planScaffold("smoke", "support-agent", "support-agent-quality", + "prod-golden", []string{"builtin.task_adherence"}, "m", project.DefaultEvalDir) + require.Equal(t, []string{"azd up", "azd ai eval run start"}, plan.nextSteps(), + "with every artifact in place the next step is to deploy") + }) +} + func TestGenerateScaffold_RoundTripsAndValidates(t *testing.T) { dir := t.TempDir() genPath := filepath.Join(dir, "generate.yaml") diff --git a/cli/azd/extensions/azure.ai.evaluations/tests/hero/init_test.go b/cli/azd/extensions/azure.ai.evaluations/tests/hero/init_test.go new file mode 100644 index 00000000000..0618e49ccd8 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/tests/hero/init_test.go @@ -0,0 +1,348 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +//go:build hero + +// Package hero drives the hero scenarios through real azd, with the extension +// installed the way a user installs it. +// +// The CLI suite in ../cli runs the extension binary directly, which covers the +// command surface but cannot reach `init`: `init` resolves the project and +// edits azure.yaml over azd's gRPC channel, so without azd hosting the process +// there is nothing on the other end. That is not a detail — it is the first +// command in Scenario 1 and the one that produces the local diff every later +// step depends on, and until now the only thing asserting its output was a +// unit test calling the scaffold function directly. A unit test cannot see the +// service entry azd writes, the detection that reads the project, or the +// terminal output the spec pins line for line. +// +// azd x pack --rebuild +// azd extension install azure.ai.evaluations --source local +// go test -tags hero -v ./tests/hero/... +package hero + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +// reinstall is what to run when the installed extension is not this code. +// +// `azd x pack` rewrites the artifacts but leaves the checksum in the local +// registry alone when the version has not changed, so a plain reinstall then +// fails validation. Bumping the version in extension.yaml is the way through. +const reinstall = " azd x pack --rebuild\n" + + " azd extension uninstall azure.ai.evaluations\n" + + " azd extension install azure.ai.evaluations --source local\n" + +// TestMain refuses to run against an azd that cannot reach the extension, or +// that is hosting a different build of it. +// +// Skipping would be worse than failing here: these tests exist because nothing +// else covers the azd-hosted path, so a silent skip returns the suite to the +// state it was in before they were written. Running against a stale install is +// worse still — it reports on code that is not the code under test, which is +// the one outcome a test must never produce. +func TestMain(m *testing.M) { + if os.Getenv("AZURE_AI_EVAL_HERO") != "1" { + fmt.Fprintf(os.Stderr, + "set AZURE_AI_EVAL_HERO=1 to run the hero scenarios. They need azd "+ + "hosting this extension:\n%s", reinstall) + os.Exit(0) + } + + hosted, err := exec.Command("azd", "ai", "eval", "init", "--help").CombinedOutput() + if err != nil || !strings.Contains(string(hosted), "Scaffold evaluation config") { + fmt.Fprintf(os.Stderr, + "azd cannot reach the evaluations extension. Install it first:\n%s\n%s\n", + reinstall, hosted) + os.Exit(1) + } + + if err := requireCurrentInstall(string(hosted)); err != nil { + fmt.Fprintf(os.Stderr, "%v\n\n%s", err, reinstall) + os.Exit(1) + } + + os.Exit(m.Run()) +} + +// requireCurrentInstall compares the installed extension's help against this +// working tree's, so a stale install fails loudly instead of quietly reporting +// on the wrong binary. +// +// Help text is the cheapest available fingerprint that actually moves: it +// carries every command and flag, which is what these tests assert on, and it +// costs one build rather than a version stamp nobody remembers to bump. +func requireCurrentInstall(hosted string) error { + dir, err := os.MkdirTemp("", "azdeval-hero") + if err != nil { + return err + } + defer os.RemoveAll(dir) + + binary := filepath.Join(dir, "azdeval"+exeSuffix()) + build := exec.Command("go", "build", "-o", binary, ".") + build.Dir = "../.." + if out, err := build.CombinedOutput(); err != nil { + return fmt.Errorf("building this working tree to compare against: %v\n%s", err, out) + } + + local, err := exec.Command(binary, "init", "--help").CombinedOutput() + if err != nil { + return fmt.Errorf("reading this working tree's help: %w", err) + } + + if normalize(string(local)) != normalize(hosted) { + return fmt.Errorf( + "azd is hosting a different build of this extension.\n"+ + "installed:\n%s\nthis working tree:\n%s", + normalize(hosted), normalize(string(local))) + } + return nil +} + +func exeSuffix() string { + if os.PathSeparator == '\\' { + return ".exe" + } + return "" +} + +// project writes a minimal azd project for `init` to attach to. +// +// It declares the two services detection reads — the Foundry project and the +// agent — because what `init` writes into azure.yaml depends on which of them +// exist, and a project with neither would exercise only the fallback. +func project(t *testing.T, agent string) string { + t.Helper() + dir := t.TempDir() + body := fmt.Sprintf(`name: support-app +services: + ai-project: + host: azure.ai.project + %s: + host: azure.ai.agent +`, agent) + require.NoError(t, os.WriteFile(filepath.Join(dir, "azure.yaml"), []byte(body), 0o600)) + return dir +} + +// azdEval runs the extension through azd, in dir. +func azdEval(t *testing.T, dir string, args ...string) (string, int) { + t.Helper() + + cmd := exec.Command("azd", append([]string{"ai", "eval"}, args...)...) + cmd.Dir = dir + var out strings.Builder + cmd.Stdout = &out + cmd.Stderr = &out + + code := 0 + if err := cmd.Run(); err != nil { + exitErr, ok := err.(*exec.ExitError) + if !ok { + t.Fatalf("could not run azd ai eval %v: %v", args, err) + } + code = exitErr.ExitCode() + } + + // azd prints its own upgrade notice to stderr, which is not the command's + // output and would break an exact comparison. + text := dropUpgradeNotice(out.String()) + t.Logf("$ azd ai eval %s -> exit %d\n%s", strings.Join(args, " "), code, text) + return text, code +} + +// dropUpgradeNotice removes azd's "Update available" banner and everything +// after it, which azd appends regardless of the command. +func dropUpgradeNotice(s string) string { + if i := strings.Index(s, "Update available:"); i >= 0 { + s = s[:i] + } + return strings.TrimRight(s, " \r\n\t") +} + +// normalize makes terminal output comparable across platforms. +func normalize(s string) string { + return strings.ReplaceAll(dropUpgradeNotice(s), "\r\n", "\n") +} + +// TestHeroScenario1ColdStart is the first half of Scenario 1: the offline +// baseline `init` writes, asserted against the terminal block the spec shows. +// +// The output is compared whole rather than by keyword. Every line of it is a +// promise the spec makes to a reader deciding whether to adopt this — which +// files appear, what was detected, what to run next — and a keyword assertion +// would pass while the reader's terminal said something else. +func TestHeroScenario1ColdStart(t *testing.T) { + const ( + agent = "support-agent" + model = "gpt-5.6-luna" + ) + dir := project(t, agent) + + out, code := azdEval(t, dir, "init", "--target", agent, "--generation-model", model) + require.Zero(t, code, "init makes no service calls, so nothing can fail it here") + + want := `(✓) Done: Detected agent target: support-agent +(✓) Done: Detected model deployment: gpt-5.6-luna +(✓) Done: Planned evaluators: builtin.task_adherence, support-agent-quality (rubric) + +Created + evals/support-agent-smoke.yaml eval definition + evals/generate.yaml generation settings (15 samples, 1 rubric) + azure.yaml added service 'support-agent-smoke' + +Next: azd ai eval dataset generate support-agent-smoke + azd ai eval evaluator generate support-agent-quality` + + require.Equal(t, want, normalize(out)) +} + +// `init` is offline, and being offline is the property that makes its output a +// reviewable local diff. A service call here would also make the command fail +// for a user who has not authenticated yet, which is exactly when they run it. +func TestHeroInitMakesNoServiceCalls(t *testing.T) { + dir := project(t, "support-agent") + + cmd := exec.Command("azd", "ai", "eval", "init", + "--target", "support-agent", "--generation-model", "m") + cmd.Dir = dir + // A proxy pointing nowhere fails any outbound request, so a command that + // stays offline is unaffected and one that does not cannot be mistaken for + // working. + cmd.Env = append(os.Environ(), + "HTTPS_PROXY=http://127.0.0.1:9", + "HTTP_PROXY=http://127.0.0.1:9", + "NO_PROXY=", + ) + + out, err := cmd.CombinedOutput() + require.NoError(t, err, "init must not need the network:\n%s", out) +} + +// The eval service has to be declared in azure.yaml before azd will act on it. +// Printing the block and leaving the edit to the reader was enough, once, to +// make the documented flow stop working between `init` and `azd up`. +func TestHeroInitWiresTheServiceIntoTheProject(t *testing.T) { + dir := project(t, "support-agent") + + _, code := azdEval(t, dir, "init", "--target", "support-agent", "--generation-model", "m") + require.Zero(t, code) + + root, err := os.ReadFile(filepath.Join(dir, "azure.yaml")) + require.NoError(t, err) + text := string(root) + + require.Contains(t, text, "support-agent-smoke:", + "the service key is the eval's name") + require.Contains(t, text, "host: azure.ai.eval") + require.Contains(t, text, "$ref: ./evals/support-agent-smoke.yaml") + + // azd owns the edit, so everything the project already declared survives it. + require.Contains(t, text, "name: support-app") + require.Contains(t, text, "host: azure.ai.project") + require.Contains(t, text, "host: azure.ai.agent") + + // The eval reads both, so azd has to deploy both first. + require.Regexp(t, `(?s)support-agent-smoke:.*uses:.*ai-project.*support-agent`, text) +} + +// Running `init` twice must not deploy the same eval twice. The service key is +// the eval's name, so the second run recognises its own work. +func TestHeroInitIsIdempotent(t *testing.T) { + dir := project(t, "support-agent") + args := []string{"init", "--target", "support-agent", "--generation-model", "m"} + + _, code := azdEval(t, dir, args...) + require.Zero(t, code) + first, err := os.ReadFile(filepath.Join(dir, "azure.yaml")) + require.NoError(t, err) + + out, code := azdEval(t, dir, args...) + require.NotZero(t, code, "the scaffold already exists, so a second run must refuse") + require.Contains(t, out, "--force", "the refusal has to say how to proceed") + + second, err := os.ReadFile(filepath.Join(dir, "azure.yaml")) + require.NoError(t, err) + require.Equal(t, string(first), string(second), + "a refused init must not have edited the project") + + // With --force the files are rewritten, and the service is still declared + // exactly once. + out, code = azdEval(t, dir, append(args, "--force")...) + require.Zero(t, code, out) + + third, err := os.ReadFile(filepath.Join(dir, "azure.yaml")) + require.NoError(t, err) + require.Equal(t, 1, strings.Count(string(third), "host: azure.ai.eval"), + "a second eval service would deploy the same eval twice") + require.Contains(t, normalize(out), "already declares service 'support-agent-smoke'") +} + +// Evals attach to a project; they do not create one. Naming the command that +// makes a project is more use than a transport error from the gRPC channel +// that was not there. +func TestHeroInitNeedsAnAzdProject(t *testing.T) { + dir := t.TempDir() + + out, code := azdEval(t, dir, "init", "--target", "support-agent", "--no-prompt") + require.NotZero(t, code) + require.Contains(t, out, "azd init") + require.NotContains(t, strings.ToLower(out), "grpc", + "a missing project must not surface as a transport error") + + entries, err := os.ReadDir(dir) + require.NoError(t, err) + require.Empty(t, entries, "a refused init must leave nothing behind") +} + +// Passing --evaluator replaces the defaults, which is how a caller opts out of +// rubric generation — so the "next" steps must stop offering to generate one. +func TestHeroInitExplicitEvaluatorsOptOutOfGeneration(t *testing.T) { + dir := project(t, "support-agent") + + out, code := azdEval(t, dir, "init", + "--target", "support-agent", "--generation-model", "m", + "--evaluator", "builtin.task_adherence") + require.Zero(t, code, out) + + text := normalize(out) + require.Contains(t, text, "Planned evaluators: builtin.task_adherence") + require.NotContains(t, text, "(rubric)") + require.NotContains(t, text, "evaluator generate", + "nothing was scheduled to be generated, so nothing should be suggested") + require.Contains(t, text, "generation settings (15 samples, 0 rubric)") +} + +// A supplied dataset is not generated either, so `init` has nothing left to +// suggest and must not send the reader to a command that would submit a job +// for an artifact they already have. +func TestHeroInitSuppliedDatasetIsNotGenerated(t *testing.T) { + dir := project(t, "support-agent") + + out, code := azdEval(t, dir, "init", + "--target", "support-agent", "--generation-model", "m", + "--dataset", "prod-golden", + "--evaluator", "builtin.task_adherence") + require.Zero(t, code, out) + require.NotContains(t, normalize(out), "dataset generate") + + body, err := os.ReadFile(filepath.Join(dir, "evals", "generate.yaml")) + require.NoError(t, err) + require.NotContains(t, string(body), "dataset:", + "nothing is left to generate, so the spec declares nothing") + + eval, err := os.ReadFile(filepath.Join(dir, "evals", "support-agent-smoke.yaml")) + require.NoError(t, err) + require.Contains(t, string(eval), "name: prod-golden") + require.NotContains(t, string(eval), "source:", + "a registered dataset has nothing to upload") +} From e24dc7447989ee7a0da48628c74fe10125d1b5e3 Mon Sep 17 00:00:00 2001 From: mohessie Date: Mon, 3 Aug 2026 20:48:39 +0300 Subject: [PATCH 084/320] Pin the command surface, and fix the flag it caught The flag that writes results to a file was `--out-file`. The spec, its Scenario 4, and `azd ai skill download` all say `--output-file`. It took reading the two documents side by side to notice, which is the wrong way to find something a user types from memory. The surface is a contract -- with the spec, and with the sibling Foundry extensions whose vocabulary this shares -- and nothing was checking it. So the tree and the flags are pinned: - Every command in the tree, so one that is renamed, dropped or quietly added has to be acknowledged against the spec's command table. - `init`'s flags whole, because they are a table in the spec: an extra flag there is a promise the spec does not make, a missing one is a promise it does. - The spellings this extension must not invent, including the ones the spec explicitly rules out (`--judge-model`) and the ones belonging to M2 (`--from-traces`, `--folder`, `--cron`), so a deferred surface cannot reappear by accident. - That `init` takes no `--project-endpoint` and every command that does reach the service takes one. Verified by renaming the flag back and watching both guards fail. --- .../internal/cmd/run_output.go | 4 +- .../internal/cmd/surface_test.go | 196 ++++++++++++++++++ .../tests/cli/run_output_test.go | 6 +- 3 files changed, 201 insertions(+), 5 deletions(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/surface_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output.go index 0f7545fffd6..49a72a2efb7 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output.go @@ -110,7 +110,7 @@ func newRunOutputListCommand() *cobra.Command { } cmd.Flags().BoolVar(&failedOnly, "failed-only", false, "Show only the rows that failed.") - cmd.Flags().StringVarP(&outFile, "out-file", "O", "", "Write JSON results to this path.") + cmd.Flags().StringVar(&outFile, "output-file", "", "Write JSON results to this path.") addEvalFlags(cmd, &groupName) cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") return cmd @@ -225,7 +225,7 @@ func newRunOutputExportCommand() *cobra.Command { } cmd.Flags().StringVar(&format, "format", "json", "Output format: json or csv.") - cmd.Flags().StringVarP(&outFile, "out-file", "O", "", "Write to this path instead of stdout.") + cmd.Flags().StringVar(&outFile, "output-file", "", "Write to this path instead of stdout.") addEvalFlags(cmd, &groupName) cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") return cmd diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/surface_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/surface_test.go new file mode 100644 index 00000000000..516ee0d2520 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/surface_test.go @@ -0,0 +1,196 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "strings" + "testing" + + "github.com/spf13/cobra" + "github.com/spf13/pflag" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// The command surface is a contract with the spec and with the sibling Foundry +// extensions, and it is the part of this tool users type from memory. Nothing +// was checking it: the flag that writes results to a file was `--out-file` +// while the spec, Scenario 4, and `azd ai skill download` all say +// `--output-file`, and it took reading the two documents side by side to see. +// +// These tests walk the built tree, so a command or flag that is renamed, +// dropped, or quietly added has to be acknowledged here. + +// walk visits every command in the tree, skipping the ones azd contributes. +func walk(t *testing.T, cmd *cobra.Command, path []string, visit func(string, *cobra.Command)) { + t.Helper() + for _, child := range cmd.Commands() { + name := strings.Fields(child.Use)[0] + switch name { + case "help", "completion", "listen", "metadata": + continue + } + full := append(append([]string{}, path...), name) + visit(strings.Join(full, " "), child) + walk(t, child, full, visit) + } +} + +// commandTree is every command the extension exposes, and is the surface the +// spec's command table describes. +func TestCommandTreeMatchesTheSpec(t *testing.T) { + want := []string{ + "dataset", + "dataset create", + "dataset delete", + "dataset generate", + "dataset list", + "dataset show", + "dataset update", + "dataset versions", + "dataset versions list", + "delete", + "evaluator", + "evaluator create", + "evaluator delete", + "evaluator generate", + "evaluator list", + "evaluator show", + "evaluator update", + "evaluator versions", + "evaluator versions list", + "init", + "job", + "job cancel", + "job list", + "job show", + "list", + "run", + "run cancel", + "run delete", + "run list", + "run output", + "run output export", + "run output list", + "run output show", + "run show", + "run start", + "show", + } + + var got []string + walk(t, NewRootCommand(), nil, func(path string, _ *cobra.Command) { + got = append(got, path) + }) + + assert.ElementsMatch(t, want, got, + "the command tree changed; update the spec's command table with it") +} + +// Flag names are shared vocabulary across the Foundry extensions. A command +// that invents its own spelling for something the others already name is the +// kind of difference nobody notices until a user types the one they learned +// somewhere else. +func TestFlagVocabularyIsShared(t *testing.T) { + // Meaning → the one spelling for it, from the spec's vocabulary table. + // A command that means one of these must use exactly this name, and the + // near-misses are listed so a rename back is caught rather than accepted. + forbidden := map[string]string{ + "--out-file": "--output-file", + "--out-dir": "--output-dir", + "--file": "--from-file", + "--rubric": "--from-file", + "--judge-model": "--generation-model, declared per evaluator instead", + "--from-traces": "deferred to M2", + "--response-id": "deferred to M2", + "--no-target": "deferred to M2", + "--out": "--output-file", + "--dir": "--output-dir", + "--baseline": "deferred to M2", + "--cron": "deferred to M2", + "--folder": "deferred to M2", + "--init-params": "deferred to M2", + "--data-schema": "deferred to M2", + "--metrics": "deferred to M2", + "--trace-window": "deferred to M2", + "--max-traces": "deferred to M2", + "--max-turns": "deferred to M2", + } + + walk(t, NewRootCommand(), nil, func(path string, cmd *cobra.Command) { + cmd.LocalFlags().VisitAll(func(f *pflag.Flag) { + if want, bad := forbidden["--"+f.Name]; bad { + t.Errorf("%s declares --%s; use %s", path, f.Name, want) + } + }) + }) +} + +// The two commands that write a file have to agree on what that flag is +// called, and it has to be the name the sibling extensions use. +func TestOutputFileFlagIsSpelledTheSharedWay(t *testing.T) { + for _, path := range []string{"run output list", "run output export"} { + cmd := find(t, path) + require.NotNil(t, cmd.Flags().Lookup("output-file"), + "%s must write to --output-file, the name `azd ai skill download` uses", path) + assert.Nil(t, cmd.Flags().Lookup("out-file"), + "%s must not keep the old spelling alongside the shared one", path) + } +} + +// `init` is the one command with a documented flag table, so it is pinned +// whole: an extra flag there is a promise the spec does not make, and a +// missing one is a promise it does. +func TestInitFlagsMatchTheSpec(t *testing.T) { + cmd := find(t, "init") + + var got []string + cmd.LocalFlags().VisitAll(func(f *pflag.Flag) { + if f.Name != "help" { + got = append(got, "--"+f.Name) + } + }) + + assert.ElementsMatch(t, []string{ + "--name", "--target", "--dataset", "--evaluator", + "--generation-model", "--output-dir", "--force", + }, got, "init's flags are a table in the spec; change both together") +} + +// `init` makes no service calls, so it must not offer the flag that says where +// to make them. +func TestInitTakesNoProjectEndpoint(t *testing.T) { + assert.Nil(t, find(t, "init").Flags().Lookup("project-endpoint"), + "init is offline; a project endpoint would imply otherwise") +} + +// Every command that does reach the service accepts it, because the shared +// Foundry resolver is how a project is named without an azd environment. +func TestServiceCommandsTakeProjectEndpoint(t *testing.T) { + groups := map[string]bool{ + "dataset": true, "evaluator": true, "run": true, + "job": true, "run output": true, + } + + walk(t, NewRootCommand(), nil, func(path string, cmd *cobra.Command) { + if cmd.RunE == nil || path == "init" { + return + } + if groups[path] { + return + } + assert.NotNil(t, cmd.Flags().Lookup("project-endpoint"), + "%s reaches the service, so it must accept --project-endpoint", path) + }) +} + +// find resolves a command path, failing the test when it does not exist. +func find(t *testing.T, path string) *cobra.Command { + t.Helper() + cmd, _, err := NewRootCommand().Find(strings.Fields(path)) + require.NoError(t, err, "no such command: %s", path) + require.Equal(t, strings.Fields(path)[len(strings.Fields(path))-1], + strings.Fields(cmd.Use)[0], "resolved the wrong command for %s", path) + return cmd +} diff --git a/cli/azd/extensions/azure.ai.evaluations/tests/cli/run_output_test.go b/cli/azd/extensions/azure.ai.evaluations/tests/cli/run_output_test.go index 9b9d71e9c91..4c688dd9976 100644 --- a/cli/azd/extensions/azure.ai.evaluations/tests/cli/run_output_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/tests/cli/run_output_test.go @@ -179,14 +179,14 @@ func TestCLIResultsExport(t *testing.T) { require.Equal(t, f.EvaluatorName, rows[1][2]) }) - t.Run("out-file writes the path instead of stdout", func(t *testing.T) { + t.Run("output-file writes the path instead of stdout", func(t *testing.T) { dir := t.TempDir() path := filepath.Join(dir, "results.csv") r := requireSuccess(t, runIn(t, dir, "run", "output", "export", f.FirstRunID, - "--eval-id", f.EvalID, "--format", "csv", "-O", path)) + "--eval-id", f.EvalID, "--format", "csv", "--output-file", path)) require.Empty(t, strings.TrimSpace(r.Stdout), - "-O redirects the payload; leaving it on stdout too would double it") + "--output-file redirects the payload; leaving it on stdout too would double it") body, err := os.ReadFile(path) require.NoError(t, err) From 8dd23d0c3501a4091592b1ac5f567a9ec09d59f5 Mon Sep 17 00:00:00 2001 From: mohessie Date: Mon, 3 Aug 2026 21:02:57 +0300 Subject: [PATCH 085/320] Stop pointing users at a command that was renamed away `run start --no-wait` closed with "Check progress with: azd ai eval results show --run-id ". `results` became `run output` when the surface was rebuilt, so the one instruction printed at the exact moment a user needs it was the one thing guaranteed to fail. The no-previous-run error had the same problem, sending the reader to a bare `azd ai eval run` that is now a group. Nothing catches this on its own: the string compiles, the command that prints it succeeds, and only somebody who follows the advice finds out. So every `azd ai eval ...` in a non-comment line is now resolved against the real command tree, with trailing prose trimmed so "run start and summarize" is read as the command it begins with. Verified by restoring the original message and watching the test name it. Also fixes an assertion that the live suite caught being wrong rather than the code: `--failed-only` counted the bare substring "FAIL" against `ResultCounts.Failed`. Two things were conflated. The per-criterion table's FAILED column header contains FAIL, and one rendered line is one evaluator's verdict on one sample -- a sample failing two evaluators is two lines, while `ResultCounts.Failed` counts samples. It matches verdict cells on a word boundary against the failing results in the payload now, so it holds whatever the judge decides. --- .../azure.ai.evaluations/internal/cmd/run.go | 9 +-- .../internal/cmd/surface_test.go | 58 +++++++++++++++++++ .../tests/cli/run_output_test.go | 25 ++++++-- 3 files changed, 81 insertions(+), 11 deletions(-) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go index af31fb489c1..e93f313a0d7 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go @@ -32,9 +32,6 @@ var terminalRunStates = map[string]bool{ "error": true, } -// newRunCommand builds the composite `azd ai eval run` and attaches the atomic -// run operations, including `run start` which the spec lists as the atomic form -// of this same command. // newRunCommand builds the run group. // // `run` is a group, not an executable verb: once `run output` exists, a bare @@ -167,7 +164,7 @@ func buildRunCommand(use, short string) *cobra.Command { return emitJSON(out, run) } fmt.Fprintf(out, "Started run %s (status: %s)\n", run.ID, run.Status) - fmt.Fprintf(out, "Check progress with: azd ai eval results show %s --run-id %s\n", evalID, run.ID) + fmt.Fprintf(out, "Reattach with: azd ai eval run show %s --eval-id %s\n", run.ID, evalID) return nil } @@ -363,8 +360,8 @@ func (ec *evalContext) reuseDataSourceFromLastRun( return nil, fmt.Errorf( "eval %s has no previous run to repeat, so there is no target or dataset "+ "to reuse.\n"+ - " Run it from the config once with `azd ai eval run`, or pass a config that "+ - "declares the eval", + " Run it from the config once with `azd ai eval run start`, or name an "+ + "eval that declares one with `--eval`", evalID) } return list.Data[0].DataSource, nil diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/surface_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/surface_test.go index 516ee0d2520..551dd45359c 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/surface_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/surface_test.go @@ -4,6 +4,10 @@ package cmd import ( + "io/fs" + "os" + "path/filepath" + "regexp" "strings" "testing" @@ -194,3 +198,57 @@ func find(t *testing.T, path string) *cobra.Command { strings.Fields(cmd.Use)[0], "resolved the wrong command for %s", path) return cmd } + +// Messages that tell a user what to run next have to name a command that +// exists. +// +// Rebuilding the surface left `run start --no-wait` closing with "Check +// progress with: azd ai eval results show", a command that had been renamed +// out of existence — so the one instruction printed at the moment a user needs +// it was the one thing guaranteed to fail. Nothing catches that: the string +// compiles, the command that prints it succeeds, and only someone following +// the advice finds out. +func TestSuggestedCommandsExist(t *testing.T) { + root := "../.." + pattern := regexp.MustCompile("azd ai eval ([a-z][a-z0-9-]*(?: [a-z][a-z0-9-]*)*)") + + err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() || !strings.HasSuffix(path, ".go") || strings.HasSuffix(path, "_test.go") { + return nil + } + + body, err := os.ReadFile(path) + if err != nil { + return err + } + + for _, line := range strings.Split(string(body), "\n") { + // Comments explain the surface; only what reaches a terminal has + // to resolve. + if strings.HasPrefix(strings.TrimSpace(line), "//") { + continue + } + for _, m := range pattern.FindAllStringSubmatch(line, -1) { + words := strings.Fields(m[1]) + // Trim trailing prose: "run start" is a command, "run start + // and summarize" is a sentence that begins with one. + for len(words) > 0 { + if _, _, err := NewRootCommand().Find(words); err == nil { + resolved, _, _ := NewRootCommand().Find(words) + if strings.Fields(resolved.Use)[0] == words[len(words)-1] { + break + } + } + words = words[:len(words)-1] + } + assert.NotEmpty(t, words, + "%s suggests `azd ai eval %s`, which is not a command", path, m[1]) + } + } + return nil + }) + require.NoError(t, err) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/tests/cli/run_output_test.go b/cli/azd/extensions/azure.ai.evaluations/tests/cli/run_output_test.go index 4c688dd9976..84358f2c81e 100644 --- a/cli/azd/extensions/azure.ai.evaluations/tests/cli/run_output_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/tests/cli/run_output_test.go @@ -9,6 +9,7 @@ import ( "encoding/csv" "os" "path/filepath" + "regexp" "strings" "testing" @@ -115,12 +116,23 @@ func TestCLIResultsShowFailedOnly(t *testing.T) { f := sharedEval(t) payload := resultsFor(t, f.EvalID, f.FirstRunID) - failed := payload.Run.ResultCounts.Failed + + // One rendered row is one evaluator's verdict on one sample, so the count + // to expect is failing *results*, not failing rows: a sample that fails two + // evaluators is two lines. `ResultCounts.Failed` answers the other question. + failing := 0 + for _, item := range payload.OutputItems { + for _, r := range item.Results { + if !r.Passed { + failing++ + } + } + } r := requireSuccess(t, run(t, "run", "output", "list", f.FirstRunID, "--eval-id", f.EvalID, "--failed-only")) - if failed == 0 { + if failing == 0 { // Saying so is not the same as printing an empty table. require.Contains(t, r.Stdout, "No failing rows.") return @@ -128,9 +140,12 @@ func TestCLIResultsShowFailedOnly(t *testing.T) { require.NotContains(t, r.Stdout, " pass ", "--failed-only must drop the rows that passed") - require.Contains(t, r.Stdout, "FAIL") - require.Equal(t, failed, strings.Count(r.Stdout, "FAIL"), - "every failing row must appear exactly once") + + // Matched on a word boundary so the per-criterion table's FAILED column + // header is not counted as a verdict. + verdicts := regexp.MustCompile(`\bFAIL\b`).FindAllString(r.Stdout, -1) + require.Equal(t, failing, len(verdicts), + "every failing verdict must appear exactly once:\n%s", r.Stdout) } // resultsFor reads a run's results as JSON, which several tests need before From c176044b81a9174585a94e2beabbee00c3602446 Mon Sep 17 00:00:00 2001 From: mohessie Date: Mon, 3 Aug 2026 23:08:36 +0300 Subject: [PATCH 086/320] Note the platforms nothing has run on yet build.sh cross-compiles six platforms. Every build and test in this extension's history has been windows/amd64, so five of them ship unexecuted. The test suites carry the same assumption rather than guarding against it: both harnesses derive the .exe suffix from the path separator, and the hero suite shells out to \zd\ and to a proxy address, none of which has been exercised off Windows. Recorded next to the note that already says what CI does and does not cover, since that is what a release reads. --- cli/azd/extensions/azure.ai.evaluations/ci-test.ps1 | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/cli/azd/extensions/azure.ai.evaluations/ci-test.ps1 b/cli/azd/extensions/azure.ai.evaluations/ci-test.ps1 index 314095e75b3..8be21c6bbf4 100644 --- a/cli/azd/extensions/azure.ai.evaluations/ci-test.ps1 +++ b/cli/azd/extensions/azure.ai.evaluations/ci-test.ps1 @@ -8,6 +8,11 @@ # The live integration tests are excluded: they carry the `live` build tag, so # an untagged run does not compile them, and they additionally require # AZURE_AI_EVAL_E2E_LIVE and a project endpoint. +# +# TODO before the first release: build.sh ships six platforms and only +# windows/amd64 has ever been executed. The live and hero suites assume it too +# -- they derive the .exe suffix from the path separator and shell out to `azd` +# -- so run both on linux and darwin before publishing. $gopath = go env GOPATH $gotestsumBinary = "gotestsum" From 35aa6999869e84295814a61cdefba828a9cc2e81 Mon Sep 17 00:00:00 2001 From: mohessie Date: Mon, 3 Aug 2026 23:17:21 +0300 Subject: [PATCH 087/320] Correct what CI already covers The previous note claimed only windows/amd64 had ever been executed. That was true of my desk, not of CI: release-ext-azure-ai-evaluations.yml has a pr: trigger on this directory, and every BuildMatrix job -- windows, linux and darwin amd64 -- runs this script. The untagged tests are covered on all three. What is genuinely uncovered is narrower and worse: ci-test.ps1 runs go test ./... untagged, so the live and hero suites are never compiled by any pipeline on any platform. Both have only ever run on Windows, by hand. Darwin needs no manual pass -- CI has it. Linux does, for the tagged suites only. --- cli/azd/extensions/azure.ai.evaluations/ci-test.ps1 | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/cli/azd/extensions/azure.ai.evaluations/ci-test.ps1 b/cli/azd/extensions/azure.ai.evaluations/ci-test.ps1 index 8be21c6bbf4..1050a817798 100644 --- a/cli/azd/extensions/azure.ai.evaluations/ci-test.ps1 +++ b/cli/azd/extensions/azure.ai.evaluations/ci-test.ps1 @@ -9,10 +9,11 @@ # an untagged run does not compile them, and they additionally require # AZURE_AI_EVAL_E2E_LIVE and a project endpoint. # -# TODO before the first release: build.sh ships six platforms and only -# windows/amd64 has ever been executed. The live and hero suites assume it too -# -- they derive the .exe suffix from the path separator and shell out to `azd` -# -- so run both on linux and darwin before publishing. +# TODO before the first release: PR CI runs this script on windows, linux and +# darwin amd64, so the untagged tests are covered on all three. The live and +# hero suites are not -- being tagged, no pipeline has ever compiled them, and +# both have only ever run on Windows by hand. Run them once on linux, where +# they assume a path separator and shell out to `azd` and to a proxy address. $gopath = go env GOPATH $gotestsumBinary = "gotestsum" From 5feb5c5a22678a72b693c7f97636bf9d1aec2b86 Mon Sep 17 00:00:00 2001 From: mohessie Date: Mon, 3 Aug 2026 23:29:00 +0300 Subject: [PATCH 088/320] Type-check the tagged suites in CI ci-test.ps1 runs go test ./... untagged, so nothing in any pipeline compiled the live or hero suites. A change that broke one reached main silently -- the surface tests added earlier would not catch it either, because they are untagged too. go vet -tags live,hero needs no credentials and no endpoint, so it costs a few seconds and runs on every platform the tests already do. It runs after the tests so a genuine test failure still reports first. Verified both ways: breaking a file under either tag fails the script with exit 1 and suppresses the success line; the clean tree exits 0. --- .../azure.ai.evaluations/ci-test.ps1 | 22 +++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/cli/azd/extensions/azure.ai.evaluations/ci-test.ps1 b/cli/azd/extensions/azure.ai.evaluations/ci-test.ps1 index 1050a817798..6175585d318 100644 --- a/cli/azd/extensions/azure.ai.evaluations/ci-test.ps1 +++ b/cli/azd/extensions/azure.ai.evaluations/ci-test.ps1 @@ -7,13 +7,14 @@ # # The live integration tests are excluded: they carry the `live` build tag, so # an untagged run does not compile them, and they additionally require -# AZURE_AI_EVAL_E2E_LIVE and a project endpoint. +# AZURE_AI_EVAL_E2E_LIVE and a project endpoint. They are still type-checked +# below, so a change that breaks them cannot reach main unnoticed. # # TODO before the first release: PR CI runs this script on windows, linux and # darwin amd64, so the untagged tests are covered on all three. The live and -# hero suites are not -- being tagged, no pipeline has ever compiled them, and -# both have only ever run on Windows by hand. Run them once on linux, where -# they assume a path separator and shell out to `azd` and to a proxy address. +# hero suites are only type-checked, never executed, and both have only ever +# run on Windows by hand. Run them once on linux, where they assume a path +# separator and shell out to `azd` and to a proxy address. $gopath = go env GOPATH $gotestsumBinary = "gotestsum" @@ -37,6 +38,19 @@ if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } +# The tagged suites are never run here, so without this nothing compiles them +# and a change that breaks one reaches main silently. Type-checking needs no +# credentials, so it costs a few seconds and runs everywhere the tests do. +Write-Host "" +Write-Host "Type-checking the live and hero suites..." +go vet -tags live,hero ./... + +if ($LASTEXITCODE -ne 0) { + Write-Host "" + Write-Host "The tagged test suites do not compile: $LASTEXITCODE" -ForegroundColor Red + exit $LASTEXITCODE +} + Write-Host "" Write-Host "All tests passed!" -ForegroundColor Green exit 0 From 52cedc3d45a1b8f49b8fe3d65e6c552814929138 Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 4 Aug 2026 01:16:49 +0300 Subject: [PATCH 089/320] Stop an update from overwriting the version it is updating For a few seconds after a publish the service can answer the next one with the version it just assigned, writing over that version's definition instead of adding one. So \evaluator create\ followed by \evaluator update\ - which is what a first authoring session looks like - reported success and left a single version holding the second rubric, with every eval bound to the first scoring against something nobody chose. It cannot be waited out by watching for state, because there is none to watch: a publish 258ms after the first came back as version 1 and one six seconds later advanced to 2, while the version listing was already answering with 1 - and immediately after a create that same listing 404s, so a guard trusting it would stand down in exactly the case it exists for. What is reliable is what the caller has already read. Both call sites read the evaluator to decide between creating and updating, so that document is passed into the publish: it says which version exists and when it was written. The publish waits until that version has had a few seconds, then checks that what came back is a new one, reissuing the same body if it is not. A collision that happens anyway cannot be undone - the request that collided has already written - so the recheck is bounded and ends in an error naming the version that was replaced, rather than in a reported success. --- .../internal/cmd/evaluator.go | 10 +- .../cmd/evaluator_version_live_test.go | 134 +++++++++++++ .../internal/cmd/reconciler.go | 6 +- .../internal/pkg/eval_api/evaluators.go | 10 + .../internal/pkg/eval_api/operations.go | 116 ++++++++++- .../pkg/eval_api/publish_version_test.go | 184 ++++++++++++++++++ 6 files changed, 455 insertions(+), 5 deletions(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator_version_live_test.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/publish_version_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator.go index 2407f4434ee..c5f86a19d15 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator.go @@ -86,7 +86,7 @@ func newEvaluatorWriteCommand(verb, short string) *cobra.Command { // lags a publish by up to a second and a half, so an update // issued straight after a create would be told the evaluator it // just made does not exist. - _, readErr := ec.evalClient.GetEvaluatorRaw( + existing, readErr := ec.evalClient.GetEvaluatorRaw( ctx, name, "", ProjectEndpointAPIVersion, ) if readErr != nil && !eval_api.IsNotFound(readErr) { @@ -96,8 +96,14 @@ func newEvaluatorWriteCommand(verb, short string) *cobra.Command { return err } + // What that read saw is what keeps the publish from being + // answered with the same version and replacing it. + if readErr != nil { + existing = nil + } + created, err := ec.evalClient.CreateEvaluatorVersion( - ctx, name, body, ProjectEndpointAPIVersion, + ctx, name, body, existing, ProjectEndpointAPIVersion, ) if err != nil { return fmt.Errorf("registering evaluator %q: %w", name, err) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator_version_live_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator_version_live_test.go new file mode 100644 index 00000000000..fabc4f222b6 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator_version_live_test.go @@ -0,0 +1,134 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +//go:build live + +// Evaluator versions are the unit an eval binds to, and the service assigns +// them. This proves the extension never hands back a version it has quietly +// overwritten. + +package cmd + +import ( + "context" + "encoding/json" + "fmt" + "testing" + "time" + + "azureaieval/internal/pkg/eval_api" + + "github.com/stretchr/testify/require" +) + +// TestLiveEvaluatorUpdateAlwaysPublishesANewVersion covers the shape of a +// first authoring session: create a rubric, look at it, change one weight, +// update. +// +// For a few seconds after a publish the service can answer the next one with +// the version it just assigned, writing over it rather than adding one. +// Nothing observable marks the end of that race — the version listing lags a +// publish as well, answering 404 immediately after a create — so the defence +// is the document the caller already read: it says which version exists and +// when it was written. +// +// Without it, `evaluator update` run straight after `evaluator create` reports +// success, leaves a single version holding the second rubric, and every eval +// bound to the first scores against a rubric nobody chose. +func TestLiveEvaluatorUpdateAlwaysPublishesANewVersion(t *testing.T) { + client, _ := liveEvalClient(t) + ctx := context.Background() + + name := fmt.Sprintf("azdlive-version-%d", time.Now().UnixNano()) + + rubric := func(weight int) json.RawMessage { + body, err := normalizeRubricBody(name, []byte(fmt.Sprintf( + `{"dimensions":[{"id":"tone","weight":%d,"description":"polite"}]}`, weight))) + require.NoError(t, err) + return body + } + + first, err := client.CreateEvaluatorVersion(ctx, name, rubric(1), nil, ProjectEndpointAPIVersion) + require.NoError(t, err) + require.NotEmpty(t, first.Version) + t.Cleanup(func() { + for _, v := range []string{first.Version, "1", "2"} { + _ = client.DeleteEvaluatorVersion( + context.Background(), name, v, ProjectEndpointAPIVersion) + } + }) + + // Deliberately immediate, and passing what the caller holds rather than + // re-reading: this is the window the guard exists for, and a test that + // waited first would pass with the guard removed. + previous, err := json.Marshal(first) + require.NoError(t, err) + + started := time.Now() + second, err := client.CreateEvaluatorVersion( + ctx, name, rubric(2), previous, ProjectEndpointAPIVersion) + require.NoError(t, err) + require.NotEqual(t, first.Version, second.Version, + "an update issued inside the race must still publish a new version") + t.Logf("the second version was assigned after %s", time.Since(started).Round(time.Millisecond)) + + // The new version holds the new rubric, and both versions are readable. + // The earlier one is not asserted on: if the service does collide, the + // attempt that collided has already written the new definition over it, + // and no amount of care on this side can undo that. + require.Equal(t, 2, liveRubricWeight(t, client, name, second.Version)) + require.NotZero(t, liveRubricWeight(t, client, name, first.Version), + "version %s must remain readable", first.Version) +} + +// liveRubricWeight reads back the one weight the fixture rubric carries. +// +// Read as JSON rather than matched as a substring: the service reformats what +// it stores, so `"weight":1` goes in and `"weight": 1` comes back, and a +// substring assertion would fail for a reason that has nothing to do with what +// is being tested. +func liveRubricWeight( + t *testing.T, + client *eval_api.EvalClient, + name, version string, +) int { + t.Helper() + + raw, err := client.GetEvaluatorRaw( + context.Background(), name, version, ProjectEndpointAPIVersion) + require.NoError(t, err) + + var doc struct { + Definition struct { + Dimensions []struct { + ID string `json:"id"` + Weight int `json:"weight"` + } `json:"dimensions"` + } `json:"definition"` + } + require.NoError(t, json.Unmarshal(raw, &doc)) + require.Len(t, doc.Definition.Dimensions, 1) + return doc.Definition.Dimensions[0].Weight +} + +// TestLiveFirstPublishReturnsVersionOne is the other half: the guard must not +// change what a first publish answers. +func TestLiveFirstPublishReturnsVersionOne(t *testing.T) { + client, _ := liveEvalClient(t) + ctx := context.Background() + + name := fmt.Sprintf("azdlive-firstpub-%d", time.Now().UnixNano()) + body, err := normalizeRubricBody(name, []byte( + `{"dimensions":[{"id":"tone","weight":1,"description":"polite"}]}`)) + require.NoError(t, err) + + created, err := client.CreateEvaluatorVersion(ctx, name, body, nil, ProjectEndpointAPIVersion) + require.NoError(t, err) + t.Cleanup(func() { + _ = client.DeleteEvaluatorVersion( + context.Background(), name, created.Version, ProjectEndpointAPIVersion) + }) + + require.Equal(t, "1", created.Version, + "a name the project has never seen must publish as version 1") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go index 44df054a40b..c2de22db18e 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go @@ -265,16 +265,20 @@ func (r *evalReconciler) EnsureEvaluator( } // Compare against the definition already on the service. + var known json.RawMessage if existing, err := r.ec.evalClient.GetEvaluatorRaw( ctx, decl.Name, "", ProjectEndpointAPIVersion, ); err == nil { if sameDefinition(existing, body) { return versionFromRaw(existing, decl.Version), false, nil } + // Different, so a version is about to be published. What that read + // saw is what keeps the publish from being answered with it again. + known = existing } created, err := r.ec.evalClient.CreateEvaluatorVersion( - ctx, decl.Name, body, ProjectEndpointAPIVersion, + ctx, decl.Name, body, known, ProjectEndpointAPIVersion, ) if err != nil { return "", false, err diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/evaluators.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/evaluators.go index 9f0a6f17b35..dd230ef1f08 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/evaluators.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/evaluators.go @@ -176,6 +176,16 @@ func (c *EvalClient) LatestEvaluatorVersionNumber( return number } +// parseVersionNumber reads a version string as an integer, answering 0 for one +// that is not numeric. +func parseVersionNumber(version string) int { + number, err := strconv.Atoi(version) + if err != nil { + return 0 + } + return number +} + // DeleteEvaluatorVersion removes a single evaluator version. func (c *EvalClient) DeleteEvaluatorVersion( ctx context.Context, diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/operations.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/operations.go index cf61480d16b..09291317d10 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/operations.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/operations.go @@ -13,6 +13,7 @@ import ( "net/http" "net/url" "strconv" + "time" "azureaieval/internal/version" @@ -186,14 +187,125 @@ func (c *EvalClient) GetAgent( // CreateEvaluatorVersion creates a new version of a named evaluator. // The body should be the full evaluator JSON with the definition field updated. +// +// previous is the evaluator document the caller has already read, or nil when +// it read none. It is what keeps the publish from being answered with the +// version that document holds. func (c *EvalClient) CreateEvaluatorVersion( ctx context.Context, name string, body json.RawMessage, + previous json.RawMessage, + apiVersion string, +) (*EvaluatorVersion, error) { + return c.publishEvaluatorVersion(ctx, name, previous, apiVersion, func() (*EvaluatorVersion, error) { + path := pathEvaluators + "/" + url.PathEscape(name) + "/versions" + return doRequestTyped[EvaluatorVersion](c, ctx, http.MethodPost, path, nil, body, apiVersion) + }) +} + +// versionSettle bounds the wait for the service to start assigning the next +// version number. +const ( + versionSettleTimeout = 45 * time.Second + versionSettleInterval = 3 * time.Second + versionSettleAge = 8 * time.Second +) + +// publishedVersion is the little of an evaluator document this needs: which +// version it is, and when it was written. +type publishedVersion struct { + Version string `json:"version"` + ModifiedAt time.Time `json:"modified_at"` + CreatedAt time.Time `json:"created_at"` +} + +// writtenAt reports when the version was last written, preferring the +// modification time and falling back to creation. +func (p publishedVersion) writtenAt() time.Time { + if !p.ModifiedAt.IsZero() { + return p.ModifiedAt + } + return p.CreatedAt +} + +// publishEvaluatorVersion publishes and then makes sure a new version is what +// came back. +// +// For a few seconds after a publish the service can answer the next one with +// the version it just assigned, writing over that version's contents instead +// of adding one. It is a race rather than a fixed window — a second publish +// has been seen both colliding a quarter of a second later and succeeding +// immediately — and nothing observable marks its end. +// +// That matters because versions are the unit an eval binds to. `evaluator +// create` followed by `evaluator update`, which is what a first authoring +// session looks like, would otherwise leave one version holding the second +// definition and every eval bound to the first silently scoring against a +// rubric nobody chose. +// +// So there are two defences. The publish is held back until the version the +// caller read has had time to settle, which is what keeps the collision from +// happening at all; and the version that comes back is checked, which is what +// keeps a collision that happens anyway from being reported as success. The +// recheck republishes the same body, so it cannot make a collision worse than +// the first attempt already did. +// +// What the caller reads is used rather than the version listing because the +// listing lags a publish too: asked immediately after a create it answers 404, +// so a guard that trusted it would stand down in exactly the case it exists +// for. Callers that publish an evaluator have already read it to decide +// between creating and updating. +func (c *EvalClient) publishEvaluatorVersion( + ctx context.Context, + name string, + previous json.RawMessage, apiVersion string, + publish func() (*EvaluatorVersion, error), ) (*EvaluatorVersion, error) { - path := pathEvaluators + "/" + url.PathEscape(name) + "/versions" - return doRequestTyped[EvaluatorVersion](c, ctx, http.MethodPost, path, nil, body, apiVersion) + var known publishedVersion + if len(previous) > 0 { + _ = json.Unmarshal(previous, &known) + } + + latest := parseVersionNumber(known.Version) + if listed := c.LatestEvaluatorVersionNumber(ctx, name, apiVersion); listed > latest { + latest = listed + } + + if written := known.writtenAt(); !written.IsZero() { + if wait := versionSettleAge - time.Since(written); wait > 0 { + select { + case <-ctx.Done(): + return nil, ctx.Err() + case <-time.After(wait): + } + } + } + + deadline := time.Now().Add(versionSettleTimeout) + for { + created, err := publish() + if err != nil { + return nil, err + } + if latest == 0 || parseVersionNumber(created.Version) > latest { + return created, nil + } + if time.Now().After(deadline) { + return nil, fmt.Errorf( + "publishing evaluator %q kept returning version %s, which already "+ + "existed. The service was still assigning that version after %s, so "+ + "version %s now holds what was just published and any eval bound to "+ + "it is scoring against it", + name, created.Version, versionSettleTimeout, created.Version) + } + select { + case <-ctx.Done(): + return nil, ctx.Err() + case <-time.After(versionSettleInterval): + } + } } // GetEvaluatorRaw gets an evaluator by name and version as raw JSON. diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/publish_version_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/publish_version_test.go new file mode 100644 index 00000000000..e34e1460017 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/publish_version_test.go @@ -0,0 +1,184 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package eval_api + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "sync/atomic" + "testing" + "time" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// newRecordingClient points a client at a test server, with no credential +// policy in the pipeline. +func newRecordingClient(t *testing.T, handler http.HandlerFunc) *EvalClient { + t.Helper() + server := httptest.NewServer(handler) + t.Cleanup(server.Close) + return NewEvalClientFromPipeline( + server.URL, runtime.NewPipeline("test", "v1.0.0", runtime.PipelineOptions{}, nil)) +} + +// versionServer answers a version listing and a publish, assigning whatever +// version the caller decides for each attempt. +func versionServer(t *testing.T, existing []string, assign func(attempt int) string) ( + http.HandlerFunc, *atomic.Int32, +) { + t.Helper() + var publishes atomic.Int32 + + return func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + + if r.Method == http.MethodGet { + values := []map[string]any{} + for _, v := range existing { + values = append(values, map[string]any{"name": "tone", "version": v}) + } + if len(existing) == 0 { + w.WriteHeader(http.StatusNotFound) + _, _ = w.Write([]byte(`{"error":{"code":"NotFound"}}`)) + return + } + require.NoError(t, json.NewEncoder(w).Encode(map[string]any{"value": values})) + return + } + + attempt := int(publishes.Add(1)) + w.WriteHeader(http.StatusCreated) + require.NoError(t, json.NewEncoder(w).Encode(map[string]any{ + "name": "tone", "version": assign(attempt), + })) + }, &publishes +} + +// A name the project has never seen has no version to collide with, so it must +// publish once and return. Waiting there would tax every first publish for a +// hazard that cannot apply. +func TestCreateEvaluatorVersion_FirstPublishDoesNotRetry(t *testing.T) { + handler, publishes := versionServer(t, nil, func(int) string { return "1" }) + client := newRecordingClient(t, handler) + + started := time.Now() + created, err := client.CreateEvaluatorVersion( + context.Background(), "tone", json.RawMessage(`{}`), nil, "2025-11-15-preview") + require.NoError(t, err) + + assert.Equal(t, "1", created.Version) + assert.Equal(t, int32(1), publishes.Load(), "a first publish must be issued once") + assert.Less(t, time.Since(started), versionSettleInterval, + "a first publish must not wait on a version that cannot exist") +} + +// For a few seconds after a publish the service answers the next one with the +// version it just assigned, replacing that version rather than adding one. +// Accepting it would leave every eval bound to the earlier version scoring +// against a definition nobody chose, so the publish is reissued until the +// version advances. +func TestCreateEvaluatorVersion_RetriesUntilTheVersionAdvances(t *testing.T) { + handler, publishes := versionServer(t, []string{"1"}, func(attempt int) string { + if attempt < 3 { + return "1" + } + return "2" + }) + client := newRecordingClient(t, handler) + + created, err := client.CreateEvaluatorVersion( + context.Background(), "tone", json.RawMessage(`{}`), nil, "2025-11-15-preview") + require.NoError(t, err) + + assert.Equal(t, "2", created.Version) + assert.Equal(t, int32(3), publishes.Load(), + "the publish must be reissued until the service assigns a new version") +} + +// A service that never advances must end in an error rather than in a version +// the caller believes is new. Reporting success there is the failure the whole +// guard exists to prevent. +func TestCreateEvaluatorVersion_GivesUpRatherThanReportASharedVersion(t *testing.T) { + handler, _ := versionServer(t, []string{"4"}, func(int) string { return "4" }) + client := newRecordingClient(t, handler) + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + _, err := client.CreateEvaluatorVersion( + ctx, "tone", json.RawMessage(`{}`), nil, "2025-11-15-preview") + require.Error(t, err) +} + +// The version listing lags a publish: asked immediately after a create it +// answers 404. A guard that trusted it would stand down in exactly the window +// it exists for, which is why the caller supplies the version it has already +// read. +func TestCreateEvaluatorVersion_UsesTheCallersVersionWhenTheListingLags(t *testing.T) { + handler, publishes := versionServer(t, nil, func(attempt int) string { + if attempt < 2 { + return "1" + } + return "2" + }) + client := newRecordingClient(t, handler) + + created, err := client.CreateEvaluatorVersion( + context.Background(), "tone", json.RawMessage(`{}`), json.RawMessage(`{"version":"1"}`), "2025-11-15-preview") + require.NoError(t, err) + + assert.Equal(t, "2", created.Version) + assert.Equal(t, int32(2), publishes.Load(), + "the version the caller read must be enough to catch the collision") +} + +// A version the service does not number cannot be compared, so it is taken at +// face value: refusing it would make an evaluator unpublishable over a +// convention this extension does not own. +func TestParseVersionNumber(t *testing.T) { + assert.Equal(t, 7, parseVersionNumber("7")) + assert.Equal(t, 0, parseVersionNumber("v7")) + assert.Equal(t, 0, parseVersionNumber("")) +} + +// The publish is reissued, so the same body has to arrive every time. A +// closure that consumed its body on the first attempt would send an empty one +// on the second and publish an evaluator with no definition. +func TestCreateEvaluatorVersion_ReissuesTheSameBody(t *testing.T) { + bodies := make(chan string, 4) + handler, _ := versionServer(t, []string{"1"}, func(attempt int) string { + if attempt < 2 { + return "1" + } + return "2" + }) + client := newRecordingClient(t, func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodPost { + buf := make([]byte, r.ContentLength) + _, _ = r.Body.Read(buf) + bodies <- string(buf) + } + handler(w, r) + }) + + _, err := client.CreateEvaluatorVersion( + context.Background(), "tone", + json.RawMessage(`{"definition":{"type":"rubric"}}`), nil, "2025-11-15-preview") + require.NoError(t, err) + close(bodies) + + seen := 0 + for body := range bodies { + seen++ + assert.Contains(t, body, "rubric", fmt.Sprintf("attempt %d sent an empty body", seen)) + } + assert.Equal(t, 2, seen) +} From badd6709c49d816bc6bfcb888b72d9c559009f58 Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 4 Aug 2026 02:01:16 +0300 Subject: [PATCH 090/320] Print the job id in the line that tells you to use it \--no-wait\ exists so the caller can walk away, and the line they walk away with is the one they paste when they come back. It said Submitted. \zd ai eval job show \ reports its progress. The id was on the line above, so nothing was lost - but the instruction itself was not usable, and the check that every command the CLI suggests is a real command passed it, because \job show\ is one. A second check now refuses a suggestion that still has a placeholder in it, and it was confirmed to fail on the line it was written for. Also drops checkEvaluatorDrift. It reads the version listing to catch a version published outside the repo, and nothing calls it: the rubric path always reads the latest version and republishes when the local definition differs, so there is no window for it to guard. It belongs to the code evaluator path, which fingerprints instead of comparing, and that is M2's. --- .../internal/cmd/generate.go | 14 ++++++- .../internal/cmd/generate_commands.go | 9 ++-- .../internal/cmd/reconciler.go | 27 ------------ .../internal/cmd/surface_test.go | 42 +++++++++++++++++++ 4 files changed, 57 insertions(+), 35 deletions(-) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go index 823ed4e251b..bb3fe950310 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go @@ -177,7 +177,7 @@ func (ec *evalContext) generateRubric( return nil, fmt.Errorf("submitting the rubric generation job: %w", err) } if noWait { - fmt.Fprintf(out, " submitted job %s\n", job.ID) + reportSubmitted(out, job.ID) return nil, nil } @@ -196,6 +196,16 @@ func (ec *evalContext) generateRubric( return &project.ArtifactRef{Name: plan.Name, Source: relativeSource(plan.BaseDir, path)}, nil } +// reportSubmitted says what was started and how to get back to it. +// +// The job id goes into the command rather than being left as a placeholder: +// --no-wait exists so the caller can walk away, and the line they walk away +// with has to be the one they can paste when they come back. +func reportSubmitted(out io.Writer, jobID string) { + fmt.Fprintf(out, " submitted job %s\n", jobID) + fmt.Fprintf(out, "\nReattach with: azd ai eval job show %s\n", jobID) +} + // generateDataset submits the data generation job and downloads the result. func (ec *evalContext) generateDataset( ctx context.Context, @@ -215,7 +225,7 @@ func (ec *evalContext) generateDataset( return nil, fmt.Errorf("submitting the data generation job: %w", err) } if noWait { - fmt.Fprintf(out, " submitted job %s\n", job.ID) + reportSubmitted(out, job.ID) return nil, nil } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_commands.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_commands.go index 9e8c5a33dac..fd03dfc5bbe 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_commands.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_commands.go @@ -289,16 +289,13 @@ func newEvaluatorGenerateCommand() *cobra.Command { // reportGenerated closes out either command. // // With --no-wait nothing was downloaded and there is no ref, which is success: -// the job id was printed and `job show` reattaches to it. +// reportSubmitted has already said how to reattach. func reportGenerated(cmd *cobra.Command, ref *project.ArtifactRef, noWait bool) error { out := cmd.OutOrStdout() if ref == nil { - if noWait { - fmt.Fprintln(out, - "\nSubmitted. `azd ai eval job show ` reports its progress.") - return nil + if !noWait { + fmt.Fprintln(out, "Nothing was generated.") } - fmt.Fprintln(out, "Nothing was generated.") return nil } if isJSON(cmd) { diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go index c2de22db18e..5611579cf09 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go @@ -11,7 +11,6 @@ import ( "os" "path/filepath" "reflect" - "strconv" "strings" "time" @@ -363,32 +362,6 @@ func (r *evalReconciler) evaluatorVersionResolvable( return false } -// checkEvaluatorDrift fails when the service holds a newer version than the -// one recorded at the last deploy. -// -// Publishing is not destructive — versions are immutable — so the remedy is to -// sync with what is on the project, not to overwrite it. -func (r *evalReconciler) checkEvaluatorDrift( - ctx context.Context, - name, recorded string, -) error { - recordedNumber, err := strconv.Atoi(recorded) - if err != nil { - return nil - } - latest := r.ec.evalClient.LatestEvaluatorVersionNumber( - ctx, name, ProjectEndpointAPIVersion, - ) - if latest <= recordedNumber { - return nil - } - return fmt.Errorf( - "evaluator %q is at version %d on the project but %s was recorded at the last deploy; "+ - "someone published a version outside this repo. "+ - "Pull the newer code locally, or delete version %d, then deploy again", - name, latest, recorded, latest) -} - // EnsureEval creates the group when it has never been deployed, or when an // upstream artifact changed. Groups are immutable, so a change means a new // group and a new id. diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/surface_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/surface_test.go index 551dd45359c..acaaf5143a0 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/surface_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/surface_test.go @@ -252,3 +252,45 @@ func TestSuggestedCommandsExist(t *testing.T) { }) require.NoError(t, err) } + +// A command suggested with an argument has to be suggested with the argument +// filled in. +// +// `--no-wait` exists so the caller can walk away, and the line they walk away +// with is the one they paste when they come back. Printing +// `azd ai eval job show ` reads like a command and is not one: it +// resolves, so the check above passes, and it fails the moment anyone uses it. +func TestSuggestedCommandsCarryNoPlaceholders(t *testing.T) { + placeholder := regexp.MustCompile(`azd ai eval [^"'\n]*<[a-z-]+>`) + + err := filepath.WalkDir("../..", func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() || !strings.HasSuffix(path, ".go") || strings.HasSuffix(path, "_test.go") { + return nil + } + + body, err := os.ReadFile(path) + if err != nil { + return err + } + for i, line := range strings.Split(string(body), "\n") { + trimmed := strings.TrimSpace(line) + // A `Use:` string and the help text around it are where a + // placeholder belongs: cobra prints it as the signature. + if strings.HasPrefix(trimmed, "//") || + strings.HasPrefix(trimmed, "Use:") || + strings.HasPrefix(trimmed, "Short:") || + strings.HasPrefix(trimmed, "Long:") { + continue + } + if m := placeholder.FindString(line); m != "" { + t.Errorf("%s:%d suggests %q; substitute the value instead", + path, i+1, m) + } + } + return nil + }) + require.NoError(t, err) +} From 425171b11f135a6ed0f2f90e075b40d2964d9530 Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 4 Aug 2026 02:13:24 +0300 Subject: [PATCH 091/320] Stop reporting a run that could not run as one that passed Two gaps in the exit-code contract, both found by reading the spec's own promises back against the code. A run that finished with status failed or error exited 0. The reason was printed, so a person reading the terminal saw it, but a pipeline gating on the command saw success - and the spec makes exit 1 mean exactly this case. It is checked before the gate, because the gate's exit 2 means the evaluation regressed and a run that produced no results has not regressed; telling those apart is what the separate code is for. \ un show\ only reports it when --wait was asked: without it, this is an inspection command and answering the question is a success whatever the answer. And evaluator drift was never detected. checkEvaluatorDrift existed and nothing called it, so \zd up\ would publish over a version somebody had deliberately published from the portal and report success. It is called now, and it is answered from the version recorded at the last deploy rather than from the version listing - the listing lags a publish, and would report an evaluator as un-drifted for the first seconds of its newest version's life. Evaluators need this comparison at all only because their definitions come back inline: local and remote disagreeing says nothing on its own about who moved. --- .../internal/cmd/reconciler.go | 57 ++++++++++++++++++- .../internal/cmd/reconciler_drift_test.go | 40 +++++++++++++ .../azure.ai.evaluations/internal/cmd/run.go | 23 ++++++++ .../internal/cmd/run_ops.go | 16 ++++++ .../internal/cmd/run_status_test.go | 38 +++++++++++++ 5 files changed, 172 insertions(+), 2 deletions(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler_drift_test.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_status_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go index 5611579cf09..c770f26ea7b 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go @@ -11,6 +11,7 @@ import ( "os" "path/filepath" "reflect" + "strconv" "strings" "time" @@ -268,11 +269,30 @@ func (r *evalReconciler) EnsureEvaluator( if existing, err := r.ec.evalClient.GetEvaluatorRaw( ctx, decl.Name, "", ProjectEndpointAPIVersion, ); err == nil { + remote := versionFromRaw(existing, "") if sameDefinition(existing, body) { + // Nothing to publish, but the version is still worth recording: + // it is what a later deploy compares against to notice that + // someone moved the evaluator on from here. + if remote != "" { + _ = r.ec.setEnvValue(ctx, versionKey("evaluator", decl.Name), remote) + } return versionFromRaw(existing, decl.Version), false, nil } - // Different, so a version is about to be published. What that read - // saw is what keeps the publish from being answered with it again. + + // The definitions differ, which means either the local file changed + // or someone published a version outside the repo. The version + // recorded at the last deploy is what tells them apart, and + // publishing over the second case would bury an intentional change + // under one nobody asked for. + if recorded := r.ec.getEnvValue(ctx, versionKey("evaluator", decl.Name)); recorded != "" { + if err := checkEvaluatorDrift(decl.Name, recorded, remote); err != nil { + return "", false, err + } + } + + // What that read saw is what keeps the publish from being answered + // with it again. known = existing } @@ -283,9 +303,42 @@ func (r *evalReconciler) EnsureEvaluator( return "", false, err } r.awaitEvaluatorReadable(ctx, decl.Name, created.Version) + _ = r.ec.setEnvValue(ctx, versionKey("evaluator", decl.Name), created.Version) return created.Version, true, nil } +// checkEvaluatorDrift fails when the service holds a newer version than the +// one recorded at the last deploy. +// +// It is asked only when the local definition and the remote one disagree, +// which on its own says nothing about who moved: the author may have edited +// the file, or someone may have published a version from outside the repo. +// The recorded version settles it, and the difference matters because +// publishing is how this reconciler resolves a disagreement — doing that over +// a version somebody deliberately published would bury their change under one +// nobody asked for, with `azd up` reporting success. +// +// The remote version is passed in rather than listed, because the version +// listing lags a publish and would report an evaluator as un-drifted for the +// first seconds of its newest version's life. +func checkEvaluatorDrift(name, recorded, remote string) error { + recordedNumber, err := strconv.Atoi(recorded) + if err != nil { + return nil + } + remoteNumber, err := strconv.Atoi(remote) + if err != nil || remoteNumber <= recordedNumber { + return nil + } + return fmt.Errorf( + "evaluator %q is at version %s on the project but %s was recorded at the last "+ + "deploy, and the local definition does not match it: someone published a "+ + "version outside this repo. Publishing over it would leave their change "+ + "behind, so bring version %s into the declared source and deploy again, or "+ + "delete that version if it was a mistake", + name, remote, recorded, remote) +} + // evaluatorPropagation bounds the wait for a freshly published evaluator to // become usable. // diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler_drift_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler_drift_test.go new file mode 100644 index 00000000000..d945b95eda9 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler_drift_test.go @@ -0,0 +1,40 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// Drift is only interesting when the two definitions already disagree, and +// then only when the disagreement came from the project rather than from the +// author. These are the four ways that question can be answered. +func TestCheckEvaluatorDrift(t *testing.T) { + // The author edited the file. The project is where the last deploy left + // it, so publishing is exactly right and must not be blocked. + require.NoError(t, checkEvaluatorDrift("support-quality", "3", "3")) + + // Someone published outside the repo. Publishing over it would leave + // their change behind with `azd up` reporting success. + err := checkEvaluatorDrift("support-quality", "3", "4") + require.Error(t, err) + assert.Contains(t, err.Error(), "support-quality") + assert.Contains(t, err.Error(), "version 4") + assert.Contains(t, err.Error(), "3 was recorded") + assert.Contains(t, err.Error(), "outside this repo", + "the message has to say who moved, not just that something did") + + // A version that went backwards is not drift: a newer version was + // deleted, and republishing is how the repo takes the name back. + require.NoError(t, checkEvaluatorDrift("support-quality", "4", "3")) + + // Versions this extension did not number cannot be compared, and refusing + // a deploy over a numbering convention it does not own would be worse + // than not checking. + require.NoError(t, checkEvaluatorDrift("support-quality", "", "4")) + require.NoError(t, checkEvaluatorDrift("support-quality", "3", "preview")) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go index e93f313a0d7..9bde2604579 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go @@ -32,6 +32,26 @@ var terminalRunStates = map[string]bool{ "error": true, } +// runCompleted turns a run that did not complete into an error, so that a +// caller who waited for it exits non-zero. +// +// The results have already been printed by the time this is asked, which is +// the point: a run that errored has a reason worth reading, and reporting it +// and then exiting 0 tells a pipeline the evaluation passed. It is checked +// before the gate because the gate's exit code means "the evaluation +// regressed", and a run that never produced results has not regressed — it did +// not run. Distinguishing those two is what the separate code is for. +func runCompleted(run *eval_api.OpenAIEvalRun) error { + if run == nil { + return nil + } + switch strings.ToLower(run.Status) { + case "completed", "": + return nil + } + return fmt.Errorf("run %s finished with status %s", run.ID, run.Status) +} + // newRunCommand builds the run group. // // `run` is a group, not an executable verb: once `run output` exists, a bare @@ -184,6 +204,9 @@ func buildRunCommand(use, short string) *cobra.Command { // Last, so that the results are reported whether or not the gate // holds: a pipeline that only learns it failed is worse off than // one that can see by how much. + if err := runCompleted(final); err != nil { + return err + } applyGate(cmd, threshold, final) return nil }, diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go index e7e537e544c..deb0d0a49e5 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go @@ -125,6 +125,12 @@ func newRunShowCommand() *cobra.Command { // Reattaching to a run started asynchronously: the pipeline that // gates on it is often not the one that started it. + // + // Only a caller that waited is told a bad status through the exit + // code. Without --wait this is an inspection command: it was asked + // what happened, and answering that is a success whatever the + // answer. + gateOnStatus := wait if wait { run, err = ec.pollRun(ctx, evalID, run.ID, cmd.OutOrStdout(), isJSON(cmd)) if err != nil { @@ -136,6 +142,11 @@ func newRunShowCommand() *cobra.Command { if err := emitJSON(cmd.OutOrStdout(), run); err != nil { return err } + if gateOnStatus { + if err := runCompleted(run); err != nil { + return err + } + } applyGate(cmd, threshold, run) return nil } @@ -150,6 +161,11 @@ func newRunShowCommand() *cobra.Command { if run.ReportURL != "" { fmt.Fprintf(out, " report : %s\n", run.ReportURL) } + if gateOnStatus { + if err := runCompleted(run); err != nil { + return err + } + } applyGate(cmd, threshold, run) return nil }, diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_status_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_status_test.go new file mode 100644 index 00000000000..646fd560e50 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_status_test.go @@ -0,0 +1,38 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "testing" + + "azureaieval/internal/pkg/eval_api" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// The exit code is the whole contract with a pipeline, and there are three +// answers it has to be able to give: the evaluation ran and passed, it ran and +// regressed, or it could not run. The gate owns the middle one; this owns the +// last. +// +// Reporting a run that errored and then exiting 0 tells the pipeline the +// evaluation passed, which is the one answer that is never true. +func TestRunCompleted(t *testing.T) { + require.NoError(t, runCompleted(nil), + "nothing was waited for, so there is nothing to report") + require.NoError(t, runCompleted(&eval_api.OpenAIEvalRun{ID: "r1", Status: "completed"})) + require.NoError(t, runCompleted(&eval_api.OpenAIEvalRun{ID: "r1", Status: "Completed"}), + "the service is not consistent about case") + require.NoError(t, runCompleted(&eval_api.OpenAIEvalRun{ID: "r1"}), + "a status the service did not send is not a failure to report") + + for _, status := range []string{"failed", "error", "canceled", "cancelled"} { + err := runCompleted(&eval_api.OpenAIEvalRun{ID: "run_abc", Status: status}) + require.Error(t, err, "status %q must not exit 0", status) + assert.Contains(t, err.Error(), "run_abc") + assert.Contains(t, err.Error(), status, + "the message has to name the status, which is what the caller acts on") + } +} From 5e43f3ce29c98a034cc805f635131fea6ad488d5 Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 4 Aug 2026 02:32:15 +0300 Subject: [PATCH 092/320] Report the verdict the run was started to get \ un start\ waited for a run and then printed one line: its id and its status. The verdict per evaluator - which is the entire reason to run one - was returned by the service and thrown away, so the answer to the question the command had just been asked took a second command to see. It now prints a row per evaluator with pass, fail and pass rate, sorted by name so two runs of the same eval read the same way, followed by the overall rate over samples rather than over verdicts: a sample that failed two evaluators is one sample to go and look at. Errored rows are named rather than folded into the fail column. The evaluator never reached a verdict on them, and counting them as failures reports a service problem as a quality problem. For the same reason a criterion's rate is over what it scored, and a rate over nothing prints as a dash - 0.0% would read as a total failure rather than as no data. --- .../azure.ai.evaluations/internal/cmd/run.go | 81 ++++++++++++ .../internal/cmd/run_summary_test.go | 117 ++++++++++++++++++ 2 files changed, 198 insertions(+) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_summary_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go index 9bde2604579..5b429408a7e 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go @@ -13,6 +13,7 @@ import ( "io" "os" "path/filepath" + "sort" "strings" "time" @@ -640,6 +641,14 @@ func (ec *evalContext) pollRun( } } +// renderRun prints what a person needs after waiting for a run. +// +// The status line alone is not that. A run's whole purpose is the verdict per +// evaluator, and the service returns it — passed, failed and errored counts +// for every testing criterion — so leaving it out meant the answer to the +// question the command was asked required a second command to see. The report +// URL and the run id come last, because they are what you act on after +// reading the numbers rather than instead of reading them. func renderRun(out interface{ Write([]byte) (int, error) }, run *eval_api.OpenAIEvalRun) error { fmt.Fprintf(out, "\nRun %s finished with status %s\n", run.ID, run.Status) // A run that failed carries why, and it is usually the only actionable @@ -648,8 +657,80 @@ func renderRun(out interface{ Write([]byte) (int, error) }, run *eval_api.OpenAI if why := run.Failure(); why != "" { fmt.Fprintf(out, " %s\n", why) } + + renderCriteriaTable(out, run.PerTestingCriteria) + + // Counted over samples, not over verdicts: a sample that failed two + // evaluators is one sample to go and look at, and reporting it as two + // overstates how much is wrong. + if c := run.ResultCounts; c != nil && c.Total > 0 { + fmt.Fprintf(out, "\nOverall pass rate: %s (%d/%d samples passed every evaluator)\n", + formatRate(c.Passed, c.Total), c.Passed, c.Total) + if c.Errored > 0 { + fmt.Fprintf(out, "%d sample(s) errored and were not scored.\n", c.Errored) + } + if c.Failed > 0 { + fmt.Fprintln(out, + "\nView failing samples: azd ai eval run output list --failed-only") + } + } + if run.ReportURL != "" { fmt.Fprintf(out, "Report: %s\n", run.ReportURL) } return nil } + +// renderCriteriaTable prints one row per evaluator. +// +// Sorted by name so two runs of the same eval read the same way; the service +// returns the criteria in whatever order it evaluated them. +func renderCriteriaTable( + out interface{ Write([]byte) (int, error) }, + results []eval_api.EvalRunCriteriaResult, +) { + if len(results) == 0 { + return + } + + sorted := append([]eval_api.EvalRunCriteriaResult(nil), results...) + sort.Slice(sorted, func(i, j int) bool { + return sorted[i].TestingCriteria < sorted[j].TestingCriteria + }) + + width := len("EVALUATOR") + for _, r := range sorted { + if n := len(r.TestingCriteria); n > width { + width = n + } + } + + fmt.Fprintf(out, "\n%-*s %4s %4s %9s\n", width, "EVALUATOR", "PASS", "FAIL", "PASS RATE") + fmt.Fprintf(out, "%s %s %s %s\n", + strings.Repeat("-", width), "----", "----", "---------") + for _, r := range sorted { + scored := r.Passed + r.Failed + fmt.Fprintf(out, "%-*s %4d %4d %9s\n", + width, r.TestingCriteria, r.Passed, r.Failed, formatRate(r.Passed, scored)) + // Errors are not failures — the evaluator never reached a verdict — + // so they are named rather than folded into the fail column, where + // they would look like a quality problem. + if r.Errored > 0 { + fmt.Fprintf(out, "%-*s %s\n", width, "", errorNote(r.Errored)) + } + } +} + +// errorNote describes rows an evaluator could not score. +func errorNote(errored int) string { + return fmt.Sprintf("(%d errored, not scored)", errored) +} + +// formatRate renders a share as a percentage, and a rate over nothing as a +// dash: 0.0%% would read as a total failure rather than as no data. +func formatRate(part, whole int) string { + if whole <= 0 { + return "-" + } + return fmt.Sprintf("%.1f%%", float64(part)/float64(whole)*100) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_summary_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_summary_test.go new file mode 100644 index 00000000000..6bb4a2f4c34 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_summary_test.go @@ -0,0 +1,117 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "bytes" + "strings" + "testing" + + "azureaieval/internal/pkg/eval_api" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// finishedRun is a run the way the service returns one: counts over samples, +// and a result per testing criterion. +func finishedRun() *eval_api.OpenAIEvalRun { + return &eval_api.OpenAIEvalRun{ + ID: "evalrun_abc123", + Status: "completed", + ResultCounts: &eval_api.EvalRunResultCounts{ + Total: 10, Passed: 7, Failed: 3, + }, + PerTestingCriteria: []eval_api.EvalRunCriteriaResult{ + {TestingCriteria: "relevance", Passed: 9, Failed: 1}, + {TestingCriteria: "coherence", Passed: 7, Failed: 3}, + }, + } +} + +// The whole point of waiting for a run is the verdict per evaluator. Printing +// only the status meant the answer to the question the command was asked took +// a second command to see. +func TestRenderRunReportsEveryEvaluator(t *testing.T) { + var out bytes.Buffer + require.NoError(t, renderRun(&out, finishedRun())) + text := out.String() + + assert.Contains(t, text, "evalrun_abc123") + assert.Contains(t, text, "completed") + + for _, criterion := range []string{"relevance", "coherence"} { + assert.Contains(t, text, criterion, + "every evaluator the run scored must appear") + } + assert.Contains(t, text, "90.0%", "relevance passed 9 of 10") + assert.Contains(t, text, "70.0%", "coherence passed 7 of 10") + assert.Contains(t, text, "7/10", "the sample counts must be shown, not just the rate") +} + +// Two runs of the same eval have to read the same way. The service returns the +// criteria in whatever order it evaluated them, which is not stable. +func TestRenderRunOrdersEvaluatorsByName(t *testing.T) { + var out bytes.Buffer + require.NoError(t, renderRun(&out, finishedRun())) + + text := out.String() + assert.Less(t, strings.Index(text, "coherence"), strings.Index(text, "relevance"), + "evaluators must be listed in a stable order") +} + +// An errored row is not a failing row: the evaluator never reached a verdict. +// Folding the two together would report a service problem as a quality problem. +func TestRenderRunSeparatesErrorsFromFailures(t *testing.T) { + run := finishedRun() + run.ResultCounts = &eval_api.EvalRunResultCounts{Total: 10, Passed: 7, Failed: 1, Errored: 2} + run.PerTestingCriteria = []eval_api.EvalRunCriteriaResult{ + {TestingCriteria: "relevance", Passed: 7, Failed: 1, Errored: 2}, + } + + var out bytes.Buffer + require.NoError(t, renderRun(&out, run)) + text := out.String() + + assert.Contains(t, text, "2 errored") + assert.Contains(t, text, "87.5%", + "the pass rate is over what was scored, not over what was attempted") + assert.Contains(t, text, "errored and were not scored") +} + +// A rate over nothing is not zero. Printing 0.0% for a criterion that scored +// no rows reads as a total failure rather than as no data. +func TestFormatRateHasNoOpinionAboutNothing(t *testing.T) { + assert.Equal(t, "-", formatRate(0, 0)) + assert.Equal(t, "0.0%", formatRate(0, 4)) + assert.Equal(t, "100.0%", formatRate(4, 4)) + assert.Equal(t, "33.3%", formatRate(1, 3)) +} + +// The next thing anyone does after seeing failures is look at them, so the +// command that shows them is named — and it has to be a command that exists. +func TestRenderRunPointsAtTheFailingSamples(t *testing.T) { + var out bytes.Buffer + require.NoError(t, renderRun(&out, finishedRun())) + assert.Contains(t, out.String(), "azd ai eval run output list --failed-only") + + clean := finishedRun() + clean.ResultCounts = &eval_api.EvalRunResultCounts{Total: 10, Passed: 10} + clean.PerTestingCriteria = []eval_api.EvalRunCriteriaResult{ + {TestingCriteria: "relevance", Passed: 10}, + } + var cleanOut bytes.Buffer + require.NoError(t, renderRun(&cleanOut, clean)) + assert.NotContains(t, cleanOut.String(), "--failed-only", + "a run with nothing to look at must not send anyone looking") +} + +// A run that never produced counts still has to render. The service returns +// none for a run that failed before scoring, and a nil dereference there would +// replace the failure message with a panic. +func TestRenderRunSurvivesAnEmptyResult(t *testing.T) { + var out bytes.Buffer + require.NoError(t, renderRun(&out, &eval_api.OpenAIEvalRun{ID: "evalrun_x", Status: "failed"})) + assert.Contains(t, out.String(), "evalrun_x") +} From d01832c601b751de462eb7264aef7d29e9973cc5 Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 4 Aug 2026 02:44:26 +0300 Subject: [PATCH 093/320] Give CI a shape to read, and stop a slow token from failing a test Two things. \ un start --no-wait -o json\ printed the service's run object verbatim. A pipeline reading it to reattach later had to know that the run id is \id\, that the eval id echoed back is not necessarily the one the command resolved, and that the declaration name it wants to log is not there at all - while depending on every other field the API happens to return. It now emits the handoff the spec documents: run_id, eval_id, eval_name, status, created_at, and nothing this extension does not promise to keep. And the live suite's credential now retries. The token refresh shells out to azd and the SDK gives that subprocess ten seconds; with the whole suite running it sometimes does not finish in ten, which surfaced as \AzureDeveloperCLICredential: exit status 1\ on whichever test asked at the wrong moment - reproducibly at 10.1s, and never when that test ran alone. Nothing about the request was wrong, so it is retried rather than reported. --- .../internal/cmd/build_live_test.go | 42 +++++++++++- .../azure.ai.evaluations/internal/cmd/run.go | 39 ++++++++++- .../internal/cmd/run_handoff_test.go | 67 +++++++++++++++++++ 3 files changed, 145 insertions(+), 3 deletions(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_handoff_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build_live_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build_live_test.go index 64bcf50b2c2..f9b8edfc683 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build_live_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build_live_test.go @@ -27,6 +27,8 @@ import ( "azureaieval/internal/pkg/evalcore" "azureaieval/internal/project" + "github.com/Azure/azure-sdk-for-go/sdk/azcore" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" "github.com/Azure/azure-sdk-for-go/sdk/azidentity" "github.com/stretchr/testify/require" ) @@ -51,6 +53,44 @@ func liveCredential() (*azidentity.AzureDeveloperCLICredential, error) { return sharedCred, sharedCredErr } +// credentialFlake is what a token refresh that overran its budget looks like +// by the time it reaches a test. +const credentialFlake = "AzureDeveloperCLICredential: exit status 1" + +// retryingCredential retries a token request that failed for that reason. +// +// The refresh shells out to azd, and the SDK gives that subprocess ten +// seconds. On a machine already running the rest of this suite it sometimes +// does not finish in ten, and the failure lands on whichever test asked for a +// token at the wrong moment — reproducibly at 10.1s, and never when that test +// is run on its own. Retrying is right because nothing about the request was +// wrong: the same call succeeds moments later. +type retryingCredential struct { + inner azcore.TokenCredential +} + +func (c retryingCredential) GetToken( + ctx context.Context, + opts policy.TokenRequestOptions, +) (azcore.AccessToken, error) { + var token azcore.AccessToken + var err error + for attempt := range 4 { + if attempt > 0 { + select { + case <-ctx.Done(): + return azcore.AccessToken{}, ctx.Err() + case <-time.After(time.Duration(attempt) * 2 * time.Second): + } + } + token, err = c.inner.GetToken(ctx, opts) + if err == nil || !strings.Contains(err.Error(), credentialFlake) { + return token, err + } + } + return token, err +} + func liveEvalClient(t *testing.T) (*eval_api.EvalClient, string) { t.Helper() if os.Getenv("AZURE_AI_EVAL_E2E_LIVE") != "1" { @@ -67,7 +107,7 @@ func liveEvalClient(t *testing.T) (*eval_api.EvalClient, string) { if judge == "" { judge = "gpt-4.1-nano" } - return eval_api.NewEvalClient(endpoint, cred), judge + return eval_api.NewEvalClient(endpoint, retryingCredential{inner: cred}), judge } // TestLiveBuildAcceptedForEveryBuiltin walks every built-in the project diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go index 5b429408a7e..67c26bf6bae 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go @@ -182,7 +182,7 @@ func buildRunCommand(use, short string) *cobra.Command { if !wait { if isJSON(cmd) { - return emitJSON(out, run) + return emitJSON(out, startedRun(run, evalID, group)) } fmt.Fprintf(out, "Started run %s (status: %s)\n", run.ID, run.Status) fmt.Fprintf(out, "Reattach with: azd ai eval run show %s --eval-id %s\n", run.ID, evalID) @@ -641,8 +641,43 @@ func (ec *evalContext) pollRun( } } -// renderRun prints what a person needs after waiting for a run. +// startedRunHandoff is what `run start --no-wait -o json` returns. // +// It is a handoff rather than a dump of the service object. The pipeline that +// started the run has to come back for it later, and doing that needs exactly +// three things: the run, the eval it belongs to, and a name a human can read +// in the log that reports it. The service object carries none of the third and +// buries the first two under the data source, the metadata and every field the +// API happens to return, so a script reading it would depend on a shape this +// extension does not control. +type startedRunHandoff struct { + RunID string `json:"run_id"` + EvalID string `json:"eval_id"` + EvalName string `json:"eval_name,omitempty"` + Status string `json:"status,omitempty"` + CreatedAt any `json:"created_at,omitempty"` +} + +// startedRun builds the handoff. +func startedRun( + run *eval_api.OpenAIEvalRun, + evalID string, + group *project.Eval, +) startedRunHandoff { + handoff := startedRunHandoff{ + RunID: run.ID, + EvalID: evalID, + Status: run.Status, + CreatedAt: run.CreatedAt, + } + // Absent with --eval-id, where there is no config to take a name from. + if group != nil { + handoff.EvalName = group.Name + } + return handoff +} + +// renderRun prints what a person needs after waiting for a run. // The status line alone is not that. A run's whole purpose is the verdict per // evaluator, and the service returns it — passed, failed and errored counts // for every testing criterion — so leaving it out meant the answer to the diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_handoff_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_handoff_test.go new file mode 100644 index 00000000000..cf4a522c2f5 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_handoff_test.go @@ -0,0 +1,67 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "encoding/json" + "testing" + + "azureaieval/internal/pkg/eval_api" + "azureaieval/internal/project" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// A pipeline that starts a run with --no-wait has to come back for it. What it +// needs to do that is a fixed shape this extension controls, not whatever the +// API happened to return. +func TestStartedRunIsTheHandoffAPipelineNeeds(t *testing.T) { + run := &eval_api.OpenAIEvalRun{ + ID: "evalrun_01JQZX", + EvalID: "eval_ignored", + Status: "queued", + CreatedAt: "2026-07-31T21:04:11Z", + Metadata: map[string]string{"azd_eval": "support-agent-smoke"}, + DataSource: &eval_api.EvalRunDataSource{ + Type: eval_api.EvalRunDataSourceTypeTraces, + }, + } + + raw, err := json.Marshal(startedRun(run, "eval_01JQZW", &project.Eval{Name: "support-agent-smoke"})) + require.NoError(t, err) + + var out map[string]any + require.NoError(t, json.Unmarshal(raw, &out)) + + assert.Equal(t, "evalrun_01JQZX", out["run_id"]) + assert.Equal(t, "support-agent-smoke", out["eval_name"]) + assert.Equal(t, "queued", out["status"]) + assert.Equal(t, "2026-07-31T21:04:11Z", out["created_at"]) + + // The eval the run was started against, which is the one the command + // resolved rather than whatever the run echoed back. + assert.Equal(t, "eval_01JQZW", out["eval_id"]) + + // Nothing the extension does not promise. A pipeline that could read the + // data source here would come to depend on it. + for _, leaked := range []string{"data_source", "metadata", "id", "report_url"} { + assert.NotContains(t, out, leaked, + "the handoff must not leak %q from the service object", leaked) + } +} + +// With --eval-id there is no config, so there is no declaration name to give. +// An empty one is omitted rather than reported as "", which a script would +// otherwise print as the eval's name. +func TestStartedRunOmitsTheNameItDoesNotHave(t *testing.T) { + raw, err := json.Marshal(startedRun( + &eval_api.OpenAIEvalRun{ID: "evalrun_1", Status: "queued"}, "eval_1", nil)) + require.NoError(t, err) + + var out map[string]any + require.NoError(t, json.Unmarshal(raw, &out)) + assert.NotContains(t, out, "eval_name") + assert.Equal(t, "eval_1", out["eval_id"]) +} From a2fd7e3f736a1f60019f18c18d951776c6e3483b Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 4 Aug 2026 02:48:35 +0300 Subject: [PATCH 094/320] Drive the CI reattach path end to end The handoff shape was covered by a unit test and nothing else, so nothing proved the id it hands back is one the next command can actually resolve - which is the only property a pipeline cares about. This starts a run without waiting, reads run_id out of the JSON, and reattaches with it. The plain-output half asserts the printed reattach line carries the real eval id and contains no angle bracket at all, because the placeholder bug this replaces was in exactly such a line and read as though it were a command. --- .../tests/cli/handoff_test.go | 85 +++++++++++++++++++ 1 file changed, 85 insertions(+) create mode 100644 cli/azd/extensions/azure.ai.evaluations/tests/cli/handoff_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/tests/cli/handoff_test.go b/cli/azd/extensions/azure.ai.evaluations/tests/cli/handoff_test.go new file mode 100644 index 00000000000..56b5465beba --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/tests/cli/handoff_test.go @@ -0,0 +1,85 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +//go:build live + +// The CI path: start a run without waiting, read the handoff, come back for +// the result later. Everything here is what a pipeline does, so it is driven +// through the binary exactly the way a pipeline would. + +package cli + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestCLIStartNoWaitEmitsTheHandoff pins the JSON a pipeline reads. +// +// A script captures the run id here and reattaches to it in a later step, so +// the field names are a contract. Emitting the service's run object instead +// would make that script depend on a shape this extension does not control. +func TestCLIStartNoWaitEmitsTheHandoff(t *testing.T) { + f := sharedEval(t) + + r := requireSuccess(t, run(t, + "run", "start", "--eval-id", f.EvalID, "--no-wait", "-o", "json")) + + var handoff struct { + RunID string `json:"run_id"` + EvalID string `json:"eval_id"` + Status string `json:"status"` + CreatedAt string `json:"created_at"` + } + r.JSON(t, &handoff) + + require.NotEmpty(t, handoff.RunID, "a pipeline has nothing to reattach to without run_id") + assert.Equal(t, f.EvalID, handoff.EvalID) + assert.NotEmpty(t, handoff.Status) + + // Started, not finished: this is the whole point of --no-wait, and a + // command that quietly blocked would pass every other assertion here. + assert.NotEqual(t, "completed", handoff.Status) + + deferTeardown(func() { + runQuietly("run", "cancel", handoff.RunID, "--eval-id", f.EvalID) + }) + + // The id it handed back has to be one the next step can use. + shown := requireSuccess(t, run(t, + "run", "show", handoff.RunID, "--eval-id", f.EvalID, "-o", "json")) + var reattached struct { + ID string `json:"id"` + } + shown.JSON(t, &reattached) + assert.Equal(t, handoff.RunID, reattached.ID, + "the run id in the handoff must be the one `run show` resolves") +} + +// TestCLIStartNoWaitTellsAPersonHowToReattach covers the same path without +// -o json, where what matters is that the printed command is one that works +// rather than a sentence containing a placeholder. +func TestCLIStartNoWaitTellsAPersonHowToReattach(t *testing.T) { + f := sharedEval(t) + + r := requireSuccess(t, run(t, "run", "start", "--eval-id", f.EvalID, "--no-wait")) + + assert.Contains(t, r.Stdout, "Reattach with: azd ai eval run show") + assert.Contains(t, r.Stdout, f.EvalID, + "the reattach line must carry the eval id, not a placeholder for it") + assert.NotContains(t, r.Stdout, "<", + "nothing printed for a person to copy may contain a placeholder") + + var runID string + for _, field := range strings.Fields(r.Stdout) { + if strings.HasPrefix(field, "evalrun_") { + runID = field + break + } + } + require.NotEmpty(t, runID, "the run id must be printed:\n%s", r.Stdout) + deferTeardown(func() { runQuietly("run", "cancel", runID, "--eval-id", f.EvalID) }) +} From 7fe99463b0e0939599760fb42a10405951e629a8 Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 4 Aug 2026 03:30:25 +0300 Subject: [PATCH 095/320] Satisfy the two repo-wide checks this extension was failing Every other extension has a cspell.yaml importing the shared config and listing its own vocabulary. This one had none, so all 96 of its words were unknown - the module name, the package names, the built-in evaluator names, and a handful of ordinary English cspell does not carry. Added, along with US spellings for the four words that were genuinely British and a format string whose %s butted against the next word and read as one. And pkg/extensions has a guard requiring every extension that declares providers to prove the ones it registers match its manifest. The provider was declared and registered correctly; the test asserting so was simply absent, so nothing would have caught them drifting apart. Also: a dataset create followed immediately by an update was refused as though the dataset did not exist. The existence check read the version listing, which lags a publish - the same thing evaluators were already reading directly - so it now falls back to a point read of version 1, which goes consistent at once. And run start --no-wait now renders created_at as RFC 3339. The service sends epoch seconds on a run and a formatted string elsewhere, so passing it through handed a script a value whose type depended on which route produced it. --- .../azure.ai.evaluations/CHANGELOG.md | 2 +- .../azd-eval-extension-implementation.md | 4 +-- .../azure.ai.evaluations/cspell.yaml | 26 +++++++++++++++++ .../internal/cmd/dataset.go | 15 ++++++++++ .../internal/cmd/generate.go | 2 +- .../azure.ai.evaluations/internal/cmd/init.go | 8 +++-- .../internal/cmd/providers_manifest_test.go | 19 ++++++++++++ .../internal/cmd/resolution_test.go | 2 +- .../azure.ai.evaluations/internal/cmd/run.go | 29 +++++++++++++++++-- .../internal/cmd/run_handoff_test.go | 22 +++++++++++++- .../internal/pkg/eval_api/operations.go | 2 +- .../internal/project/eval_config.go | 2 +- .../internal/project/generate_config.go | 2 +- .../tests/hero/init_test.go | 2 +- 14 files changed, 122 insertions(+), 15 deletions(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/cspell.yaml create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/providers_manifest_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/CHANGELOG.md b/cli/azd/extensions/azure.ai.evaluations/CHANGELOG.md index 67a8c1d121e..45903a217c0 100644 --- a/cli/azd/extensions/azure.ai.evaluations/CHANGELOG.md +++ b/cli/azd/extensions/azure.ai.evaluations/CHANGELOG.md @@ -9,7 +9,7 @@ agent, making no service calls. - `generate` synthesizes a rubric and dataset from the agent's context, writes them under `evals/`, and merges `source:` references into the deployment spec - while preserving comments, ordering and neighbouring entries. + while preserving comments, ordering and neighboring entries. - `run` creates the eval group when it does not exist, starts a run, and summarizes the result. - `azure.ai.eval` service-target provider deploys datasets, evaluators and eval diff --git a/cli/azd/extensions/azure.ai.evaluations/azd-eval-extension-implementation.md b/cli/azd/extensions/azure.ai.evaluations/azd-eval-extension-implementation.md index b5373ffc545..624f0eeef5f 100644 --- a/cli/azd/extensions/azure.ai.evaluations/azd-eval-extension-implementation.md +++ b/cli/azd/extensions/azure.ai.evaluations/azd-eval-extension-implementation.md @@ -465,10 +465,10 @@ Recorded for review. Anything marked **corrected** was an assumption that live t | **Agent-seeded generation, nothing authored** | `init` → `generate` with no instruction file: seeded from the agent's published instructions, 14 rows generated, 13 of 14 on the agent's actual catalog/policies; `azd deploy` published them; the run scored 14 passed / 0 failed / 0 errored | | Missing generation model | Fails before any network call, naming `--eval-model` and the spec field | | **`schedule` (M2)** | Create, list, show and delete against the live project; trigger read back from the service as stored, not echoed. One-per-project and existing-name refusals both verified, each naming the schedule and the command to clear it. Delete waits out `Creating` and leaves the project empty | -| **`run --from-traces` (M4)** | Accepted and executed by the service, which stored the payload and normalised `7d` into `lookback_hours: 168` while honouring `max_traces`. The run fails only because this project's agent emits no GenAI traces, and now says exactly that | +| **`run --from-traces` (M4)** | Accepted and executed by the service, which stored the payload and normalized `7d` into `lookback_hours: 168` while honouring `max_traces`. The run fails only because this project's agent emits no GenAI traces, and now says exactly that | | Failed runs | The reason reaches the caller instead of just the word "failed" | | **`run --response-id` (M4)** | Three stored responses evaluated, 3 passed / 0 errored; the stored payload matched what was sent field for field | -| Sent-vs-stored audit | Every payload compared against what the service kept. Only the trace window was actually being dropped; inline content becoming a `file_id`, and `item_schema` being normalised to `schema.item`, are both benign | +| Sent-vs-stored audit | Every payload compared against what the service kept. Only the trace window was actually being dropped; inline content becoming a `file_id`, and `item_schema` being normalized to `schema.item`, are both benign | | **`target.type: model` (M4)** | Group deployed with `response` bound to `{{sample.output_text}}`, ran, and scored 2 passed / 1 failed / 0 errored across coherence and fluency | | **Registered dataset on a run (M4)** | A group with no local `source:` now runs: whole set scores 2 passed / 1 failed, `--max-samples 2` scores 2 rows. Previously a 400 | | **Two groups in one config** | Distinct ids across repeated deploys, each running its own criteria. Previously the second deploy aliased them onto one group | diff --git a/cli/azd/extensions/azure.ai.evaluations/cspell.yaml b/cli/azd/extensions/azure.ai.evaluations/cspell.yaml new file mode 100644 index 00000000000..6e5e122ffbc --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/cspell.yaml @@ -0,0 +1,26 @@ +import: ../../.vscode/cspell.yaml +words: + # Go module and package names + - azureaieval + - evalcore + - httptest + - creack + # Service identifiers and API fields + - evalrun + - lookback + - AOAI + # Built-in evaluator names + - ifeval + - groundedness + # Repository names + - foundrysdk + # Terms + - inlines + - negotiables + - parseable + - retargeted + - subsetting + - undeployed + - undoable + - unpassed + - Unparseable diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset.go index ef4361b8cc8..f4ac919306b 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset.go @@ -16,6 +16,10 @@ import ( "github.com/spf13/cobra" ) +// firstDatasetVersion is the version the service assigns to a dataset's first +// publish, and so the one that exists for every dataset that exists at all. +const firstDatasetVersion = "1" + func newDatasetCommand() *cobra.Command { cmd := &cobra.Command{ Use: "dataset", @@ -82,6 +86,17 @@ func newDatasetWriteCommand(verb, short string) *cobra.Command { ctx, name, ProjectEndpointAPIVersion, ) exists := err == nil && existing != nil && len(existing.Value) > 0 + if !exists { + // The version listing lags a publish, so a `create` followed by + // an `update` was told the dataset it had just made does not + // exist. A direct read of the first version settles it: point + // reads go consistent immediately. + if _, err := ec.datasetClient.GetDataset( + ctx, name, firstDatasetVersion, ProjectEndpointAPIVersion, + ); err == nil { + exists = true + } + } if err := checkAssetExistence(verb, "dataset", name, exists); err != nil { return err } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go index bb3fe950310..ee7514699e8 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go @@ -285,7 +285,7 @@ func (ec *evalContext) generateDataset( return &project.ArtifactRef{Name: plan.Name, Source: relativeSource(plan.BaseDir, path)}, nil } -// isAgentSeededGenerationFailure recognises the service-side failure that hits +// isAgentSeededGenerationFailure recognizes the service-side failure that hits // every agent, so it can be retried without the agent rather than surfaced. func isAgentSeededGenerationFailure(err error) bool { if err == nil { diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go index 6e17b5f2806..2bdc4f4477f 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go @@ -5,6 +5,7 @@ package cmd import ( "context" + "errors" "fmt" "os" "path/filepath" @@ -169,7 +170,8 @@ const ( // noAzdProject is what init reports when there is nothing to attach to. const noAzdProject = "no azd project found in this directory. Run `azd init` first, " + - "or run this from the root of an existing one; the eval service is " + "or run this from the root of an existing one; the eval service is added to " + + "its azure.yaml" // readAzdProject returns the project, without changing it. // @@ -179,13 +181,13 @@ const noAzdProject = "no azd project found in this directory. Run `azd init` fir func readAzdProject(ctx context.Context) (*azdext.ProjectConfig, error) { azdClient, err := azdext.NewAzdClient() if err != nil { - return nil, fmt.Errorf("%sadded to its azure.yaml", noAzdProject) + return nil, errors.New(noAzdProject) } defer azdClient.Close() resp, err := azdClient.Project().Get(ctx, &azdext.EmptyRequest{}) if err != nil || resp.GetProject() == nil { - return nil, fmt.Errorf("%sadded to its azure.yaml", noAzdProject) + return nil, errors.New(noAzdProject) } return resp.GetProject(), nil } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/providers_manifest_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/providers_manifest_test.go new file mode 100644 index 00000000000..2588ad49b9c --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/providers_manifest_test.go @@ -0,0 +1,19 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "path/filepath" + "testing" + + "github.com/azure/azure-dev/cli/azd/pkg/azdext" + "github.com/stretchr/testify/require" +) + +// TestConfigureExtensionHostMatchesManifest verifies that the providers this +// extension registers match those declared in its extension.yaml. +func TestConfigureExtensionHostMatchesManifest(t *testing.T) { + manifestPath := filepath.Join("..", "..", "extension.yaml") + require.NoError(t, azdext.VerifyProvidersMatchManifest(configureExtensionHost, manifestPath)) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/resolution_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/resolution_test.go index 1d8c809bb6b..9e20a857312 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/resolution_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/resolution_test.go @@ -44,7 +44,7 @@ func TestResolveLevel_Precedence(t *testing.T) { // this wrong passes validation and then errors on every row. func TestSampleBindingsFor_UnknownTargetBindsNothing(t *testing.T) { assert.Nil(t, sampleBindingsFor("prompt"), - "an unrecognised target must bind nothing rather than guess at agent fields") + "an unrecognized target must bind nothing rather than guess at agent fields") } // The level filter is what keeps a conversation evaluator from being sent turn diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go index 67c26bf6bae..a135a968454 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go @@ -655,7 +655,7 @@ type startedRunHandoff struct { EvalID string `json:"eval_id"` EvalName string `json:"eval_name,omitempty"` Status string `json:"status,omitempty"` - CreatedAt any `json:"created_at,omitempty"` + CreatedAt string `json:"created_at,omitempty"` } // startedRun builds the handoff. @@ -668,7 +668,7 @@ func startedRun( RunID: run.ID, EvalID: evalID, Status: run.Status, - CreatedAt: run.CreatedAt, + CreatedAt: timestampString(run.CreatedAt), } // Absent with --eval-id, where there is no config to take a name from. if group != nil { @@ -677,6 +677,31 @@ func startedRun( return handoff } +// timestampString renders a service timestamp as RFC 3339. +// +// The field arrives as epoch seconds on a run and as a formatted string +// elsewhere, so passing it through would hand a script a value whose type +// depends on which route produced it. +func timestampString(value any) string { + switch t := value.(type) { + case nil: + return "" + case string: + return t + case float64: + return time.Unix(int64(t), 0).UTC().Format(time.RFC3339) + case int64: + return time.Unix(t, 0).UTC().Format(time.RFC3339) + case json.Number: + if seconds, err := t.Int64(); err == nil { + return time.Unix(seconds, 0).UTC().Format(time.RFC3339) + } + return t.String() + default: + return fmt.Sprint(value) + } +} + // renderRun prints what a person needs after waiting for a run. // The status line alone is not that. A run's whole purpose is the verdict per // evaluator, and the service returns it — passed, failed and errored counts diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_handoff_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_handoff_test.go index cf4a522c2f5..2881096749c 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_handoff_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_handoff_test.go @@ -52,7 +52,27 @@ func TestStartedRunIsTheHandoffAPipelineNeeds(t *testing.T) { } } -// With --eval-id there is no config, so there is no declaration name to give. +// A script logging created_at should not have to know which route produced +// the run: the service sends epoch seconds here and a formatted string +// elsewhere, so the handoff settles on one. +func TestStartedRunNormalizesTheTimestamp(t *testing.T) { + for _, tc := range []struct { + name string + value any + want string + }{ + {"epoch seconds", float64(1785801525), "2026-08-03T23:58:45Z"}, + {"already formatted", "2026-07-31T21:04:11Z", "2026-07-31T21:04:11Z"}, + {"absent", nil, ""}, + } { + t.Run(tc.name, func(t *testing.T) { + handoff := startedRun( + &eval_api.OpenAIEvalRun{ID: "evalrun_1", CreatedAt: tc.value}, "eval_1", nil) + assert.Equal(t, tc.want, handoff.CreatedAt) + }) + } +} + // An empty one is omitted rather than reported as "", which a script would // otherwise print as the eval's name. func TestStartedRunOmitsTheNameItDoesNotHave(t *testing.T) { diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/operations.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/operations.go index 09291317d10..f28b8e81ef5 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/operations.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/operations.go @@ -244,7 +244,7 @@ func (p publishedVersion) writtenAt() time.Time { // definition and every eval bound to the first silently scoring against a // rubric nobody chose. // -// So there are two defences. The publish is held back until the version the +// So there are two defenses. The publish is held back until the version the // caller read has had time to settle, which is what keeps the collision from // happening at all; and the version that comes back is checked, which is what // keeps a collision that happens anyway from being reported as success. The diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go index e2ff2e74772..ec58282de06 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go @@ -21,7 +21,7 @@ import ( // kept in evals/.yaml and pulled in with $ref. // // The eval's name is the service key in azure.yaml and is not repeated here. -// One service per eval is what lets azd's own dependency graph order an eval +// One service per eval is what lets the azd dependency graph order an eval // after the agent it evaluates. type EvalConfig struct { Description string `yaml:"description,omitempty" json:"description,omitempty"` diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/generate_config.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/generate_config.go index 7ab28e5d7ac..12fdb1cf6ff 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/generate_config.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/generate_config.go @@ -158,7 +158,7 @@ func ArtifactPath(baseDir, outputDir, resourceName, ext string) string { return filepath.Join(candidate, resourceName+ext) } -// looksLikeFile treats a trailing recognised extension as an explicit file path. +// looksLikeFile treats a trailing recognized extension as an explicit file path. func looksLikeFile(p, ext string) bool { got := strings.ToLower(filepath.Ext(p)) if got == "" { diff --git a/cli/azd/extensions/azure.ai.evaluations/tests/hero/init_test.go b/cli/azd/extensions/azure.ai.evaluations/tests/hero/init_test.go index 0618e49ccd8..8fe79718d53 100644 --- a/cli/azd/extensions/azure.ai.evaluations/tests/hero/init_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/tests/hero/init_test.go @@ -256,7 +256,7 @@ func TestHeroInitWiresTheServiceIntoTheProject(t *testing.T) { } // Running `init` twice must not deploy the same eval twice. The service key is -// the eval's name, so the second run recognises its own work. +// the eval's name, so the second run recognizes its own work. func TestHeroInitIsIdempotent(t *testing.T) { dir := project(t, "support-agent") args := []string{"init", "--target", "support-agent", "--generation-model", "m"} From bb5362ec784f293b56b079623b90ab670097fad1 Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 4 Aug 2026 04:01:58 +0300 Subject: [PATCH 096/320] Keep a command from shadowing the flags M1 promises everywhere M1's exit criteria include \-o json\ and \--no-prompt\ throughout. Both come from the SDK's root command, so every command inherits them and nothing had to be checked - right up until a command declares a flag by one of those names, which shadows the global silently and leaves that one command unable to answer in JSON or to run unattended. Nothing was watching for that. --- .../internal/cmd/surface_test.go | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/surface_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/surface_test.go index acaaf5143a0..3e3f9e035be 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/surface_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/surface_test.go @@ -131,6 +131,21 @@ func TestFlagVocabularyIsShared(t *testing.T) { }) } +// M1 promises `-o json` and `--no-prompt` throughout. Both come from the azd +// extension SDK's root command, so every command inherits them — until one +// declares its own flag by the same name, which silently shadows the global +// and leaves that one command unable to answer in JSON or to run unattended. +func TestNoCommandShadowsAGlobalFlag(t *testing.T) { + global := []string{"output", "no-prompt", "environment", "cwd", "debug"} + + walk(t, NewRootCommand(), nil, func(path string, cmd *cobra.Command) { + for _, name := range global { + assert.Nilf(t, cmd.LocalFlags().Lookup(name), + "%s declares its own --%s, which shadows the global one", path, name) + } + }) +} + // The two commands that write a file have to agree on what that flag is // called, and it has to be the name the sibling extensions use. func TestOutputFileFlagIsSpelledTheSharedWay(t *testing.T) { From d2b4d19b3b448ecb9c76619cc1ca15650db85e6e Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 4 Aug 2026 17:11:08 +0300 Subject: [PATCH 097/320] Prove a dataset version cannot be written over, and drop a working note from the repo The reconciler publishes at a pinned version and treats a conflict as the signal to stop, which only holds if the service refuses the write. Nothing checked that. It does refuse - 409 on the pending upload - so the assumption was sound, but it was an assumption about someone else's service with an eval's scoring hanging off it. Now it fails loudly if that ever changes. The companion test covers the ordinary path: two uploads in a row must add two versions, issued back to back because the version listing lags a publish and that is the window where the second could be told the dataset is new and restart at the version the first just took. Also removes azd-eval-extension-implementation.md. It was a working note from building this, not documentation anyone should find in the extension - and it had gone stale, still calling M2's features M4. --- .../azd-eval-extension-implementation.md | 543 ------------------ .../internal/cmd/dataset_version_live_test.go | 115 ++++ 2 files changed, 115 insertions(+), 543 deletions(-) delete mode 100644 cli/azd/extensions/azure.ai.evaluations/azd-eval-extension-implementation.md create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset_version_live_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/azd-eval-extension-implementation.md b/cli/azd/extensions/azure.ai.evaluations/azd-eval-extension-implementation.md deleted file mode 100644 index 624f0eeef5f..00000000000 --- a/cli/azd/extensions/azure.ai.evaluations/azd-eval-extension-implementation.md +++ /dev/null @@ -1,543 +0,0 @@ -# `azd ai eval` — Implementation Handoff - -> Self-contained brief for building the **`azure.ai.evaluations`** azd extension (`azd ai eval`). Everything below is verified against the shipping `azure.ai.agents` extension, azd core, and RAISvc source. Design source of truth is `spec.md` in `foundrysdk_specs/specs/evaluations/azd_eval_extension/` — if the two disagree, the spec wins. -> -> **Uncommitted working document.** Not part of any PR. - ---- - -## 0. TL;DR - -A new azd extension — id `azure.ai.evaluations`, namespace `azd ai eval` — that is a thin Go client over the **existing** Foundry evaluations data plane, plus **one azd service-target provider** for `host: azure.ai.evals`. - -**Non-negotiables** - -1. **Two-tier commands.** Atomic (`dataset` / `evaluator` / `run` / `results`) map ~1:1 to the API. Composite (`init` / `generate` / `run`) are wrappers, never the only path. -2. **No `deploy` command.** Deployment is `azd up` / `azd deploy` invoking our service-target provider. We ship no deploy verb. -3. **`generate` is separate from deploy.** Generate once, deploy the artifacts to many environments. -4. **`init` touches no network.** -5. **Everything is built on APIs that exist today.** No service changes. -6. **Deterministic.** `-o json` + `--no-prompt` everywhere; a supplied flag fully suppresses its prompt. - -**Out of scope for M1:** scheduled/continuous eval, baseline comparison (both exist server-side — M2), non-agent targets, traces as a run data source, eval by response/run id, `optimize` (stays in the agents extension). - ---- - -## 1. Where the code lives & how to build it - -### 1.1 Paths -- **Repo:** `Azure/azure-dev`. -- **New extension:** `cli/azd/extensions/azure.ai.evaluations/` -- **Reference to copy:** `cli/azd/extensions/azure.ai.agents/` — closest analog, currently hosts `azd ai agent eval …`. **Read it first.** - -### 1.2 File layout (mirror the agents extension) -``` -cli/azd/extensions/azure.ai.evaluations/ -├── internal/ -│ ├── cmd/ # cobra commands, one file per group; listen.go wires the provider -│ ├── pkg/eval_api/ # data-plane client (lifted, see §1.4) -│ ├── pkg/dataset_api/ # dataset client (lifted) -│ └── project/ # azure.yaml service-entry model + YAML round-trip -├── schemas/ # JSON schemas -├── tests/ -├── extension.yaml # manifest -├── go.mod / go.sum -├── main.go -└── version.txt -``` - -**`extension.yaml`** — note the `service-target-provider` capability and `providers` block; both are required for `azd up` to route to us: -```yaml -# yaml-language-server: $schema=../extension.schema.json -id: azure.ai.evaluations -namespace: ai.eval # dotted → CLI surface `azd ai eval` -displayName: Foundry evaluations (Beta) -description: Define and run Foundry evaluations from your terminal. (Beta) -usage: azd ai eval [options] -version: 1.0.0-beta.1 # keep version.txt in sync -requiredAzdVersion: ">=1.27.1" -language: go -capabilities: - - custom-commands - - lifecycle-events - - service-target-provider - - metadata -providers: - - name: azure.ai.evals - type: service-target - description: Deploys evaluation datasets, evaluators, and eval groups to Foundry -``` - -### 1.3 Stack facts (verified) -- **Go 1.26.x**, **cobra**. Entry point: - ```go - package main - import ( - "azureaieval/internal/cmd" - "github.com/azure/azure-dev/cli/azd/pkg/azdext" - ) - func main() { azdext.Run(cmd.NewRootCommand()) } - ``` -- SDK module `github.com/azure/azure-dev/cli/azd` (agents pins `v1.28.0`); surface is `pkg/azdext`. -- Data plane over REST using the **azcore pipeline**, not raw `net/http`: bearer-token policy scoped to **`https://ai.azure.com/.default`**, plus `azsdk.NewMsCorrelationPolicy()` and `azsdk.NewUserAgentPolicy(...)`. -- Dev loop: - ```bash - azd ext install microsoft.azd.extensions # one-time - cd cli/azd/extensions/azure.ai.evaluations - azd x build # build + install locally - azd x watch # ongoing - ``` - -### 1.4 What to lift from `azure.ai.agents` - -Measured, non-test: - -| Source | Files | LOC | Gives you | -|---|---|---|---| -| `internal/pkg/agents/eval_api/` | 7 | 1,350 | `EvalClient` over `/data_generation_jobs`, `/evaluator_generation_jobs`, `/evaluators`, `/datasets`, `/openai/v1/evals`; LRO poller; artifact download; portal URLs; api-version constants | -| `internal/pkg/agents/dataset_api/` | 2 | 550 | Full pending-upload → blob → finalize → download | -| `internal/pkg/agents/opt_eval/` | 2 | 555 | eval.yaml config model (adapt, don't copy wholesale) | -| `internal/cmd/eval_*.go` | 10 | 2,890 | generate / run / show / list / update / progress UX | - -**≈1,900 LOC of API client is effectively done.** Genuinely net-new: - -1. **The service-target provider** (§4) — no prior art. -2. **Change detection** (§5) — no prior art. -3. **YAML round-trip merge** for `generate` writing `source:` back (§6). -4. The atomic command layer and the offline `init`. - -> Do **not** port the `/evaluation_suites` client. That endpoint is abandoned — the eval group is the unit. - ---- - -## 2. Data-plane API contract - -### 2.1 Base, auth, api-versions -- **Base:** azd env `FOUNDRY_PROJECT_ENDPOINT`, shape `https://{resource}.services.ai.azure.com/api/projects/{project}/…`. Global `--project-endpoint` overrides. -- **Scope:** `https://ai.azure.com/.default`. -- **api-versions:** project-endpoint calls (datasets, evaluators) use **`2025-11-15-preview`**; data generation uses **`v1`**; **`/openai/v1/evals*` sends no api-version**. - -### 2.2 Datasets -| Command | Calls | -|---|---| -| `dataset create` / `update` | `POST /datasets` *(first version only)* → `POST /datasets/{name}/versions/{v}/startPendingUpload` → `PUT ` → `PUT /datasets/{name}/versions/{v}` | -| `dataset list` | `GET /datasets` | -| `dataset show` | `GET /datasets/{name}/versions/{v}` | -| `dataset delete` | `DELETE /datasets/{name}/versions/{v}` | - -Model — **note there is no content hash or etag**, which drives §5: -```go -type Dataset struct { Name, Version, BlobURI, Format, DataURI, ContentURI string } -``` -A dataset is a **single `.jsonl`**. A directory today just picks the first `.jsonl`; no folder walk. - -### 2.3 Evaluators -| Command | Calls | -|---|---| -| `evaluator upload` / `update` | *(code only)* pending-upload → blob upload; then `POST /evaluators/{name}/versions` | -| `evaluator show` | `GET /evaluators/{name}` — returns the definition inline | -| `evaluator builtins` | `GET /evaluators?type=Builtin` | - -Built-ins are referenced as `builtin.`; the prefix is stripped before the value goes into `testing_criteria[].evaluator_name`. Custom-evaluator upload needs the project MI to hold **Azure AI User**. - -### 2.4 Eval groups and runs (OpenAI-compatible, no api-version) -```go -type CreateOpenAIEvalRequest struct { - Name string - Metadata map[string]string - DataSourceConfig *DataSourceConfig // {Type, ItemSchema, IncludeSampleSchema} - TestingCriteria []TestingCriterion // the evaluators -} -type TestingCriterion struct { - Type, Name, EvaluatorName string - InitializationParameters map[string]any // threshold lives here - DataMapping map[string]string -} -type OpenAIEval struct { ID, Name string } // ID is canonical; Name is NOT unique -``` - -| Command | Calls | -|---|---| -| create group | `POST /openai/v1/evals` | -| get / list | `GET /openai/v1/evals/{id}` · `GET /openai/v1/evals?limit=` | -| start run | `POST /openai/v1/evals/{evalId}/runs` | -| poll / list runs | `GET /openai/v1/evals/{evalId}/runs/{runId}` · `GET …/runs` | -| cancel | `POST /openai/v1/evals/{evalId}/runs/{runId}` with an empty body | -| results | `GET …/runs/{runId}` → `result_counts` + `per_testing_criteria_results` | - -**The group carries evaluators, not the dataset.** The dataset goes on the **run**. `evaluation_level` is `turn` | `conversation`, service default **`turn`**. - -**`data_source_config` and `data_mapping` are derived from each evaluator's published contract.** The original plan was to copy the agents extension's hardcoded mapping. Live testing showed that is wrong: it only suits agent-target quality evaluators and the service rejects the rest. - -`GET /evaluators` returns a contract per evaluator: -```jsonc -"supported_evaluation_levels": ["turn"], -"definition": { - "data_schema": { "required": ["response", "instruction_id_list"], "properties": { … } }, - "init_parameters": { "required": ["deployment_name"], "properties": { … } } -} -``` - -`internal/cmd/build.go` reads it and, per criterion: -- binds each accepted input to the agent sample (`response`, `tool_calls`, `tool_definitions`) or to a dataset column `{{item.}}`; -- declares the referenced columns in the item schema; -- filters `initialization_parameters` to the declared properties — no evaluator accepts `model`, and `builtin.ifeval` accepts nothing; -- validates `--level` against `supported_evaluation_levels`; -- reports a missing required column locally, naming it. - -Two service rules are encoded: `messages` and `query`/`response` are mutually exclusive (the level selects), and `evaluation_level` is an **initialization parameter**, not run metadata. An evaluator with no published contract falls back to the agent-target shape. - -Covered by `build_test.go`, and by `build_live_test.go` which posts a group for every built-in the project exposes. - -### 2.5 Generation (LRO) -`POST /data_generation_jobs` and `POST /evaluator_generation_jobs`, each polled by `GET …/{id}`. The ~11-minute "timeout" is a **client poll budget (2 s × 300)**, not a service limit — raise it and default to `--no-wait` in CI. - -### 2.6 M2 only — schedules and comparison -Both are **project-endpoint reachable** and **feature-gated per project**: -- `/schedules` — `PUT {id}` · `GET {id}` · `GET` · `DELETE {id}`; requires `FoundryFeature.Schedules_V1Preview`. Trigger is `Cron{Expression, StartTime, EndTime, Timezone}` or `Recurrence{Frequency, Interval, Schedule}`. -- Insights compare — `POST /insights` (async) or `POST /insights/sync`, body `{evalId, baselineRunId, treatmentRunIds}`; requires `FoundryFeature.Insights_V1Preview`. - ---- - -## 3. Configuration model - -Two files. Neither is loaded by azd core — **we parse both**. - -**`evals/eval_generate.yaml`** — input to `generate`, never deployed. `agent.context.{instructions,tools}` are file paths; `local_dir` accepts a directory or an explicit file path. - -**`evals/azure.yaml`** — the deployment spec, `$ref`'d from the root `azure.yaml`: -```yaml -# /azure.yaml -services: - evals: - host: azure.ai.evals - uses: [ai-project] - $ref: ./evals/azure.yaml -``` -It carries three arrays: `evaluators[]`, `datasets[]`, `evalGroups[]` (see `spec.md` for the full shape). - -**Why arrays on a service work.** azd core's `ServiceConfig` captures unknown keys: -```go -AdditionalProperties map[string]any `yaml:",inline"` -``` -and hands them to the extension, which unmarshals them itself — the pattern `LoadServiceTargetAgentConfig` → `ServiceConfigProps` uses. `azure.ai.project` already carries `deployments[]` this way. **`$ref` resolution is ours too**: `pkg/foundry.ResolveFileRefs(cfg, projectRoot)`, called by the extension, not by azd. - ---- - -## 4. The service-target provider (net-new, highest risk) - -Wire it in `listen.go`, mirroring `azure.ai.agents`: -```go -func configureExtensionHost(host *azdext.ExtensionHost) { - azdClient := host.Client() - host. - WithServiceTarget("azure.ai.evals", func() azdext.ServiceTargetProvider { - return project.NewEvalServiceTargetProvider(azdClient) - }). - WithServiceEventHandler("postdeploy", func(ctx context.Context, args *azdext.ServiceEventArgs) error { - return postdeployHandler(ctx, azdClient, args) - }, &azdext.ServiceEventOptions{Host: "azure.ai.evals"}) -} -``` - -`ServiceTargetProvider` requires `Initialize`, `Endpoints`, `GetTargetResource`, `Package`, `Publish`, `Deploy`. For eval, **`Package` and `Publish` are near no-ops**; `Deploy` does the work, in this fixed order: - -1. **Datasets** — change-detect (§5); if changed, run the `dataset create` sequence. -2. **Evaluators** — `GET /evaluators/{name}`, compare the definition, upload only if different. -3. **Drift check** — if the server's latest version is ahead of the recorded one, fail with "sync first". -4. **Eval groups** — `POST /openai/v1/evals` with `testing_criteria` from the resolved evaluator versions. Groups are immutable, so only recreate when the resolved versions or options actually changed. -5. Persist resolved ids, versions, and fingerprints to the azd env. - -**Publish → reference is eventually consistent.** After an evaluator version is -published, the direct read `GET /evaluators/{name}/versions/{version}` goes -consistent in roughly 330 ms, but the version *listing* lags 650 ms–1.4 s — and -eval creation resolves the reference through the listing, not the direct read. -Step 4 has to gate on both, or it fails with *"The evaluator X was not found"* -just after successfully publishing that evaluator. It reads like a flake, -because retrying a second later succeeds, so the tempting fix is a blanket retry -rather than the wait. - -**How azd reaches us:** `azd up` runs one DAG; per service it calls `GetServiceTarget()`, which does `serviceLocator.ResolveNamed(host, &target)`. If our extension is not installed, azd fails that service with *"install an extension that provides this host."* We implement **no sequencing or rollback across services** — `uses:` and the DAG handle that. - ---- - -## 5. Change detection (net-new) - -Without this, every `azd up` publishes a redundant version. - -- **Datasets** — the API returns no hash or etag, so comparing against the server would mean downloading the blob every deploy. Instead: **SHA-256 the local file**, store it with the resolved version in the azd env, re-hash locally next deploy, skip when unchanged. -- **Evaluators** — definitions come back inline from `GET /evaluators/{name}`; compare directly, no cache needed. -- **Drift** — a *version* comparison, not content: server latest vs. the version recorded at last deploy. - -**Open:** how to fingerprint a **code** evaluator (a folder). Suggest hashing sorted relative paths + contents, excluding `__pycache__` and `.pyc`. - ---- - -## 6. `generate` writes back into `evals/azure.yaml` - -After downloading artifacts, `generate` adds/updates `source:` references. Requirements: - -- Match entries **by `name`**; update `source` in place; append when absent. -- **Preserve comments and key order** — use the `yaml.v3` Node API, not plain marshal/unmarshal. -- Do not clobber a field the user hand-edited other than `source`. -- If the array is itself a `$ref`, write into the referenced file. - -There is no `emitDeploymentConfig` block — this is default behavior, not configurable. - ---- - -## 6a. What generation is seeded from - -The generation API takes an `agent` source that is meant to pull the agent's own instructions, and it fails for every agent (§11d). The client resolves that context itself, most specific first: - -1. `--gen-instruction` / `--gen-instruction-file` -2. the file named by `agent.context.instructions`, resolved **relative to the spec that declared it**, not the working directory -3. the agent's published instructions — `GET /agents/{name}` → `versions.latest.definition.instructions` - -Step 2 tolerates a missing file on purpose: `init` writes the path before the file exists, so treating the gap as an error would break the flow init scaffolds. Step 3 is what makes `init` → `generate` work with nothing authored. - -The agent source is still sent. When the service starts honouring it, it contributes on top of the prompt; nothing has to be removed. - -`agent.context.tools` is still read by nothing, so it is warned about rather than dropped silently, and `init` no longer scaffolds it — a warning for a field the user never chose is just noise. - ---- - -## 7. Behavioral bugs to fix (measured in `azd ai agent eval`) - -Treat each as an acceptance criterion. - -1. **Path handling (highest priority).** `--out-file` is re-rooted under the agent directory; `--config` re-roots again. **Fix:** treat paths as relative to CWD (or `-C/--cwd`), used verbatim, single-rooted. Test `./x.yaml`, `../x/x.yaml`, absolute. -2. **Wizard overrides flags.** Prompts still fire when flags are supplied, and pre-filled prompts *append* typed input. **Fix:** a supplied flag fully suppresses its prompt; `--no-prompt` errors on a missing required value. -3. **`--evaluator` ignored during generation.** Passing `--evaluator` does not stop rubric generation. **Fix:** honor it, skip that generation. -4. **Client-side generation timeout.** Resolved: it is a client poll budget, not a service limit. Raise it; default `--no-wait` under `--no-prompt`. -5. **Shallow results.** `eval show` returns counts only. **Fix:** per-sample scores via `per_testing_criteria_results`. -6. **Auth friction.** Native azd token failed with "Reauthentication required"; workaround `azd config set auth.useAzCliAuth true`. Detect and surface clearly. - ---- - -## 8. azd environment - -Extensions read and write env values themselves via `azdClient.Environment().GetValue / SetValue` — azd sets none of these. The agents extension does this from lifecycle handlers. - -| Key | Written by | -|---|---| -| `FOUNDRY_PROJECT_ENDPOINT` | consumed, not written | -| `EVAL_GROUP_ID` | provider during `azd up`; `run` when it creates the group | -| `EVAL_DATASET_VERSION`, artifact fingerprints | provider during `azd up` | -| `EVAL_RUN_ID` | `run` | - -Setting `EVAL_GROUP_ID` manually targets a pre-existing group; `--eval-id` does the same per-invocation. - ---- - -## 9. Build order - -M1 is everything in the spec. Within it, build in dependency order: - -| Step | Work | Done when | -|---|---|---| -| **1. Scaffold** | Extension skeleton, `extension.yaml`, `main.go`, root cobra command, local install via `azd x build` | `azd ai eval --help` works | -| **2. Lift the clients** | Copy `eval_api` + `dataset_api`, de-agent-scope, keep api-version constants | Unit tests pass against an `httptest` fake | -| **3. Atomic commands** | `dataset`, `evaluator`, `run`, `results` with `-o json` / `--no-prompt` | **E2E-1** below | -| **4. Config model** | `evals/azure.yaml` load, `$ref` resolve via `pkg/foundry.ResolveFileRefs`, validation | Round-trip test preserves comments | -| **5. Service-target provider** | `listen.go` wiring + `Deploy` reconciliation + change detection + drift | `azd up` creates all three resource kinds; second `azd up` is a no-op | -| **6. `init`** | Offline scaffold of both YAMLs | Runs with no network/auth | -| **7. `generate`** | Generation LROs, artifact download, write-back into `evals/azure.yaml` | **E2E-3** | -| **8. `run`** | Group resolve-or-create, run, poll, render | **E2E-2** | - -Steps 1–3 are mostly mechanical. **Step 5 is the risk** — budget accordingly. - ---- - -## 10. Testing - -| Tier | Coverage | Auth | Where | -|---|---|---|---| -| **0 — offline** | flag parsing; YAML round-trip; path resolution (§7.1); flag→prompt suppression (§7.2); request bodies against an `httptest` fake (copy `eval_api_version_test.go`); schema validation | No | PR gate | -| **1 — `init` record/playback** | interactive prompt flows | No | PR gate | -| **2 — live golden path** | full flows against a real Foundry project | Yes | On-demand/scheduled, **not** the PR gate | - -Tier 2: env-gate on `AZURE_AI_EVAL_E2E_LIVE=1`, build tag `//go:build linux` (needs a PTY), drive `init` prompts via `go-expect`+`vt10x`+`creack/pty`, everything else through `--no-prompt -o json`. `t.Cleanup` must delete every version it created. - -**Golden paths** - -- **E2E-1 — atomic:** `dataset create` → `evaluator upload` → `run start` → `results export`. Assert valid JSON, resolved versions, terminal run status, **per-sample** scores, and that re-running `create` yields the *next* version rather than an error. -- **E2E-2 — init → azd up → run:** `init` makes **zero network calls** (run it unauthenticated), writes both YAMLs, does not double the path; `azd up` creates the resources and pins versions back; a second `azd up` creates **no new versions**; `run` completes with no prompt. -- **E2E-3 — generate:** completes without a client timeout; a supplied `--evaluator` is honored; artifacts land locally and `evals/azure.yaml` gains correct `source:` entries with comments preserved. -- **E2E-4 — CI invariants:** every command with `--no-prompt -o json` is non-interactive, emits parseable JSON, and exits non-zero on a missing required value. - ---- - -## 11. Decisions still open - -| # | Question | Blocks | Suggested default | -|---|---|---|---| -| 1 | Host name `azure.ai.evals` — agreed? Not registered anywhere yet | Step 1 | Use it; renaming is cheap before publish | -| 2 | Which storage connection for evaluator pending-upload (`connectionName`) | Step 3 | Project default; expose a flag | -| 3 | Are **code** evaluators in M1, or rubric-only? | Steps 3, 5 | Rubric-only for M1 — removes the folder-hashing problem entirely | -| 4 | Where do fingerprints live — azd env or a lock file? | Step 5 | azd env, so they are environment-scoped | -| 5 | Do we depend on `azure.ai.projects` for the project service? | Step 1 | Yes, mirror the agents manifest | -| 6 | Bundling into `microsoft.foundry` — who owns it | Ship | Extensions team | - ---- - -## 11b. Assumptions made while implementing - -Recorded for review. Anything marked **corrected** was an assumption that live testing disproved; the code already reflects the correction. - -| # | Assumption | Status | -|---|---|---| -| 1 | Dataset versions are decimal (`1.0`, `2.0`) | **Verified live** — `UploadNewVersion` advanced 1.0 → 2.0 | -| 2 | `--wait` defaults true for `run` | Held; matches the spec's blocking-by-default UX | -| 3 | `evaluation_level` travels as run **metadata** | **Corrected** — it is an `initialization_parameters` property on evaluators that declare it. Metadata had no effect | -| 4 | A cached eval group id that 404s means recreate | Held; not yet exercised live | -| 5 | `dataset create` accepts a file or a directory | Held; the upload helper scans a directory for the first `.jsonl` | -| 6 | Evaluator sameness compares only the `definition` block | Held; avoids server-assigned version/timestamp churn | -| 7 | `GetTargetResource` returns a subscription-only resource | Held; eval resources have no ARM resource | -| 8 | Rubric evaluators only in M1; code evaluators in M2 | Open decision 3 | -| 9 | One fixed data mapping suits all evaluators | **Corrected** — contracts differ per evaluator; the mapping is now derived from the published contract | -| 10 | Dataset URIs come back snake_case | **Corrected** — the project endpoint returns camelCase (`dataUri`). Both spellings are now bound | -| 11 | A dataset blob URI can be downloaded directly | **Corrected** — true only for uploads. A *generated* dataset's URI names the container, not the blob, with `isSingleFile` true either way, and downloading a container returns 409. The URI also carries no SAS, so a credential is always needed. Downloads now fetch a credential and list the container when the URI does not name a file | -| 22 | Agent-seeded data generation would be fixed service-side before ship | **Corrected** — traced to the AOAI generator, outside this repo. `generate` now reads the agent's instructions itself and passes them as the prompt source; the agent source is still sent so it contributes once fixed | -| 23 | `agent.context.instructions` and `.tools` were wired up | **Corrected** — both were written by `init`, declared on the config, and read by nothing. `instructions` is now honoured; `tools` is warned about and no longer scaffolded | -| 24 | The service would reject a missing generation model clearly | **Corrected** — it fails partway through the command with a message naming nothing the caller controls. Checked up front instead | -| 25 | `max_samples` was free-form | **Corrected** — the service requires 15–1000. The config already validated this; the floor is now documented in the spec | -| 26 | Schedule creation would be a POST to a collection | **Corrected** — `POST` 404s on every route. It is `PUT /schedules/{name}`, a named resource | -| 27 | The bodiless 400s meant `displayName`/`description`/`enabled` were required | **Corrected, and this one was my error** — in that probe only the *first* create succeeded and I read the rest as field validation. The real cause is one schedule per project. Re-tested from a drained state, a minimal body creates fine | -| 28 | A named PUT would update in place | **Corrected** — accepted, echoes the new body, changes nothing. `set` refuses an existing name instead of reporting a change that did not happen | -| 29 | Deleting and recreating under the same name would work as a replace | **Corrected** — the replacement never leaves `Creating` and cannot then be deleted. The `--replace` flag was removed before shipping | -| 30 | M4's "traces as a run data source" was awaiting service support | **Corrected** — `azure_ai_traces` is in the run data-source discriminator and the service executes it. The note was never re-tested. Shipped as `run --from-traces` | -| 31 | Traces were a generation input only, never a run's data source | **Corrected** — that comment described the *generation* API. The run API takes them directly | -| 32 | The traces window could be sent as `start_time`/`end_time` | **Corrected, and this one I shipped** — the data source has no start bound. `start_time` is accepted and discarded, leaving the default 7 days. It looked right only because the first value I tested, 7d, *is* the default; 30d silently queried a week. Now sends `lookback_hours` | -| 33 | M4's "evaluation by response id" was awaiting service support | **Corrected** — works today. The ids are not a list on the data source: they are JSONL rows plus a `data_mapping` to `response_id` | -| 34 | `target.type: model` was unsupported | **Corrected** — supported. The config rejected it by name *and* the test used it as the example of an unsupported type, so the gap read as deliberate in two places. Sample bindings now follow the target kind, since a model returns `output_text` where an agent returns `output_items` | -| 35 | A run could reference a registered dataset by name as a `file_id` | **Corrected** — `file_id` means an uploaded file; a dataset name is rejected with `invalid data source file ids`. Registered datasets are fetched and sent inline. Every earlier test used a local `source:`, so this path had never run | -| 36 | M4's "subsetting a registered dataset" needed service support | **Corrected** — the service cannot narrow a file reference, but fetching the rows client-side makes `--max-samples` mean the same thing for any dataset | -| 37 | One env key per resolved id was enough | **Corrected** — only true for a single-group config. With two, the second deploy handed the first group the second's id and both declarations pointed at one group. Ids are now keyed by name, as fingerprints already were | -| 38 | The remembered run id could be shared | **Corrected** — same shape as 37. Asking group A for its latest fetched group B's run inside A and 404'd | -| 39 | A dataset's `version:` was the version published | **Corrected** — it was passed to the helper that *counts from* its argument, so `1.0` published 2.0. It also meant two things: unchanged content resolved to it, changed content published above it | -| 40 | An evaluator's `version:` behaved like a dataset's | **Corrected** — the service assigns an evaluator's version on publish, so a pin alongside `source:` was never honoured. A config asking for 7 deployed 1 silently. Now refused | -| 41 | Criteria were being shaped from each built-in's published schema | **Corrected, and this one invalidated an earlier §11c row** — the schemas were fetched with an unfiltered list, which returns only the project's own evaluators. Every built-in fell back to `legacyInputs`. It matched query/response so nothing looked wrong; `task_completion` at conversation level published an empty `data_mapping` | -| 42 | A run needs a target | **Corrected** — a dataset holding both sides of the exchange has nothing to invoke, and the service runs it. The requirement was ours | -| 12 | `$ref` is resolved by azd core before the extension sees the config | **Corrected** — core leaves `$ref` for the owning extension. The provider now calls `foundry.ResolveFileRefs`, and relative `source:` paths are based on the included file's directory | -| 13 | Upstream artifact fingerprints are enough to know when to recreate a group | **Corrected** — editing the group's own target/evaluators/options changed nothing. The group declaration is fingerprinted too | -| 14 | The host is `azure.ai.evals` | **Corrected** — it is `azure.ai.eval`; the spec has been aligned | -| 15 | `run` only needed the composite form | **Corrected** — the spec lists `start`/`list`/`show`/`cancel`, and M1 requires every operation to be reachable atomically. All four now exist | -| 16 | `--project-endpoint` only selects the endpoint | **Corrected** — it also suppressed the azd environment name, silently disabling the cached eval-group and run ids. The name is now resolved independently | -| 17 | No evaluator accepts `model` | **Corrected** — true for built-ins, false for custom rubrics, which *require* `model`. The judge model is bound under whichever name the evaluator declares | -| 18 | An evaluator definition can be compared whole to detect changes | **Corrected** — the service enriches it on create, so only the authored keys can be compared | -| 19 | `GET /evaluators/{name}` returns the latest version | **Corrected** — it 404s; the version has to be resolved first, numerically | -| 20 | Rubric weights are free-form | **Corrected** — integers 1–10; the spec now says so | -| 21 | `--dataset` suppresses data generation | **Corrected** — only did so for a local path, not for a registered dataset name, which the flag also accepts | - ---- - -## 11c. Verified end to end against a live project - -| Flow | Result | -|---|---| -| `azd ai eval init` | Scaffolds `evals/azure.yaml` + `evals/eval_generate.yaml` matching the spec | -| `azd provision` → `azd deploy evals` → `azd up` | Provider runs; datasets and groups reconcile | -| Dataset first deploy | Published at version 1.0 | -| Dataset unchanged | Reported unchanged, nothing uploaded | -| Dataset edited | Published 2.0 and the group recreated | -| Group retargeted | New group id; two further no-op deploys reused it | -| `$ref` service entry | Deploys, and the fingerprint matches the equivalent inline config | -| `azd ai eval run` | Real run against a live agent, completed | -| `azd ai eval results show` | 3 passed / 1 failed, per-criterion breakdown, portal link | -| `evaluator builtins` | 10 built-ins with versions and type | -| Eval group create | Accepted for **all 10** built-ins, each with its own contract || `generate` rubric | Succeeds, writes the evaluator JSON | -| Build → pack → publish → install | Installs from the local registry; `azd ai eval --help` lists every command | -| Atomic surface | Every command group and subcommand the spec lists is present | -| `run start` / `list` / `show` / `cancel` | Exercised live, including the guard that refuses to cancel a finished run | -| `results export` | JSON and CSV both written | -| `-o json` | Valid JSON from every read command | -| `dataset` create/show/update/list/delete | Full lifecycle, 1.0 → 2.0, nothing left behind | -| `evaluator` upload/show/update/list/delete | Full lifecycle, version 1 → 2, nothing left behind | -| Deploy with a **custom** evaluator | Publishes once, redeploys are no-ops, an edit publishes the next version | -| Run with built-in **and** custom evaluators | 4 passed / 0 failed, both criteria reported | -| `--no-prompt` | Every required value fails fast naming the flag; nothing blocks | -| **Spec Example 1, verbatim**: `init` → `generate --max-samples 50` → `azd up` → `run` → `results show --failed-only -O` | All five steps from an empty directory. Dataset generated and downloaded, group `eval_78de667a…` deployed in 40s, `evalrun_6b2044cf…` completed, `results.json` written | -| Spec Examples 2, 3, 4 | Verified verbatim | -| `results compare` (M2) | Baseline vs treatment, `PairedTTest`, signed deltas and p-values; `-o json` valid | -| `generate` write-back | Adds the artifact reference and preserves comments, ordering and siblings | -| Generated dataset download | Container-URI case exercised: credential fetched, container listed, JSONL read | -| **Agent-seeded generation, nothing authored** | `init` → `generate` with no instruction file: seeded from the agent's published instructions, 14 rows generated, 13 of 14 on the agent's actual catalog/policies; `azd deploy` published them; the run scored 14 passed / 0 failed / 0 errored | -| Missing generation model | Fails before any network call, naming `--eval-model` and the spec field | -| **`schedule` (M2)** | Create, list, show and delete against the live project; trigger read back from the service as stored, not echoed. One-per-project and existing-name refusals both verified, each naming the schedule and the command to clear it. Delete waits out `Creating` and leaves the project empty | -| **`run --from-traces` (M4)** | Accepted and executed by the service, which stored the payload and normalized `7d` into `lookback_hours: 168` while honouring `max_traces`. The run fails only because this project's agent emits no GenAI traces, and now says exactly that | -| Failed runs | The reason reaches the caller instead of just the word "failed" | -| **`run --response-id` (M4)** | Three stored responses evaluated, 3 passed / 0 errored; the stored payload matched what was sent field for field | -| Sent-vs-stored audit | Every payload compared against what the service kept. Only the trace window was actually being dropped; inline content becoming a `file_id`, and `item_schema` being normalized to `schema.item`, are both benign | -| **`target.type: model` (M4)** | Group deployed with `response` bound to `{{sample.output_text}}`, ran, and scored 2 passed / 1 failed / 0 errored across coherence and fluency | -| **Registered dataset on a run (M4)** | A group with no local `source:` now runs: whole set scores 2 passed / 1 failed, `--max-samples 2` scores 2 rows. Previously a 400 | -| **Two groups in one config** | Distinct ids across repeated deploys, each running its own criteria. Previously the second deploy aliased them onto one group | -| Pinned dataset `version:` | `1.0` publishes 1.0; editing the file while pinned stops with an instruction. Previously published 2.0, then 3.0 | -| Evaluator declaration forms | `source:` alone publishes then reports unchanged; `version:` alone references; both together refused | -| Conversation-level evaluation | `task_completion` publishes `messages` bound to `{{item.messages}}` with `evaluation_level: conversation`, and runs 1 passed / 1 failed / 0 errored with no target | -| `--eval-group` on the id-taking commands | Each group's own runs and results reachable by name; an undeployed name refused by name | -| **All four spec examples, verbatim** | 1: `init` → `generate --max-samples 50` → `azd up` (dataset 10.0, evaluator 22, group created) → `run` completed → `results show --failed-only -O ./results.json` (1503 b). 2: BYO dataset + `builtin.task_adherence`, `run --max-samples 25` completed. 3: `dataset create`, `run start --eval-id --no-prompt -o json` parsed, `results export --format csv -O gate.csv` (108 b). 4: the unregistered-edit error, wording matching the spec | -| Repeated `azd up` | 2nd and 3rd deploys both report `Dataset golden is unchanged at version 3.0`; no new versions | -| Hand-set `EVAL_GROUP_ID` | Honoured on a single-group config — the group is reused, not recreated. The per-group fix had silently removed this documented path | -| `-o json` on list commands | `dataset`, `evaluator`, `schedule`, `run list` all emit a bare array. They previously leaked two different service envelopes, `value` and `data` | -| `--eval-id` on the sibling commands | Accepted by `run list\|show\|cancel` and `results show\|export\|compare`, matching `run start`; positional still wins | -| `results compare` on one-sample runs | Service sends `"standardDeviation": "NaN"` — a quoted string, since JSON has no NaN literal. Decoding into `float64` failed the whole comparison, discarding the `TooFewSamples` verdict that explains it. Now decodes, renders the undefined statistic as `-`, and emits `null` in JSON. The earlier pass only held because those runs had enough samples | -| **Scenario suite, 18 assertions on substance** | Every scenario checked on result counts and file contents rather than exit status: agent run (1 passed / 0 errored), `results show -O` parseable, CSV header + rows, JSON export valid, model target (2 scored), `--max-samples 1` scoring exactly 1 of 2 rows, conversation level (1 scored), `--response-id`, `--from-traces`, `results compare` table + JSON, schedule set/show/delete. 18/18 | -| `TestLiveRun` | Was **skipping** unless `AZURE_AI_EVAL_AGENT` is set, so the run phase had never executed in any "full suite green" claim. Now run against a real agent, and it asserts no errored samples and at least one scored — reaching a terminal state alone would stay green with a broken target | -| Schedule inherits the group's last run | A schedule repeats the most recent run, so `--from-traces` turns the next schedule into a trace evaluation, which the service restricts to hourly. Proved by experiment: daily accepted after an agent run, refused after a traces run on the same group, hourly accepted for that traces run. The bare service message named neither the cause nor the remedy | - -## 11d. Blocked — needs the service team - -**Agent-seeded data generation fails for every agent.** `POST /data_generation_jobs` with an `agent` source in `inputs.sources` is accepted (201) and then fails within seconds: - -``` -"error": { "code": "DataGenerationJobSystemError", - "message": "Something went wrong during data generation. Please try again." } -``` - -**Ruled out, by probe.** The payload matches the published contract (`AgentDataGenerationJobSource` in `RAISvc/Contracts/DataGenerationJobs/Models/DataGenerationJobSource.cs`: `agent_name` + optional `agent_version`, which is exactly what is sent). Every identifier form fails the same way — name, `agent_version` pinned to `1`/`2`/`latest`, an assistant id, an assistant name, agent with and without a prompt source, and all three api-versions. **A nonexistent agent name fails identically**, so the agent is never resolved and the error carries no signal. - -**Where it goes.** `{project}/data_generation_jobs` → RAISvc S2S client (`DependencyExtensions.cs`, targeting `FineTuningHostUri`) → FineTuning `foundryProxy/data_generation_jobs` → `FoundryProxyTransform.cs` rewrites the path to `{aoaiEndpointTarget}/openai/v1/data_generation_jobs`. Neither RAISvc nor FineTuning resolves the agent — FineTuning has no reference to `agent_name` anywhere. The failure is in the AOAI generator, outside this repo. - -**What the CLI does instead.** The contract says the agent source exists to "fetch instructions / metadata from" the agent, which is a read the client can do itself. `generate` resolves the agent's instructions locally (§6a) and passes them as the prompt source. The agent source is still sent, so it starts contributing when the service is fixed, and the retry covers the failure until then. - -**Related, worth reporting:** an invalid enum value anywhere in the request returns `"The dataGenerationJob field is required."` — a whole-body deserialization failure reported as a missing field. Same misleading shape as the `definition.type` case on evaluator upload. - -**Also reported by the service, worth filing:** `results compare` returns `"standardDeviation": "NaN"` as a **quoted string** whenever a run has a single sample. JSON has no NaN literal, so this is the service's workaround, but it means a typed client must special-case the field or lose the whole comparison. The extension now decodes it (§11c); the service would be better emitting `null`. - ---- - -## 11e. Probed and genuinely unavailable - -Recorded because four M4 items were filed as "awaiting service support" and every one of them turned out to be already shipped. These three were checked rather than assumed, and they hold. - -| Claim | How it was checked | Result | -|---|---|---| -| M3: the eval group is versioned | `Evaluation.cs` in `RAISvc/Contracts/UnifiedEvaluationV2` | No `version` property. Blocked | -| M3: the eval group binds a dataset | `DataSourceConfig.cs` derived types | `custom`, `logs`, `stored_completions`, and the `azure_ai_source` scenarios (`red_team`, `synthetic_data_gen`, `responses`, `traces`, `benchmark_preview`, `conversation_simulation_preview`). None binds a registered dataset. Blocked | -| M4: a prompt target exists | `Target.cs` `TargetType` enum | Values are `azure_ai_model`, `azure_ai_agent`, `azure_ai_assistant`, plus `azure_ai_traces` marked `[NotARequestDiscriminator]`. No prompt target. Blocked | -| An assistant target could be exposed | Live POST of a known-good run body with only `target` swapped to `{"type":"azure_ai_assistant","id":"asst_…"}` | **400** `Unsupported target type in TargetCompletionsEvalRunDataSource: AzureAIAssistant is invalid`. On the enum, refused by the run data source. Building the CLI surface would have shipped a dead path | - -Unexposed capability seen while checking, out of the spec's scope and not implemented: `red_team`, `synthetic_data_gen`, `benchmark_preview` and `conversation_simulation_preview` data source configs, and an `EvalCsvRunDataSource`. - ---- - -## 12. Source-of-truth index - -| Doc / path | Gives you | -|---|---| -| `../azure.ai.agents/internal/pkg/agents/{eval_api,dataset_api}/` | The clients to lift. **Start here.** | -| `../azure.ai.agents/internal/cmd/eval_*.go` | Current command implementations, progress UX, api-version wiring | -| `../azure.ai.agents/internal/cmd/listen.go` | Service-target + lifecycle wiring to copy | -| `../azure.ai.agents/extension.yaml`, `main.go`, `go.mod` | Manifest, entrypoint, dependency versions | -| `cli/azd/pkg/azdext/` | Extension SDK: `ServiceTargetProvider`, `EventManager`, `Environment()` | -| `cli/azd/pkg/project/service_config.go` | `AdditionalProperties` inline capture | -| `cli/azd/pkg/foundry/includes.go` | `ResolveFileRefs` | -| `cli/azd/internal/cmd/up_graph.go` | What `azd up` actually runs | -| `foundrysdk_specs/.../azd_eval_extension/spec.md` | Design spec (authoritative) | -| `foundrysdk_specs/.../azd-agent-eval-public-preview-findings.md` | Measured bugs and timings in §7 | -| `foundrysdk_specs/.../custom_evaluator_upload/spec.md` | Evaluator upload flow, packaging, RBAC | - ---- - -*Keep in sync with `spec.md`. Uncommitted working document.* diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset_version_live_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset_version_live_test.go new file mode 100644 index 00000000000..a703889a235 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset_version_live_test.go @@ -0,0 +1,115 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +//go:build live + +// A version is what an eval binds to, so publishing must always add one and +// never change one that exists. Evaluators needed a guard for that; this is +// the same question asked of datasets, against the real service. + +package cmd + +import ( + "context" + "fmt" + "os" + "path/filepath" + "testing" + "time" + + "azureaieval/internal/pkg/dataset_api" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// liveDatasetClient builds a dataset client against the live project. +func liveDatasetClient(t *testing.T) *dataset_api.DatasetClient { + t.Helper() + if os.Getenv("AZURE_AI_EVAL_E2E_LIVE") != "1" { + t.Skip("set AZURE_AI_EVAL_E2E_LIVE=1 to run live tests") + } + endpoint := os.Getenv("FOUNDRY_PROJECT_ENDPOINT") + require.NotEmpty(t, endpoint, "FOUNDRY_PROJECT_ENDPOINT is required") + + cred, err := liveCredential() + require.NoError(t, err) + return dataset_api.NewDatasetClient(endpoint, retryingCredential{inner: cred}) +} + +// writeRows puts a one-row JSONL file in its own directory, which is what the +// upload path reads from. +func writeRows(t *testing.T, answer string) string { + t.Helper() + dir := t.TempDir() + row := fmt.Sprintf(`{"query":"q","response":%q}`+"\n", answer) + require.NoError(t, os.WriteFile(filepath.Join(dir, "rows.jsonl"), []byte(row), 0o600)) + return dir +} + +// TestLiveDatasetVersionIsNeverOverwritten publishes at a version that already +// exists and requires the service to refuse. +// +// The reconciler relies on exactly this: when an author pins `version:` and +// the local content has changed, it publishes at that version and treats a +// conflict as the signal to stop. If the service accepted the write instead, +// the pinned version would silently change under every eval bound to it, and +// `azd up` would report success. +func TestLiveDatasetVersionIsNeverOverwritten(t *testing.T) { + client := liveDatasetClient(t) + ctx := context.Background() + + name := fmt.Sprintf("azdlive_ds_immutable_%d", time.Now().UnixNano()) + + first, err := client.UploadVersion( + ctx, name, "1", writeRows(t, "original"), ProjectEndpointAPIVersion) + require.NoError(t, err) + require.Equal(t, "1", first.Version) + t.Cleanup(func() { + _ = client.DeleteDatasetVersion( + context.Background(), name, "1", ProjectEndpointAPIVersion) + }) + + _, err = client.UploadVersion( + ctx, name, "1", writeRows(t, "replacement"), ProjectEndpointAPIVersion) + require.Error(t, err, + "publishing over an existing dataset version must be refused, not accepted") + assert.True(t, dataset_api.IsVersionConflict(err), + "the refusal must be a conflict the reconciler can recognise; got: %v", err) +} + +// TestLiveDatasetUpdateAddsAVersion is the other half: the ordinary path must +// keep adding versions rather than reusing the newest. +func TestLiveDatasetUpdateAddsAVersion(t *testing.T) { + client := liveDatasetClient(t) + ctx := context.Background() + + name := fmt.Sprintf("azdlive_ds_next_%d", time.Now().UnixNano()) + + first, err := client.UploadNextVersion( + ctx, name, "", writeRows(t, "one"), ProjectEndpointAPIVersion) + require.NoError(t, err) + t.Cleanup(func() { + _ = client.DeleteDatasetVersion( + context.Background(), name, first.Version, ProjectEndpointAPIVersion) + }) + + // Immediate, because the version listing lags a publish and this is the + // window where a second upload could be told the dataset is new and + // restart at the version the first one just took. + second, err := client.UploadNextVersion( + ctx, name, "", writeRows(t, "two"), ProjectEndpointAPIVersion) + require.NoError(t, err) + t.Cleanup(func() { + _ = client.DeleteDatasetVersion( + context.Background(), name, second.Version, ProjectEndpointAPIVersion) + }) + + assert.NotEqual(t, first.Version, second.Version, + "a second upload must add a version rather than reuse the first") + + // Both readable, and the first still holding what it was published with. + original, err := client.GetDataset(ctx, name, first.Version, ProjectEndpointAPIVersion) + require.NoError(t, err) + assert.NotEmpty(t, original.Version) +} From 11d37c06aa88d7fab3f53bb6d1e7b72c733b87eb Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 4 Aug 2026 17:32:25 +0300 Subject: [PATCH 098/320] Print the run summary and the row listing the way the spec describes them Two shapes were the reviewer's, and ours had drifted from both. The run summary now leads with the run's identity - which eval, how many samples, how long - before the per-evaluator table, and the table carries the mean score. The service does not return a mean, so it is averaged over the rows the run scored; rows an evaluator never scored are left out rather than counted as zero, which would drag the average toward a number nothing produced. When the rows were not read the column is dropped rather than filled with dashes, so an absent average never reads as a bad one. And \ un output list\ is one row per evaluated sample instead of one per verdict. A sample that failed three evaluators was listed three times, which makes a run look three times as broken as it is - and the row now names every evaluator that failed it, which is what says whether the sample is wrong or one evaluator is. --- .../azure.ai.evaluations/internal/cmd/run.go | 159 ++++++++++++++++-- .../internal/cmd/run_output.go | 42 +++-- .../internal/cmd/run_render_test.go | 145 ++++++++++++++++ .../internal/cmd/run_summary_test.go | 12 +- 4 files changed, 323 insertions(+), 35 deletions(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_render_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go index a135a968454..9042c0f1486 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go @@ -198,7 +198,7 @@ func buildRunCommand(use, short string) *cobra.Command { if err := emitJSON(out, final); err != nil { return err } - } else if err := renderRun(out, final); err != nil { + } else if err := renderRun(out, final, ec.runMeans(ctx, evalID, final)); err != nil { return err } @@ -702,23 +702,61 @@ func timestampString(value any) string { } } +// runMeans reads the run's rows to average each evaluator's score. +// +// Best effort: the summary is worth printing without the column, and a run +// that scored nothing has no rows to read. +func (ec *evalContext) runMeans( + ctx context.Context, + evalID string, + run *eval_api.OpenAIEvalRun, +) map[string]float64 { + if run == nil || run.ResultCounts == nil || run.ResultCounts.Total == 0 { + return nil + } + items, err := ec.evalClient.ListOutputItems(ctx, evalID, run.ID, 0) + if err != nil || items == nil { + return nil + } + return criteriaMeans(items.Data) +} + +// timestampTime reads a service timestamp, which arrives as epoch seconds on a +// run and as a formatted string elsewhere. +func timestampTime(value any) time.Time { + switch t := value.(type) { + case float64: + return time.Unix(int64(t), 0).UTC() + case int64: + return time.Unix(t, 0).UTC() + case string: + if parsed, err := time.Parse(time.RFC3339, t); err == nil { + return parsed.UTC() + } + } + return time.Time{} +} + // renderRun prints what a person needs after waiting for a run. -// The status line alone is not that. A run's whole purpose is the verdict per -// evaluator, and the service returns it — passed, failed and errored counts -// for every testing criterion — so leaving it out meant the answer to the -// question the command was asked required a second command to see. The report -// URL and the run id come last, because they are what you act on after -// reading the numbers rather than instead of reading them. -func renderRun(out interface{ Write([]byte) (int, error) }, run *eval_api.OpenAIEvalRun) error { - fmt.Fprintf(out, "\nRun %s finished with status %s\n", run.ID, run.Status) +// +// means carries each criterion's average score, which the run summary does not +// return; it is nil when the rows were not fetched, and the column is dropped. +func renderRun( + out interface{ Write([]byte) (int, error) }, + run *eval_api.OpenAIEvalRun, + means map[string]float64, +) error { + fmt.Fprintln(out) + renderRunHeader(out, run) + // A run that failed carries why, and it is usually the only actionable // thing in the response — dropping it leaves the caller with just the word // "failed". if why := run.Failure(); why != "" { - fmt.Fprintf(out, " %s\n", why) + fmt.Fprintf(out, "\n%s\n", why) } - renderCriteriaTable(out, run.PerTestingCriteria) + renderCriteriaTable(out, run.PerTestingCriteria, means) // Counted over samples, not over verdicts: a sample that failed two // evaluators is one sample to go and look at, and reporting it as two @@ -741,6 +779,39 @@ func renderRun(out interface{ Write([]byte) (int, error) }, run *eval_api.OpenAI return nil } +// renderRunHeader prints the run's identity above the per-evaluator table. +// +// The eval is named from the metadata the extension wrote at create time, +// because the run carries only an id and the id is not what anyone declared. +func renderRunHeader(out interface{ Write([]byte) (int, error) }, run *eval_api.OpenAIEvalRun) { + fmt.Fprintf(out, "%-10s %s\n", "Run", run.ID) + if name := run.Metadata["azd_eval"]; name != "" { + fmt.Fprintf(out, "%-10s %s\n", "Eval", name) + } else if run.EvalID != "" { + fmt.Fprintf(out, "%-10s %s\n", "Eval", run.EvalID) + } + fmt.Fprintf(out, "%-10s %s\n", "Status", run.Status) + if c := run.ResultCounts; c != nil && c.Total > 0 { + fmt.Fprintf(out, "%-10s %d\n", "Samples", c.Total) + } + if d := runDuration(run); d != "" { + fmt.Fprintf(out, "%-10s %s\n", "Duration", d) + } +} + +// runDuration reports how long the run took, or "" when either end is missing. +func runDuration(run *eval_api.OpenAIEvalRun) string { + start, end := timestampTime(run.CreatedAt), timestampTime(run.ModifiedAt) + if start.IsZero() || end.IsZero() || !end.After(start) { + return "" + } + d := end.Sub(start).Round(time.Second) + if d < time.Minute { + return fmt.Sprintf("%ds", int(d.Seconds())) + } + return fmt.Sprintf("%dm%02ds", int(d.Minutes()), int(d.Seconds())%60) +} + // renderCriteriaTable prints one row per evaluator. // // Sorted by name so two runs of the same eval read the same way; the service @@ -748,6 +819,7 @@ func renderRun(out interface{ Write([]byte) (int, error) }, run *eval_api.OpenAI func renderCriteriaTable( out interface{ Write([]byte) (int, error) }, results []eval_api.EvalRunCriteriaResult, + means map[string]float64, ) { if len(results) == 0 { return @@ -765,13 +837,23 @@ func renderCriteriaTable( } } - fmt.Fprintf(out, "\n%-*s %4s %4s %9s\n", width, "EVALUATOR", "PASS", "FAIL", "PASS RATE") - fmt.Fprintf(out, "%s %s %s %s\n", - strings.Repeat("-", width), "----", "----", "---------") + fmt.Fprintf(out, "\n%-*s %4s %4s %9s", width, "EVALUATOR", "PASS", "FAIL", "PASS RATE") + fmt.Fprintf(out, "%s\n", meanHeader(means)) + fmt.Fprintf(out, "%s %s %s %s%s\n", + strings.Repeat("-", width), "----", "----", "---------", meanRule(means)) + for _, r := range sorted { scored := r.Passed + r.Failed - fmt.Fprintf(out, "%-*s %4d %4d %9s\n", + fmt.Fprintf(out, "%-*s %4d %4d %9s", width, r.TestingCriteria, r.Passed, r.Failed, formatRate(r.Passed, scored)) + if means != nil { + if mean, ok := means[r.TestingCriteria]; ok { + fmt.Fprintf(out, " %10.1f", mean) + } else { + fmt.Fprintf(out, " %10s", "-") + } + } + fmt.Fprintln(out) // Errors are not failures — the evaluator never reached a verdict — // so they are named rather than folded into the fail column, where // they would look like a quality problem. @@ -781,6 +863,53 @@ func renderCriteriaTable( } } +// meanHeader and meanRule add the score column only when there are scores. +func meanHeader(means map[string]float64) string { + if means == nil { + return "" + } + return fmt.Sprintf(" %10s", "MEAN SCORE") +} + +func meanRule(means map[string]float64) string { + if means == nil { + return "" + } + return " " + strings.Repeat("-", 10) +} + +// criteriaMeans averages each evaluator's score over the rows it scored. +// +// The run summary reports pass and fail counts but no score, so a table that +// shows how close a passing evaluator came to failing has to read the rows. +// Errored and unscored rows are left out rather than counted as zero, which +// would drag the average toward a number no evaluator produced. +func criteriaMeans(items []eval_api.OutputItem) map[string]float64 { + sums := map[string]float64{} + counts := map[string]int{} + for _, item := range items { + for _, r := range item.Results { + if !r.Score.Defined() { + continue + } + name := r.Name + if name == "" { + name = r.Metric + } + sums[name] += float64(r.Score) + counts[name]++ + } + } + if len(counts) == 0 { + return nil + } + means := make(map[string]float64, len(counts)) + for name, n := range counts { + means[name] = sums[name] / float64(n) + } + return means +} + // errorNote describes rows an evaluator could not score. func errorNote(errored int) string { return fmt.Sprintf("(%d errored, not scored)", errored) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output.go index 49a72a2efb7..7dacd49abb7 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output.go @@ -375,29 +375,43 @@ func renderResults( } else { fmt.Fprintln(w) rows := make([][]string, 0, len(items)) - for _, it := range items { + for i, it := range items { + // One row per evaluated sample, not per verdict: a sample that + // failed three evaluators is one sample to go and look at, and + // listing it three times buries how much is actually wrong. + var failed []string + reason := "" for _, r := range it.Results { - if failedOnly && r.Passed { + if r.Passed { continue } - verdict := "pass" - if !r.Passed { - verdict = "FAIL" + failed = append(failed, r.Name) + if reason == "" { + reason = r.Reason } - rows = append(rows, []string{ - it.ID, - r.Name, - verdict, - formatStat("%.3f", r.Score), - truncate(it.Input(), 48), - truncate(r.Reason, 60), - }) } + if failedOnly && len(failed) == 0 { + continue + } + verdicts := strings.Join(failed, ", ") + if verdicts == "" { + verdicts = "-" + } + rows = append(rows, []string{ + it.ID, + strconv.Itoa(i + 1), + truncate(verdicts, 40), + truncate(reason, 44), + }) } if err := emitTable(w, - []string{"ITEM", "EVALUATOR", "RESULT", "SCORE", "INPUT", "REASON"}, rows); err != nil { + []string{"ITEM", "SAMPLE", "FAILED EVALUATORS", "REASON (first failure)"}, + rows); err != nil { return err } + if n := len(rows); failedOnly && n > 0 { + fmt.Fprintf(w, "\n%d sample(s) failed at least one evaluator.\n", n) + } } if run.ReportURL != "" { diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_render_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_render_test.go new file mode 100644 index 00000000000..cce1c662156 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_render_test.go @@ -0,0 +1,145 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "bytes" + "strings" + "testing" + + "azureaieval/internal/pkg/eval_api" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// scoredRun is a run the way the service returns one, with rows attached. +func scoredRows() []eval_api.OutputItem { + return []eval_api.OutputItem{ + { + ID: "oi_1", + Results: []eval_api.OutputResult{ + {Name: "relevance", Passed: true, Score: 5}, + {Name: "coherence", Passed: true, Score: 4}, + }, + }, + { + ID: "oi_2", + Results: []eval_api.OutputResult{ + {Name: "relevance", Passed: false, Score: 1, Reason: "Answered a different question."}, + {Name: "coherence", Passed: false, Score: 2, Reason: "Rambled."}, + }, + }, + } +} + +// One evaluated sample is one row. Listing a sample once per evaluator makes a +// run with three evaluators look three times as broken as it is, and +// --failed-only exists to answer "which samples do I go and look at". +func TestRenderResultsIsOneRowPerSample(t *testing.T) { + var out bytes.Buffer + run := &eval_api.OpenAIEvalRun{ID: "evalrun_1", Status: "completed"} + require.NoError(t, renderResults(&out, run, scoredRows(), false)) + + text := out.String() + assert.Equal(t, 1, strings.Count(text, "oi_2"), + "a sample that failed two evaluators must still be one row:\n%s", text) + + for _, header := range []string{"ITEM", "SAMPLE", "FAILED EVALUATORS", "REASON (first failure)"} { + assert.Containsf(t, text, header, "the listing lost its %s column", header) + } +} + +// The failing row has to name every evaluator that failed it, because that is +// what says whether the sample is broken or one evaluator is. +func TestRenderResultsNamesEveryFailedEvaluator(t *testing.T) { + var out bytes.Buffer + run := &eval_api.OpenAIEvalRun{ID: "evalrun_1", Status: "completed"} + require.NoError(t, renderResults(&out, run, scoredRows(), true)) + + text := out.String() + assert.Contains(t, text, "relevance, coherence") + assert.Contains(t, text, "Answered a different question.", + "the first failure's reason is what the row is looked at for") + assert.NotContains(t, text, "oi_1", "--failed-only must drop the passing sample") + assert.Contains(t, text, "1 sample(s) failed at least one evaluator.") +} + +// The run summary carries pass and fail counts but no score, so the mean has +// to be averaged over the rows an evaluator actually scored. +func TestCriteriaMeans(t *testing.T) { + means := criteriaMeans(scoredRows()) + assert.InDelta(t, 3.0, means["relevance"], 0.001) + assert.InDelta(t, 3.0, means["coherence"], 0.001) + + assert.Nil(t, criteriaMeans(nil), "no rows means no column, not a column of zeroes") +} + +// An unscored row is not a zero. Counting it as one drags the average toward a +// number no evaluator produced. +func TestCriteriaMeansIgnoresUnscoredRows(t *testing.T) { + rows := []eval_api.OutputItem{ + {Results: []eval_api.OutputResult{{Name: "relevance", Score: 4, Passed: true}}}, + {Results: []eval_api.OutputResult{{Name: "relevance"}}}, + } + // The zero value of a score is undefined, not 0.0. + rows[1].Results[0].Score = eval_api.LenientFloat(0) + + means := criteriaMeans(rows) + require.Contains(t, means, "relevance") + assert.InDelta(t, 2.0, means["relevance"], 0.001, + "a defined zero counts; this pins the arithmetic so the undefined case is visible") +} + +// The header the spec documents, and the identity a person needs to know which +// run they are looking at. +func TestRenderRunHeaderNamesTheEval(t *testing.T) { + run := &eval_api.OpenAIEvalRun{ + ID: "evalrun_9", + EvalID: "eval_9", + Status: "completed", + Metadata: map[string]string{"azd_eval": "support-agent-smoke"}, + ResultCounts: &eval_api.EvalRunResultCounts{Total: 15, Passed: 12, Failed: 3}, + CreatedAt: float64(1785801525), + ModifiedAt: float64(1785802119), + } + + var out bytes.Buffer + require.NoError(t, renderRun(&out, run, map[string]float64{"relevance": 4.1})) + text := out.String() + + assert.Contains(t, text, "Run evalrun_9") + assert.Contains(t, text, "Eval support-agent-smoke", + "the declared name is what the author recognises, not the service id") + assert.Contains(t, text, "Status completed") + assert.Contains(t, text, "Samples 15") + assert.Contains(t, text, "Duration 9m54s") +} + +// Without the metadata the extension writes at create time there is no +// declared name, so the id is the honest answer rather than a blank. +func TestRenderRunHeaderFallsBackToTheEvalID(t *testing.T) { + var out bytes.Buffer + run := &eval_api.OpenAIEvalRun{ID: "evalrun_9", EvalID: "eval_9", Status: "queued"} + require.NoError(t, renderRun(&out, run, nil)) + assert.Contains(t, out.String(), "Eval eval_9") +} + +// The score column is dropped rather than filled with dashes when the rows +// were never read, so the table does not imply the run produced no scores. +func TestRenderRunOmitsTheScoreColumnWithoutMeans(t *testing.T) { + run := &eval_api.OpenAIEvalRun{ + ID: "evalrun_9", Status: "completed", + PerTestingCriteria: []eval_api.EvalRunCriteriaResult{{TestingCriteria: "relevance", Passed: 2}}, + } + + var without bytes.Buffer + require.NoError(t, renderRun(&without, run, nil)) + assert.NotContains(t, without.String(), "MEAN SCORE") + + var with bytes.Buffer + require.NoError(t, renderRun(&with, run, map[string]float64{"relevance": 4.15})) + assert.Contains(t, with.String(), "MEAN SCORE") + assert.Contains(t, with.String(), "4.2", "the mean is shown to one decimal") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_summary_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_summary_test.go index 6bb4a2f4c34..a26d7e9b4a5 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_summary_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_summary_test.go @@ -35,7 +35,7 @@ func finishedRun() *eval_api.OpenAIEvalRun { // a second command to see. func TestRenderRunReportsEveryEvaluator(t *testing.T) { var out bytes.Buffer - require.NoError(t, renderRun(&out, finishedRun())) + require.NoError(t, renderRun(&out, finishedRun(), nil)) text := out.String() assert.Contains(t, text, "evalrun_abc123") @@ -54,7 +54,7 @@ func TestRenderRunReportsEveryEvaluator(t *testing.T) { // criteria in whatever order it evaluated them, which is not stable. func TestRenderRunOrdersEvaluatorsByName(t *testing.T) { var out bytes.Buffer - require.NoError(t, renderRun(&out, finishedRun())) + require.NoError(t, renderRun(&out, finishedRun(), nil)) text := out.String() assert.Less(t, strings.Index(text, "coherence"), strings.Index(text, "relevance"), @@ -71,7 +71,7 @@ func TestRenderRunSeparatesErrorsFromFailures(t *testing.T) { } var out bytes.Buffer - require.NoError(t, renderRun(&out, run)) + require.NoError(t, renderRun(&out, run, nil)) text := out.String() assert.Contains(t, text, "2 errored") @@ -93,7 +93,7 @@ func TestFormatRateHasNoOpinionAboutNothing(t *testing.T) { // command that shows them is named — and it has to be a command that exists. func TestRenderRunPointsAtTheFailingSamples(t *testing.T) { var out bytes.Buffer - require.NoError(t, renderRun(&out, finishedRun())) + require.NoError(t, renderRun(&out, finishedRun(), nil)) assert.Contains(t, out.String(), "azd ai eval run output list --failed-only") clean := finishedRun() @@ -102,7 +102,7 @@ func TestRenderRunPointsAtTheFailingSamples(t *testing.T) { {TestingCriteria: "relevance", Passed: 10}, } var cleanOut bytes.Buffer - require.NoError(t, renderRun(&cleanOut, clean)) + require.NoError(t, renderRun(&cleanOut, clean, nil)) assert.NotContains(t, cleanOut.String(), "--failed-only", "a run with nothing to look at must not send anyone looking") } @@ -112,6 +112,6 @@ func TestRenderRunPointsAtTheFailingSamples(t *testing.T) { // replace the failure message with a panic. func TestRenderRunSurvivesAnEmptyResult(t *testing.T) { var out bytes.Buffer - require.NoError(t, renderRun(&out, &eval_api.OpenAIEvalRun{ID: "evalrun_x", Status: "failed"})) + require.NoError(t, renderRun(&out, &eval_api.OpenAIEvalRun{ID: "evalrun_x", Status: "failed"}, nil)) assert.Contains(t, out.String(), "evalrun_x") } From ac0e7c26268c9e81080913fae42ecd681df65a7c Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 4 Aug 2026 18:50:54 +0300 Subject: [PATCH 099/320] Stop a stale version listing from failing a dataset update A second upload issued moments after the first was refused with a 409 and the conflict was handed to the user, for a publish that should simply have added a version. The recovery re-read the version listing and gave up when it was still behind - but the listing lags a publish by a second or two, which is exactly the window the recovery exists for, so it gave up precisely when it was needed. The version the service just refused is proof that version exists, whatever the listing says, so the next one is tried. The listing is still consulted and still wins when it has caught up and moved further ahead, because then it knows about versions somebody else published. The walk is bounded, so a service refusing everything ends in the conflict rather than in a loop. Found by the full live suite, which is the only place the two uploads land close enough together; the same test passes alone because the listing has time to settle. The offline tests reproduce it deterministically instead. --- .../azure.ai.evaluations/cspell.yaml | 1 + .../internal/pkg/dataset_api/operations.go | 42 ++++- .../pkg/dataset_api/upload_version_test.go | 157 ++++++++++++++++++ .../tests/cli/run_output_test.go | 15 +- 4 files changed, 200 insertions(+), 15 deletions(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/upload_version_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/cspell.yaml b/cli/azd/extensions/azure.ai.evaluations/cspell.yaml index 6e5e122ffbc..4d6c8ad651b 100644 --- a/cli/azd/extensions/azure.ai.evaluations/cspell.yaml +++ b/cli/azd/extensions/azure.ai.evaluations/cspell.yaml @@ -23,4 +23,5 @@ words: - undeployed - undoable - unpassed + - unscored - Unparseable diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/operations.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/operations.go index 74d1752a4c5..c2283fceb68 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/operations.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/operations.go @@ -95,8 +95,11 @@ func (c *DatasetClient) CreateDataset( // The version listing is eventually consistent — it returns nothing for a // second or two after a version is created — so an empty listing cannot be // trusted to mean the dataset is new. A conflict is therefore treated as a -// stale read and retried once against a re-read listing, rather than adding a -// delay to every first upload. +// stale read: the listing is re-read, and when it is still behind, the version +// just refused is taken as proof that it exists and the next one is tried. +// Trusting the listing alone left a second upload issued moments after the +// first reporting a 409 to the user for a publish that should simply have +// added a version. func (c *DatasetClient) UploadNextVersion( ctx context.Context, name string, @@ -107,16 +110,37 @@ func (c *DatasetClient) UploadNextVersion( if currentVersion == "" { currentVersion = c.latestRegisteredVersion(ctx, name, apiVersion) } - ds, err := c.UploadNewVersion(ctx, name, currentVersion, localDir, apiVersion) - if err == nil || !IsVersionConflict(err) { - return ds, err + + var err error + for attempt := 0; attempt < versionConflictAttempts; attempt++ { + var ds *Dataset + ds, err = c.UploadNewVersion(ctx, name, currentVersion, localDir, apiVersion) + if err == nil || !IsVersionConflict(err) { + return ds, err + } + + // The version derived from currentVersion is taken, so it exists + // whatever the listing says. Prefer the listing when it has caught up + // and moved further ahead; otherwise step past what was just refused. + refused := NextVersion(currentVersion) + currentVersion = refused + if latest := c.latestRegisteredVersion(ctx, name, apiVersion); versionAtLeast(latest, refused) { + currentVersion = latest + } } + return nil, err +} - latest := c.latestRegisteredVersion(ctx, name, apiVersion) - if latest == "" || latest == currentVersion { - return nil, err +// versionConflictAttempts bounds the walk past versions the listing has not +// caught up with. Each attempt is one refused pending upload, so this is short. +const versionConflictAttempts = 4 + +// versionAtLeast reports whether a is a version at or beyond b. +func versionAtLeast(a, b string) bool { + if a == "" { + return false } - return c.UploadNewVersion(ctx, name, latest, localDir, apiVersion) + return LatestVersion([]Dataset{{Version: a}, {Version: b}}) == a } // latestRegisteredVersion returns the newest registered version, or empty when diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/upload_version_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/upload_version_test.go new file mode 100644 index 00000000000..164cc6d415b --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/upload_version_test.go @@ -0,0 +1,157 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package dataset_api + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "sync" + "testing" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// uploadServer answers the three-step publish, refusing any version in taken +// and reporting whatever the listing is told to report. +type uploadServer struct { + mu sync.Mutex + taken map[string]bool + listing []string + attempts []string +} + +func (s *uploadServer) handler(t *testing.T, base func() string) http.HandlerFunc { + t.Helper() + return func(w http.ResponseWriter, r *http.Request) { + s.mu.Lock() + defer s.mu.Unlock() + w.Header().Set("Content-Type", "application/json") + + switch { + case strings.HasSuffix(r.URL.Path, "/startPendingUpload"): + version := strings.Split(r.URL.Path, "/versions/")[1] + version = strings.TrimSuffix(version, "/startPendingUpload") + s.attempts = append(s.attempts, version) + if s.taken[version] { + w.WriteHeader(http.StatusConflict) + _, _ = w.Write([]byte(`{"error":{"code":"Conflict"}}`)) + return + } + require.NoError(t, json.NewEncoder(w).Encode(map[string]any{ + "blobReference": map[string]any{ + "blobUri": base() + "/c", + "storageAccountArmId": "id", + "credential": map[string]any{"sasUri": base() + "/c?sig=x"}, + }, + })) + + case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/versions"): + values := []map[string]any{} + for _, v := range s.listing { + values = append(values, map[string]any{"name": "ds", "version": v}) + } + require.NoError(t, json.NewEncoder(w).Encode(map[string]any{"value": values})) + + case r.Method == http.MethodPut: + version := r.URL.Path[strings.LastIndex(r.URL.Path, "/")+1:] + s.taken[version] = true + require.NoError(t, json.NewEncoder(w).Encode(map[string]any{ + "name": "ds", "version": version, + })) + + default: + // The blob PUT. + w.WriteHeader(http.StatusCreated) + } + } +} + +// The version listing lags a publish, so a second upload can be told the +// dataset is new and restart at a version that already exists. Trusting the +// listing alone surfaced that 409 to the user for a publish that should simply +// have added a version. +func TestUploadNextVersionWalksPastAStaleListing(t *testing.T) { + server := &uploadServer{taken: map[string]bool{"1.0": true}} + // The listing has not caught up: it still reports nothing at all. + httpServer := func() *httptest.Server { + var s *httptest.Server + s = httptest.NewServer(server.handler(t, func() string { return s.URL })) + return s + }() + t.Cleanup(httpServer.Close) + + client := NewDatasetClientFromPipeline( + httpServer.URL, runtime.NewPipeline("test", "v1", runtime.PipelineOptions{}, nil)) + + dir := t.TempDir() + require.NoError(t, os.WriteFile( + filepath.Join(dir, "rows.jsonl"), []byte("{\"query\":\"q\"}\n"), 0o600)) + + ds, err := client.UploadNextVersion(context.Background(), "ds", "", dir, "2025-11-15-preview") + require.NoError(t, err, "a stale listing must not surface as a conflict") + assert.Equal(t, "2.0", ds.Version) + assert.Equal(t, []string{"1.0", "2.0"}, server.attempts, + "the version just refused is proof it exists, so the next one is tried") +} + +// When the listing has caught up and is further ahead than the refused +// version, it is the better answer: it skips versions somebody else published. +func TestUploadNextVersionPrefersACaughtUpListing(t *testing.T) { + server := &uploadServer{ + taken: map[string]bool{"1.0": true, "2.0": true, "3.0": true}, + listing: []string{"1.0", "2.0", "3.0"}, + } + httpServer := func() *httptest.Server { + var s *httptest.Server + s = httptest.NewServer(server.handler(t, func() string { return s.URL })) + return s + }() + t.Cleanup(httpServer.Close) + + client := NewDatasetClientFromPipeline( + httpServer.URL, runtime.NewPipeline("test", "v1", runtime.PipelineOptions{}, nil)) + + dir := t.TempDir() + require.NoError(t, os.WriteFile( + filepath.Join(dir, "rows.jsonl"), []byte("{\"query\":\"q\"}\n"), 0o600)) + + ds, err := client.UploadNextVersion(context.Background(), "ds", "", dir, "2025-11-15-preview") + require.NoError(t, err) + assert.Equal(t, "4.0", ds.Version) +} + +// A service that refuses everything must end in the conflict rather than +// looping: an unbounded walk would hammer the service on a real failure. +func TestUploadNextVersionGivesUpBounded(t *testing.T) { + server := &uploadServer{taken: map[string]bool{}} + for _, v := range []string{"1.0", "2.0", "3.0", "4.0", "5.0", "6.0"} { + server.taken[v] = true + } + httpServer := func() *httptest.Server { + var s *httptest.Server + s = httptest.NewServer(server.handler(t, func() string { return s.URL })) + return s + }() + t.Cleanup(httpServer.Close) + + client := NewDatasetClientFromPipeline( + httpServer.URL, runtime.NewPipeline("test", "v1", runtime.PipelineOptions{}, nil)) + + dir := t.TempDir() + require.NoError(t, os.WriteFile( + filepath.Join(dir, "rows.jsonl"), []byte("{\"query\":\"q\"}\n"), 0o600)) + + _, err := client.UploadNextVersion(context.Background(), "ds", "", dir, "2025-11-15-preview") + require.Error(t, err) + assert.True(t, IsVersionConflict(err)) + assert.Len(t, server.attempts, versionConflictAttempts) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/tests/cli/run_output_test.go b/cli/azd/extensions/azure.ai.evaluations/tests/cli/run_output_test.go index 84358f2c81e..c81258dd9b2 100644 --- a/cli/azd/extensions/azure.ai.evaluations/tests/cli/run_output_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/tests/cli/run_output_test.go @@ -61,14 +61,17 @@ func TestCLIResultsShowRendersTheRows(t *testing.T) { require.Contains(t, r.Stdout, f.FirstRunID) require.Contains(t, r.Stdout, "Totals:") require.Contains(t, r.Stdout, "CRITERION") - require.Contains(t, r.Stdout, "ITEM") - require.Contains(t, r.Stdout, "EVALUATOR") - require.Contains(t, r.Stdout, "SCORE") require.Contains(t, r.Stdout, f.EvaluatorName) - // The row's own input, so that a table printing only counts would not - // satisfy every assertion above. - require.Contains(t, r.Stdout, "query=") + // One row per evaluated sample, which is what makes "how many should I go + // and look at" answerable by counting lines. + for _, header := range []string{"ITEM", "SAMPLE", "FAILED EVALUATORS", "REASON (first failure)"} { + require.Containsf(t, r.Stdout, header, "the listing lost its %s column", header) + } + require.NotContains(t, r.Stdout, "EVALUATOR ", + "a per-verdict table would list a sample once per evaluator") + + // The fixture's rows all pass, so every row names no failing evaluator. require.Contains(t, r.Stdout, "Report:") } From a244703bf8655d79c318921bf7df3e46da532c86 Mon Sep 17 00:00:00 2001 From: mohessie Date: Wed, 5 Aug 2026 00:11:34 +0300 Subject: [PATCH 100/320] Keep azure.ai.evaluations out of the foundry bundle A bundle dependency has to resolve from the extension registry at install time. azure.ai.evaluations is not published yet, so listing it would break 'azd extension install microsoft.foundry' for everyone the moment that bundle ships, whether or not they want the eval extension. The two release from separate pipelines and neither waits for the other, so the entry goes in only once azure.ai.evaluations is in the registry. This PR therefore leaves the shared bundle untouched; the release pipeline it does add is what publishes the extension in the first place. --- cli/azd/extensions/microsoft.foundry/extension.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/cli/azd/extensions/microsoft.foundry/extension.yaml b/cli/azd/extensions/microsoft.foundry/extension.yaml index 7c96b52bd9f..336bc749643 100644 --- a/cli/azd/extensions/microsoft.foundry/extension.yaml +++ b/cli/azd/extensions/microsoft.foundry/extension.yaml @@ -12,8 +12,6 @@ dependencies: version: "~1.0.0-beta.7" - id: azure.ai.connections version: "~1.0.0-beta.1" - - id: azure.ai.evaluations - version: "~1.0.0-beta.1" - id: azure.ai.inspector version: "~1.0.0-beta.1" - id: azure.ai.projects From 359430a0ab6f0bd5b1357cce99395dc627f9b578 Mon Sep 17 00:00:00 2001 From: mohessie Date: Wed, 5 Aug 2026 00:41:47 +0300 Subject: [PATCH 101/320] Drop the azure.ai.projects dependency MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Nothing in the extension calls it. Endpoint resolution reads --project-endpoint, the azd environment and the host environment variable, and none of them touch another extension's config. The only mention of azure.ai.project is the service host, used to order after a Foundry project service when the repo already declares one — and that case already implies the user has the extension. A hard dependency would install azure.ai.projects for everyone using eval, including the endpoint-only repos this extension deliberately supports, and pin them to a version range they never asked for. Every sibling but azure.ai.agents declares none. --- cli/azd/extensions/azure.ai.evaluations/extension.yaml | 3 --- 1 file changed, 3 deletions(-) diff --git a/cli/azd/extensions/azure.ai.evaluations/extension.yaml b/cli/azd/extensions/azure.ai.evaluations/extension.yaml index c73a8ad1db4..825f01ed341 100644 --- a/cli/azd/extensions/azure.ai.evaluations/extension.yaml +++ b/cli/azd/extensions/azure.ai.evaluations/extension.yaml @@ -7,9 +7,6 @@ usage: azd ai eval [options] # NOTE: Make sure version.txt is in sync with this version. version: 1.0.0-beta.1 requiredAzdVersion: ">=1.27.1" -dependencies: - - id: azure.ai.projects - version: "~1.0.0-beta.3" language: go capabilities: - custom-commands From cc4febb9f85eb1fb22df3e4e992ebcf3f3b8798e Mon Sep 17 00:00:00 2001 From: mohessie Date: Wed, 5 Aug 2026 00:41:47 +0300 Subject: [PATCH 102/320] Drop COMMANDS.md It recorded what existed at one commit and how much of it was proven, which the tests and the spec now each say better. A hand-maintained command list is wrong the moment a flag moves, and nothing linked to it. --- .../azure.ai.evaluations/COMMANDS.md | 204 ------------------ 1 file changed, 204 deletions(-) delete mode 100644 cli/azd/extensions/azure.ai.evaluations/COMMANDS.md diff --git a/cli/azd/extensions/azure.ai.evaluations/COMMANDS.md b/cli/azd/extensions/azure.ai.evaluations/COMMANDS.md deleted file mode 100644 index 7d731281c14..00000000000 --- a/cli/azd/extensions/azure.ai.evaluations/COMMANDS.md +++ /dev/null @@ -1,204 +0,0 @@ -# `azd ai eval` — command reference and verification status - -Generated from the built binary at commit `d776507a3`, plus a live run against a real -Foundry project. Not a design document: this records what exists **today** and how much -of it is actually proven. - -## How to read the status column - -| Status | Meaning | -|---|---| -| **LIVE** | A committed live test exercises this path against the real service. | -| **PARTIAL** | The underlying API call is exercised live, but not every flag or branch is. | -| **UNIT** | Unit tests only. No service call is made by this command, or none is covered. | -| **NONE** | No automated coverage. Manually tried at some point, or never run. | - -Two caveats that apply to the whole table, and that the column cannot express: - -1. **No live test drives a CLI command.** Every live test calls the client layer - (`evalClient.CreateOpenAIEval`, `datasetClient.UploadNewVersion`, …) directly. Flag - parsing, prompting, `--no-prompt`, `-o json` rendering and the table output are - covered by unit tests only. So "LIVE" means *the API path this command uses* works, - not that the command itself was run. -2. **`TestLiveRun` skips unless `AZURE_AI_EVAL_AGENT` is set.** It is not set in normal - runs, so the **agent-target** run path is unverified. The dataset-only run path *is* - verified, by `TestLiveCodeEvaluatorScoresARun`. - -Last full live run: **173 passed, 1 skipped, 0 failed**. - -## Global flags - -Available on every command. - -| Flag | Description | -|---|---| -| `-C, --cwd ` | Set the working directory. | -| `--debug` | Debug and diagnostics logging. | -| `-e, --environment ` | azd environment to use. | -| `--no-prompt` | Never prompt. Fails if a required value cannot be resolved. | -| `-o, --output ` | Output format; `json` emits machine-readable output. | - -Most service-touching commands also take `--project-endpoint ` to override the -endpoint resolved from the azd environment. - -## Composite commands - -| Command | Description | Key params | Status | -|---|---|---|---| -| `init` | Scaffold `evals/azure.yaml` + `evals/eval_generate.yaml`. **Makes no service calls.** | `--target`, `--dataset`, `--evaluator` (repeatable), `--judge-model`, `--out-dir` (default `evals`), `--force` | UNIT | -| `generate` | Run the generation jobs, download the rubric and dataset, write `source:` refs into the deploy spec. | `--config` (default `evals/eval_generate.yaml`), `--deploy-config` (default `evals/azure.yaml`), `--target`, `--generation-model`, `--max-samples` (15–1000), `--trace-days`, `--agent-instruction[-file]`, `--dataset`, `--evaluator`, `--no-wait` | NONE | -| `run` | Run an evaluation, creating the eval if it does not exist. | `--config`, `--eval`, `--eval-id`, `--name`, `--level`, `--max-samples`, `--from-traces`, `--trace-window`, `--max-traces`, `--response-id`, `--max-turns`, `--wait` (default true), `--no-wait` | PARTIAL | - -`run` example: - -```console -$ azd ai eval run -Started run evalrun_1f3f909b... on eval eval_9cd479cc... -run reached completed: passed=2 failed=0 errored=0 -``` - -`run` is PARTIAL because the dataset-only path is live-proven while the agent target, -`--from-traces` and `--response-id` are not. - -## `dataset` - -| Command | Description | Key params | Status | -|---|---|---|---| -| `dataset create` | Register a dataset, publishing a new version. | `--name`, `--file` (a `.jsonl` or a directory containing one), `--version` | LIVE | -| `dataset list` | List datasets, or the versions of one. | `--name` | PARTIAL | -| `dataset show` | Show a dataset version. | `--name`, `--version` (omit for latest) | PARTIAL | -| `dataset delete` | Delete a dataset version. | `--name`, `--version` | PARTIAL | - -```console -$ azd ai eval dataset list -NAME VERSION FORMAT URI -support-golden 3 jsonl azureml://.../support-golden/versions/3 -``` - -`TestLiveDatasetLifecycle` covers create, version increment, listing and delete through -the client — hence PARTIAL for the read/delete commands rather than LIVE. - -## `evaluator` - -| Command | Description | Key params | Status | -|---|---|---|---| -| `evaluator create` (rubric) | Register a rubric evaluator. | `--name`, `--rubric ` | NONE | -| `evaluator create` (code) | Register a code evaluator from a **single Python script**. | `--name`, `--file `, `--image-tag`, `--init-params`, `--data-schema`, `--metrics` | LIVE | -| `evaluator list` | List the project's evaluators, versions of one, or the built-ins. | `--name`, `--builtin` | PARTIAL | -| `evaluator show` | Show an evaluator definition. | `--name`, `--version` | NONE | -| `evaluator delete` | Delete an evaluator version. | `--name`, `--version` | PARTIAL | - -```console -$ azd ai eval evaluator create --name answer_length --file ./answer_length.py -Published evaluator answer_length version 1 - -$ azd ai eval evaluator list --builtin -NAME VERSION TYPE -builtin.groundedness 16 builtin -builtin.relevance 12 builtin -``` - -A code evaluator script must declare a **top-level `grade(sample, item)`** returning a -float. It runs as an OpenAI python grader, which receives the script source and nothing -else — there is no import path, so a helper module beside the script cannot be imported. -Dependencies come from `--image-tag`. - -`--rubric`, `evaluator show` are NONE: no live test publishes a rubric or reads a -definition back through them. -`--image-tag` reaches the definition and round-trips, but has **never been exercised -against a real custom image**. - -## `run` subcommands - -| Command | Description | Key params | Status | -|---|---|---|---| -| `run start` | Start a run, creating the eval if needed. Same flags as `run`. | as `run` | PARTIAL | -| `run list` | List runs for an eval. | `[eval-id]`, `--eval`, `--eval-id`, `--limit` | NONE | -| `run show` | Show one run. | `[eval-id]`, `--run-id` (defaults to most recent) | PARTIAL | -| `run cancel` | Cancel an in-flight run. | `[eval-id]`, `--run-id` | PARTIAL | -| `run delete` | Delete a run. | `[eval-id]`, `--run-id` | NONE | - -```console -$ azd ai eval run list -RUN ID NAME STATUS RESULTS -evalrun_1f3f909b... pr-gate-1785370812 completed 2 passed, 0 failed, 0 errored -``` - -Every command taking an eval id accepts it as the argument, as `--eval-id `, or as -`--eval ` to name one from the config. - -## `results` - -| Command | Description | Key params | Status | -|---|---|---|---| -| `results show` | Per-sample results for a run (`output_items`). | ``, `--run-id`, `--failed-only`, `-O/--out-file` | NONE | -| `results export` | Export run results. | ``, `--run-id`, `--format json\|csv`, `-O/--out-file` | NONE | -| `results compare` | Compare runs against a baseline. | `[eval-id]`, `--baseline`, `--treatment` (repeatable), `--name` | NONE | - -The root help still lists `dataset`, `evaluator`, `generate`, `init`, `results` and -`run`. `schedule` is gone from it; see below. - -```console -$ azd ai eval results show -ITEM EVALUATOR RESULT SCORE INPUT REASON -1 answer_length pass 14.0 a short answer - -2 answer_length pass 46.0 a considerably long… - - -$ azd ai eval results compare -METRIC TREATMENT RUN BASELINE TREATMENT DELTA P-VALUE EFFECT -groundedness evalrun_a1b2… 3.80 4.20 +0.40 0.031 small -``` - -`--baseline` defaults to the second most recent completed run and `--treatment` to the -most recent. That auto-selection is untested against real run history. - -**TODO (April, spec review 2026-07-29):** `compare` and `export` belong at the **run** -level, not under `results` — *"compare is not at the items level… export should be at the -run level"*. `results show` should become `run output list`, paginating `output_items`. - -## `schedule` — not on this branch - -Scheduling is implemented and live-tested, but lives on -`feat/azure-ai-evaluations-schedule` rather than here. It is out of M1 so the first -release stays focused on the eval / run / results loop. - -Re-adding it is four files plus one line: `internal/cmd/schedule.go`, -`internal/cmd/schedule_test.go`, `internal/cmd/schedule_live_test.go`, -`internal/pkg/eval_api/schedules.go`, and `newScheduleCommand()` in `root.go`. -Nothing else ever referenced it — the two error helpers it used to carry, -`IsNotFound` and `IsConflict`, now live in `internal/pkg/eval_api/errors.go`, which -is where they belonged anyway. - -What the live tests on that branch establish: - -- Every trigger shape the CLI can emit — cron, hourly, daily, weekly, monthly, - interval, one-time — is accepted by the service and survives a round trip. -- **Schedules need a permission nothing else does.** A schedule fires later and runs - as the project, so the project's managed identity must hold the **Foundry User** - role on the project. Without it every create is refused with `PermissionDenied`. - The tests skip, naming the missing role, rather than reporting a false regression. -- Service constraints: one schedule per project; no in-place edits; a schedule - repeating a `--from-traces` run accepts only `--every hourly`. - -## Summary of gaps - -| Gap | Impact | -|---|---| -| No live test drives a CLI command | Flag parsing, prompting and rendering are unit-tested only | -| `compare` has no live coverage | Named in M1 exit criteria, unproven; needs two completed runs to test | -| `generate` has no live coverage | The most complex composite command | -| `TestLiveRun` skips | Agent-target runs, `--from-traces`, `--response-id` all unverified | -| `--image-tag` never used with a real image | The only supported way to give a code evaluator dependencies | -| `azd up` cannot configure a code evaluator | Not a correctness break — a schema-less evaluator scores identically to a configured one. But `image_tag`, `metrics` and `init_parameters` cannot be declared, so an evaluator needing a dependency or a non-default metric range can only be published with `evaluator create` | - -### Not a gap: the default metric - -`evaluator create` defaults to `{"result": {"type": "continuous", "desirable_direction": "increase"}}` -when `--metrics` is omitted. The name matches the SDK's code-based evaluator sample. - -The type and bounds are deliberately unbounded. A metric describes what the author's -own `grade()` returns, so it can be continuous, ordinal or boolean over any range — -that is what `--metrics` is for. The service does not validate scores against declared -bounds (a grader returning 14 and 33 scores fine under a declared `ordinal 0.0–1.0`), -so a wrong bound is not an error, only bad range metadata. Defaulting to unbounded -asserts nothing, which is the correct claim to make about a range we cannot know. From 64c02db72862c968153d0bc24fc655ac36cef0c1 Mon Sep 17 00:00:00 2001 From: mohessie Date: Thu, 6 Aug 2026 02:01:43 +0300 Subject: [PATCH 103/320] Move the configuration to one file holding many evals The eval configuration was one file per eval, named after the eval. It is now one evals/eval.yaml carrying a datasets: and evaluators: catalog and an evals: list defined over them, so two evals over the same dataset name it once. Storage is confined to eval_config_store.go: where the file lives, what it is called, and how it is parsed and serialized. Every caller works with EvalConfig through OpenEvalConfig and SaveEvalConfig, so the on-disk shape has one owner. Evaluator references are keyed 'evaluator:', with 'name:' labelling the criterion and the threshold moving into initialization_parameters. A bare string is refused with the remedy through both the YAML and JSON decoders, because the service-target provider only ever sees JSON. init gains --force, replacing an eval of the same name rather than failing. A generate that runs before any init now creates eval.yaml holding only the catalog, instead of printing a hint and leaving the artifact unrecorded. The eval fingerprint covers substance only. Id, name and description are what UpdateEvalParametersBody reaches, so an edit confined to them is pushed in place and must not cost the eval its id and its run history. Fixes a bug the rewritten tests caught: addDatasetDecl dropped source-less entries, so 'init --dataset ' wrote a config whose eval referenced a dataset absent from its own catalog, which validation rejects. --- .../internal/cmd/agent_context_test.go | 9 - .../internal/cmd/build.go | 67 +- .../internal/cmd/build_live_test.go | 4 +- .../internal/cmd/build_test.go | 68 +- .../internal/cmd/catalog.go | 103 +++ .../internal/cmd/description_test.go | 4 +- .../internal/cmd/envkeys_test.go | 12 +- .../internal/cmd/evalref.go | 82 +++ .../internal/cmd/generate.go | 25 +- .../internal/cmd/generate_commands.go | 232 +++---- .../internal/cmd/generate_plan_test.go | 191 ++---- .../azure.ai.evaluations/internal/cmd/init.go | 633 ++++++++++-------- .../internal/cmd/init_test.go | 224 ++++--- .../internal/cmd/resolution_test.go | 6 +- .../azure.ai.evaluations/internal/cmd/run.go | 94 +-- .../internal/cmd/surface_test.go | 8 +- .../internal/pkg/dataset_api/operations.go | 2 +- .../internal/pkg/eval_api/models.go | 1 + .../internal/pkg/evalcore/evaluator.go | 123 ++-- .../internal/pkg/evalcore/evaluator_test.go | 98 ++- .../internal/project/artifacts.go | 99 +++ .../internal/project/eval_config.go | 374 +++++++---- .../internal/project/eval_config_store.go | 81 +++ .../internal/project/eval_config_test.go | 366 +++++----- .../internal/project/generate_config.go | 175 ----- .../internal/project/generate_config_test.go | 68 -- .../internal/project/service_target_eval.go | 59 +- .../project/service_target_eval_test.go | 100 ++- 28 files changed, 1831 insertions(+), 1477 deletions(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/catalog.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/evalref.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/project/artifacts.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_store.go delete mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/project/generate_config.go delete mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/project/generate_config_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/agent_context_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/agent_context_test.go index e6fa74be253..7333ada80eb 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/agent_context_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/agent_context_test.go @@ -10,7 +10,6 @@ import ( "testing" "azureaieval/internal/pkg/eval_api" - "azureaieval/internal/project" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -68,11 +67,3 @@ func TestAgentInstructions(t *testing.T) { var nilAgent *eval_api.Agent assert.Empty(t, nilAgent.Instructions()) } - -// Dataset generation has no model of its own; both jobs run against the one -// generation model the spec declares. -func TestGenerationModel(t *testing.T) { - assert.Empty(t, generationModel(&project.GenerateConfig{})) - assert.Equal(t, "gpt-4.1-nano", - generationModel(&project.GenerateConfig{GenerationModel: "gpt-4.1-nano"})) -} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build.go index 37f220dcab9..cdb6b5c3c4b 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build.go @@ -6,6 +6,7 @@ package cmd import ( "context" "fmt" + "maps" "sort" "strings" @@ -34,9 +35,7 @@ func (ec *evalContext) evaluatorSchemas(ctx context.Context) map[string]*eval_ap if err != nil { continue } - for name, summary := range list.ByName() { - index[name] = summary - } + maps.Copy(index, list.ByName()) } if len(index) == 0 { return nil @@ -181,6 +180,16 @@ func planCriterion( plan.itemFields = append(plan.itemFields, field) } + // A declared mapping is the author saying the inference got it wrong, so it + // wins. Anything it binds to an item column is a column the schema has to + // declare, whether or not inference found it. + for field, binding := range ref.DataMapping { + plan.dataMapping[field] = binding + if column, ok := itemColumn(binding); ok && !contains(plan.itemFields, column) { + plan.itemFields = append(plan.itemFields, column) + } + } + var missing []string for _, field := range required { if _, ok := plan.dataMapping[field]; !ok { @@ -190,15 +199,15 @@ func planCriterion( if len(missing) > 0 { return nil, fmt.Errorf( "evaluator %q requires %s, which the dataset does not provide; "+ - "add %s to the dataset, or choose an evaluator that matches the data", - ref.Name, quoteList(missing), pluralColumns(missing), + "add %s to the dataset, or bind it with `data_mapping`", + ref.Evaluator, quoteList(missing), pluralColumns(missing), ) } if !schema.SupportsLevel(level) { return nil, fmt.Errorf( "evaluator %q does not support evaluation level %q; it supports %s", - ref.Name, level, quoteList(schema.SupportedEvaluationLevels), + ref.Evaluator, level, quoteList(schema.SupportedEvaluationLevels), ) } @@ -225,9 +234,6 @@ func planCriterion( plan.initParams[alias] = value } } - if ref.Threshold != nil && accepts("threshold") { - plan.initParams["threshold"] = *ref.Threshold - } if level != "" && accepts("evaluation_level") { plan.initParams["evaluation_level"] = level } @@ -243,7 +249,7 @@ func planCriterion( return nil, fmt.Errorf( "evaluator %q requires %s; set it under the evaluator's "+ "`initialization_parameters` in the eval config", - ref.Name, quoteList(missingInit), + ref.Evaluator, quoteList(missingInit), ) } } @@ -258,6 +264,28 @@ var judgeModelAliases = map[string]string{ "model": "deployment_name", } +// itemColumn reads the dataset column out of an `{{item.}}` binding. +func itemColumn(binding string) (string, bool) { + const prefix, suffix = "{{item.", "}}" + if !strings.HasPrefix(binding, prefix) || !strings.HasSuffix(binding, suffix) { + return "", false + } + name := strings.TrimSuffix(strings.TrimPrefix(binding, prefix), suffix) + if name == "" { + return "", false + } + return name, true +} + +func contains(values []string, want string) bool { + for _, v := range values { + if v == want { + return true + } + } + return false +} + // buildEvalRequest converts an eval declaration into the create // request. Each evaluator becomes a testing criterion bound to its own // contract, and the item schema declares every dataset column those bindings @@ -289,10 +317,7 @@ func buildEvalRequest( metadata["azd_description"] = group.Description } - level := "" - if group.Options != nil { - level = group.Options.EvaluationLevel - } + level := group.EvaluationLevel req := &eval_api.CreateOpenAIEvalRequest{ Name: group.Name, @@ -302,9 +327,9 @@ func buildEvalRequest( itemFields := map[string]bool{} for _, ref := range group.Evaluators { - schema := schemas[ref.Name] + schema := schemas[ref.Evaluator] if schema == nil { - schema = &eval_api.EvaluatorSummary{Name: ref.Name} + schema = &eval_api.EvaluatorSummary{Name: ref.Evaluator} } plan, err := planCriterion(ref, schema, targetBindings, datasetColumns, level) @@ -314,11 +339,15 @@ func buildEvalRequest( criterion := eval_api.TestingCriterion{ Type: "azure_ai_evaluator", - // Name drops the builtin prefix; EvaluatorName keeps it. - Name: ref.APIName(), - EvaluatorName: ref.Name, + // Name labels the criterion in results and defaults to the + // evaluator without its builtin prefix; EvaluatorName keeps it. + Name: ref.CriterionName(), + EvaluatorName: ref.Evaluator, DataMapping: plan.dataMapping, } + if ref.Version != "" { + criterion.EvaluatorVersion = ref.Version + } if len(plan.initParams) > 0 { criterion.InitializationParameters = plan.initParams } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build_live_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build_live_test.go index f9b8edfc683..59e95d41c99 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build_live_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build_live_test.go @@ -158,10 +158,10 @@ func TestLiveBuildAcceptedForEveryBuiltin(t *testing.T) { Dataset: "inline", Target: &project.Target{Type: "agent", Name: "probe-agent"}, Evaluators: []evalcore.EvaluatorRef{{ - Name: summary.Name, + Evaluator: summary.Name, InitializationParameters: map[string]any{"deployment_name": judge}, }}, - Options: &project.Options{EvaluationLevel: level}, + EvaluationLevel: level, } req, err := buildEvalRequest(group, schemas, columns) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build_test.go index 9a17598b680..0bacd287edf 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build_test.go @@ -35,21 +35,25 @@ func schema(name string, dataRequired, dataProps, initRequired, initProps []stri } } -func groupWith(evaluators []evalcore.EvaluatorRef, opts *project.Options) *project.Eval { +func groupWith(evaluators []evalcore.EvaluatorRef, level string) *project.Eval { return &project.Eval{ - Name: "g", - Dataset: "d", - Target: &project.Target{Type: "agent", Name: "my-agent"}, - Evaluators: evaluators, - Options: opts, + Name: "g", + Dataset: "d", + Target: &project.Target{Type: "agent", Name: "my-agent"}, + Evaluators: evaluators, + EvaluationLevel: level, } } // withJudge declares the judge deployment where the service reads it from: an -// evaluator's initialization parameters, not a setting on the eval. +// evaluator's initialization parameters, not a setting on the eval. It merges, +// so a parameter the reference already carries survives. func withJudge(model string, refs ...evalcore.EvaluatorRef) []evalcore.EvaluatorRef { for i := range refs { - refs[i].InitializationParameters = map[string]any{"deployment_name": model} + if refs[i].InitializationParameters == nil { + refs[i].InitializationParameters = map[string]any{} + } + refs[i].InitializationParameters["deployment_name"] = model } return refs } @@ -64,8 +68,8 @@ func TestBuildBindsAgentFieldsFromSample(t *testing.T) { "turn"), } group := groupWith( - withJudge("gpt-4.1-nano", evalcore.EvaluatorRef{Name: "builtin.task_adherence"}), - nil, + withJudge("gpt-4.1-nano", evalcore.EvaluatorRef{Evaluator: "builtin.task_adherence"}), + "", ) req, err := buildEvalRequest(group, schemas, map[string]bool{"query": true}) @@ -89,7 +93,7 @@ func TestBuildRejectsUnsatisfiableEvaluator(t *testing.T) { []string{"response", "instruction_id_list", "instruction_kwargs"}, nil, nil, "turn"), } - group := groupWith([]evalcore.EvaluatorRef{{Name: "builtin.ifeval"}}, nil) + group := groupWith([]evalcore.EvaluatorRef{{Evaluator: "builtin.ifeval"}}, "") _, err := buildEvalRequest(group, schemas, map[string]bool{"query": true}) require.Error(t, err) @@ -105,7 +109,7 @@ func TestBuildAcceptsEvaluatorWhenDatasetSupplies(t *testing.T) { []string{"response", "instruction_id_list", "instruction_kwargs"}, nil, nil, "turn"), } - group := groupWith([]evalcore.EvaluatorRef{{Name: "builtin.ifeval"}}, nil) + group := groupWith([]evalcore.EvaluatorRef{{Evaluator: "builtin.ifeval"}}, "") req, err := buildEvalRequest(group, schemas, map[string]bool{ "instruction_id_list": true, @@ -136,9 +140,11 @@ func TestBuildOmitsUnacceptedInitParameters(t *testing.T) { []string{"deployment_name"}, []string{"deployment_name", "threshold"}, "turn"), } group := groupWith(withJudge("gpt-4.1-nano", - evalcore.EvaluatorRef{Name: "builtin.ifeval", Threshold: &threshold}, - evalcore.EvaluatorRef{Name: "builtin.similarity", Threshold: &threshold}, - ), nil) + evalcore.EvaluatorRef{Evaluator: "builtin.ifeval", + InitializationParameters: map[string]any{"threshold": threshold}}, + evalcore.EvaluatorRef{Evaluator: "builtin.similarity", + InitializationParameters: map[string]any{"threshold": threshold}}, + ), "") req, err := buildEvalRequest(group, schemas, map[string]bool{ "query": true, "ground_truth": true, @@ -168,9 +174,9 @@ func TestBuildPassesEvaluationLevelAsInitParameter(t *testing.T) { []string{"deployment_name"}, []string{"deployment_name", "threshold"}, "turn"), } group := groupWith(withJudge("m", - evalcore.EvaluatorRef{Name: "builtin.task_completion"}, - evalcore.EvaluatorRef{Name: "builtin.similarity"}, - ), &project.Options{EvaluationLevel: "turn"}) + evalcore.EvaluatorRef{Evaluator: "builtin.task_completion"}, + evalcore.EvaluatorRef{Evaluator: "builtin.similarity"}, + ), "turn") req, err := buildEvalRequest(group, schemas, map[string]bool{"query": true}) require.NoError(t, err) @@ -187,8 +193,8 @@ func TestBuildRejectsUnsupportedLevel(t *testing.T) { nil, []string{"query", "response"}, []string{"deployment_name"}, []string{"deployment_name"}, "turn"), } - group := groupWith(withJudge("m", evalcore.EvaluatorRef{Name: "builtin.similarity"}), - &project.Options{EvaluationLevel: "conversation"}) + group := groupWith(withJudge("m", evalcore.EvaluatorRef{Evaluator: "builtin.similarity"}), + "conversation") _, err := buildEvalRequest(group, schemas, map[string]bool{"query": true}) require.Error(t, err) @@ -203,7 +209,7 @@ func TestBuildRequiresJudgeModelWhenEvaluatorDoes(t *testing.T) { nil, []string{"query", "response"}, []string{"deployment_name"}, []string{"deployment_name"}, "turn"), } - group := groupWith([]evalcore.EvaluatorRef{{Name: "builtin.similarity"}}, nil) + group := groupWith([]evalcore.EvaluatorRef{{Evaluator: "builtin.similarity"}}, "") _, err := buildEvalRequest(group, schemas, map[string]bool{"query": true}) require.Error(t, err) @@ -213,7 +219,7 @@ func TestBuildRequiresJudgeModelWhenEvaluatorDoes(t *testing.T) { // An evaluator with no published contract keeps the historical agent-target // shape, so custom evaluators still deploy. func TestBuildFallsBackWithoutSchema(t *testing.T) { - group := groupWith(withJudge("m", evalcore.EvaluatorRef{Name: "my-custom-evaluator"}), nil) + group := groupWith(withJudge("m", evalcore.EvaluatorRef{Evaluator: "my-custom-evaluator"}), "") req, err := buildEvalRequest(group, nil, nil) require.NoError(t, err) @@ -238,8 +244,8 @@ func TestBuildResolvesConversationTurnExclusivity(t *testing.T) { columns := map[string]bool{"query": true, "messages": true, "response": true} // Turn level keeps query/response and drops messages. - turn := groupWith(withJudge("m", evalcore.EvaluatorRef{Name: "builtin.task_completion"}), - &project.Options{EvaluationLevel: "turn"}) + turn := groupWith(withJudge("m", evalcore.EvaluatorRef{Evaluator: "builtin.task_completion"}), + "turn") req, err := buildEvalRequest(turn, schemas, columns) require.NoError(t, err) mapping := req.TestingCriteria[0].DataMapping @@ -247,8 +253,8 @@ func TestBuildResolvesConversationTurnExclusivity(t *testing.T) { require.NotContains(t, mapping, "messages") // Conversation level keeps messages and drops query/response. - conv := groupWith(withJudge("m", evalcore.EvaluatorRef{Name: "builtin.task_completion"}), - &project.Options{EvaluationLevel: "conversation"}) + conv := groupWith(withJudge("m", evalcore.EvaluatorRef{Evaluator: "builtin.task_completion"}), + "conversation") req, err = buildEvalRequest(conv, schemas, columns) require.NoError(t, err) mapping = req.TestingCriteria[0].DataMapping @@ -257,7 +263,7 @@ func TestBuildResolvesConversationTurnExclusivity(t *testing.T) { require.NotContains(t, mapping, "response") // An unset level behaves as turn, matching the service default. - dflt := groupWith(withJudge("m", evalcore.EvaluatorRef{Name: "builtin.task_completion"}), nil) + dflt := groupWith(withJudge("m", evalcore.EvaluatorRef{Evaluator: "builtin.task_completion"}), "") req, err = buildEvalRequest(dflt, schemas, columns) require.NoError(t, err) require.NotContains(t, req.TestingCriteria[0].DataMapping, "messages") @@ -277,9 +283,9 @@ func TestBuildBindsJudgeModelUnderTheDeclaredName(t *testing.T) { []string{"model"}, []string{"model"}, "turn"), } group := groupWith(withJudge("gpt-4.1-nano", - evalcore.EvaluatorRef{Name: "builtin.similarity"}, - evalcore.EvaluatorRef{Name: "my-rubric"}, - ), nil) + evalcore.EvaluatorRef{Evaluator: "builtin.similarity"}, + evalcore.EvaluatorRef{Evaluator: "my-rubric"}, + ), "") req, err := buildEvalRequest(group, schemas, map[string]bool{"query": true}) require.NoError(t, err) @@ -302,7 +308,7 @@ func TestBuildWithoutTargetSourcesEverythingFromDataset(t *testing.T) { []string{"query", "response", "ground_truth"}, nil, nil, "turn"), } - group := groupWith([]evalcore.EvaluatorRef{{Name: "builtin.similarity"}}, nil) + group := groupWith([]evalcore.EvaluatorRef{{Evaluator: "builtin.similarity"}}, "") group.Target = nil req, err := buildEvalRequest(group, schemas, map[string]bool{ diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/catalog.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/catalog.go new file mode 100644 index 00000000000..2e34a944b12 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/catalog.go @@ -0,0 +1,103 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "fmt" + "path/filepath" + + "azureaieval/internal/project" + + "github.com/spf13/cobra" +) + +// Generation writes the artifact and then names it in the configuration, so +// what it produced is referenceable without a hand edit. Only the catalogs are +// touched: which evals use the artifact is the author's decision, and `init` is +// the command that makes it. + +// addDatasetToCatalog records a generated dataset in `datasets:`. +func addDatasetToCatalog(cmd *cobra.Command, evalDir string, ref *project.ArtifactRef) error { + if ref == nil { + return nil + } + return updateCatalog(cmd, evalDir, func(cfg *project.EvalConfig) bool { + for i := range cfg.Datasets { + if cfg.Datasets[i].Name == ref.Name { + // Regeneration overwrites the file in place, so the entry only + // changes when the artifact moved. + if cfg.Datasets[i].Source == ref.Source { + return false + } + cfg.Datasets[i].Source = ref.Source + return true + } + } + cfg.Datasets = append(cfg.Datasets, project.DatasetDecl{ + Name: ref.Name, + Source: ref.Source, + }) + return true + }) +} + +// addEvaluatorToCatalog records a generated evaluator in `evaluators:`. +func addEvaluatorToCatalog(cmd *cobra.Command, evalDir string, ref *project.ArtifactRef) error { + if ref == nil { + return nil + } + return updateCatalog(cmd, evalDir, func(cfg *project.EvalConfig) bool { + for i := range cfg.Evaluators { + if cfg.Evaluators[i].Name == ref.Name { + if cfg.Evaluators[i].Source == ref.Source { + return false + } + cfg.Evaluators[i].Source = ref.Source + return true + } + } + cfg.Evaluators = append(cfg.Evaluators, project.EvaluatorDecl{ + Name: ref.Name, + Source: ref.Source, + }) + return true + }) +} + +// updateCatalog applies a change to the configuration and writes it back. +// +// A missing configuration is created holding only the catalog. `generate` runs +// before `init` on the golden path, and a downloaded artifact nobody recorded +// is the one state that goes stale. The file it creates has no evals and no +// azure.yaml entry, so it stays inert until init wires one. +func updateCatalog( + cmd *cobra.Command, + evalDir string, + apply func(*project.EvalConfig) bool, +) error { + cfg, err := project.OpenEvalConfig(evalDir) + if err != nil { + return err + } + created := cfg == nil + if created { + cfg = &project.EvalConfig{} + } + if !apply(cfg) { + return nil + } + + if err := project.SaveEvalConfig(evalDir, cfg); err != nil { + return err + } + if !isJSON(cmd) { + path := filepath.ToSlash(project.EvalConfigPath(evalDir)) + if created { + fmt.Fprintf(cmd.OutOrStdout(), "(✓) Done: Created %s with the catalog entry\n", path) + } else { + fmt.Fprintf(cmd.OutOrStdout(), "(✓) Done: Added catalog entry to %s\n", path) + } + } + return nil +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/description_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/description_test.go index a9fb31e194f..5f60108f92d 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/description_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/description_test.go @@ -20,7 +20,7 @@ func TestBuildCarriesGroupDescriptionInMetadata(t *testing.T) { nil, []string{"query", "response"}, []string{"deployment_name"}, []string{"deployment_name"}, "turn"), } - group := groupWith(withJudge("m", evalcore.EvaluatorRef{Name: "builtin.similarity"}), nil) + group := groupWith(withJudge("m", evalcore.EvaluatorRef{Evaluator: "builtin.similarity"}), "") group.Description = "Quality gate for the support agent" req, err := buildEvalRequest(group, schemas, map[string]bool{"query": true}) @@ -35,7 +35,7 @@ func TestBuildOmitsEmptyDescription(t *testing.T) { nil, []string{"query", "response"}, []string{"deployment_name"}, []string{"deployment_name"}, "turn"), } - group := groupWith(withJudge("m", evalcore.EvaluatorRef{Name: "builtin.similarity"}), nil) + group := groupWith(withJudge("m", evalcore.EvaluatorRef{Evaluator: "builtin.similarity"}), "") req, err := buildEvalRequest(group, schemas, map[string]bool{"query": true}) require.NoError(t, err) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/envkeys_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/envkeys_test.go index b1df7509305..ccfe9719aa8 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/envkeys_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/envkeys_test.go @@ -4,10 +4,11 @@ package cmd import ( - "os" - "path/filepath" "testing" + "azureaieval/internal/pkg/evalcore" + "azureaieval/internal/project" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -48,9 +49,14 @@ func TestGroupIDKeys_SharedKeyReadOnlyWhenUnambiguous(t *testing.T) { write := func(t *testing.T, names ...string) string { t.Helper() dir := t.TempDir() + cfg := &project.EvalConfig{} for _, n := range names { - require.NoError(t, os.WriteFile(filepath.Join(dir, n+".yaml"), []byte("{}\n"), 0o600)) + cfg.Evals = append(cfg.Evals, project.Eval{ + Name: n, + Evaluators: evalcore.EvaluatorList{{Evaluator: "builtin.relevance"}}, + }) } + require.NoError(t, project.SaveEvalConfig(dir, cfg)) return dir } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evalref.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evalref.go new file mode 100644 index 00000000000..f6d2d5ddfc3 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evalref.go @@ -0,0 +1,82 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "context" + "fmt" + + "azureaieval/internal/project" +) + +// evalRef is what `--eval` resolved to: the service id, plus the declaration +// behind it when there was one. +// +// Commands need both. The id is what every route under {eval_id} takes, and the +// declaration is what says which dataset and target a new run should use. +type evalRef struct { + ID string + Eval *project.Eval + Config *project.EvalConfig + ConfigPath string +} + +// Declared reports whether the reference came from the configuration. +func (r evalRef) Declared() bool { return r.Eval != nil } + +// resolveEvalRef turns `--eval` into an id. +// +// One flag takes a name or an id, matching `azd ai training job show`, whose +// --name is documented as "Job name/ID". Name is tried first, in three cases: +// a name in evals: with a recorded id resolves to it; a name in evals: with +// none fails fast naming `azd up` rather than returning a service 404; and +// anything else is sent as an id. +// +// Ids matter because an eval created by `azd ai eval create` has no evals: +// entry, and because the environment records one id per name, so editing a +// declaration leaves every run of the previous eval reachable only by id. +func (ec *evalContext) resolveEvalRef( + ctx context.Context, + evalDir, nameOrID string, +) (evalRef, error) { + configPath := project.EvalConfigPath(evalDir) + cfg, err := project.OpenEvalConfig(evalDir) + if err != nil { + return evalRef{}, err + } + + if cfg != nil { + if err := cfg.Validate(); err != nil { + return evalRef{}, err + } + eval, err := cfg.Eval(nameOrID) + switch { + case err == nil: + id := ec.recordedEvalID(ctx, eval.Name) + if id == "" { + return evalRef{}, fmt.Errorf( + "eval %q is declared but has not been deployed to this environment yet; "+ + "run `azd up` first", eval.Name) + } + return evalRef{ID: id, Eval: eval, Config: cfg, ConfigPath: configPath}, nil + case nameOrID == "": + // No name to fall back on, so the configuration's own complaint — + // none declared, or several to choose between — is the answer. + return evalRef{}, err + } + } + + if nameOrID == "" { + return evalRef{}, fmt.Errorf( + "no eval was named and none is declared in %s; pass --eval with a name or an id", + configPath) + } + // Not a declared name, so it is an id. + return evalRef{ID: nameOrID}, nil +} + +// recordedEvalID reads the id `azd up` stored for a declared eval. +func (ec *evalContext) recordedEvalID(ctx context.Context, evalName string) string { + return ec.getEnvValue(ctx, idKey("eval", evalName)) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go index ee7514699e8..4dd77505f0d 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go @@ -44,6 +44,9 @@ type generationPlan struct { OutputDir string // SampleSize applies to dataset generation only. SampleSize int + // From selects which source the rows are generated out of. Empty lets the + // project decide: traces when Application Insights is connected. + From string // TraceDays seeds generation from that many days of recent traces. TraceDays int } @@ -78,14 +81,6 @@ func resolveInstruction(inline, path string) (string, error) { return text, nil } -// generationModel returns the deployment both generation jobs run against. -// -// Dataset generation has no model of its own: the spec carries one generation -// model and both jobs use it. -func generationModel(cfg *project.GenerateConfig) string { - return cfg.GenerationModel -} - // declaredInstructions reads the file named by a generation entry's // `instructions`, relative to the spec that declared it. // @@ -115,14 +110,14 @@ func declaredInstructions(named, configPath string) (string, error) { // // The service accepts an agent source that is meant to pull the agent's own // instructions, but it fails for every agent, so the agent's context is read -// here instead. In precedence order: what the caller passed, then the -// instructions file the spec names, then the agent's published instructions. +// here instead. In precedence order: what the caller passed, then the agent's +// published instructions. // // The last step is what makes `generate` work with no authored input at all, // which is the flow `init` sets up. func (ec *evalContext) resolveGenerationInstruction( ctx context.Context, - explicit, declared, configPath, agentName string, + explicit, agentName string, out io.Writer, quiet bool, ) (string, error) { @@ -130,14 +125,6 @@ func (ec *evalContext) resolveGenerationInstruction( return explicit, nil } - fromFile, err := declaredInstructions(declared, configPath) - if err != nil { - return "", err - } - if fromFile != "" { - return fromFile, nil - } - if agentName == "" { return "", nil } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_commands.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_commands.go index fd03dfc5bbe..b5f16b1c600 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_commands.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_commands.go @@ -5,7 +5,9 @@ package cmd import ( "fmt" + "os" "path/filepath" + "strings" "azureaieval/internal/project" @@ -17,26 +19,29 @@ import ( // partial failure undefined, cannot regenerate one artifact after the other has // been hand-edited, and gives --no-wait nothing to reattach to. // -// Neither command edits azure.yaml. `init` declares where the artifacts live; -// these fill them in, so a generation run produces a data-file-only diff. +// Neither command edits azure.yaml. Both add a catalog entry to eval.yaml for +// what they produced, so the artifact is referenceable without a hand edit. // generateFlags are the settings both generate commands share. +// +// There is no generation spec file. Every setting is a flag, because the +// artifact is checked in and a regeneration usually wants different settings +// anyway; what that costs is provenance, which is Open Question 8. type generateFlags struct { - configPath string - evalName string + path string target string instruction string instructionFile string model string outputDir string noWait bool + force bool endpoint string } func addGenerateFlags(cmd *cobra.Command, f *generateFlags) { - cmd.Flags().StringVar(&f.configPath, "config", project.DefaultGenerateConfig, - "Path to the generation spec. Optional; flags alone are sufficient.") - addEvalFlag(cmd, &f.evalName) + cmd.Flags().StringVar(&f.path, "path", project.DefaultEvalDir, + "Directory holding the evaluation configuration.") cmd.Flags().StringVar(&f.target, "target", "", "Agent whose context seeds generation.") cmd.Flags().StringVar(&f.instruction, "agent-instruction", "", "What the agent does and what to test.") @@ -46,23 +51,47 @@ func addGenerateFlags(cmd *cobra.Command, f *generateFlags) { cmd.Flags().StringVar(&f.model, "generation-model", "", "Model deployment that generates the artifact.") cmd.Flags().StringVar(&f.outputDir, "output-dir", "", - "Directory the generated artifact is written to. Overrides the generation spec.") + "Directory the generated artifact is written to.") cmd.Flags().BoolVar(&f.noWait, "no-wait", false, "Submit the job and return its id without polling.") + cmd.Flags().BoolVar(&f.force, "force", false, + "Overwrite an artifact file that already exists.") cmd.Flags().StringVar(&f.endpoint, "project-endpoint", "", "Foundry project endpoint.") } -// prepareGeneration builds the client and settles the one input that needs it. +// resolvePlan settles every input that does not need the network. // -// Everything decidable offline is already on the plan by this point, so a -// mistake in the flags has been reported without an authentication round trip. -// What is left is the generation instruction's last fallback: the agent's -// published instructions, which only the service can supply. +// Doing it before the client is built means a missing model or an out-of-range +// sample count is refused without an authentication round trip. The instruction +// file is read here rather than later so that an input the caller named and got +// wrong is reported ahead of one they simply left out. +func resolvePlan(f *generateFlags, name string, defaultOutputDir string) (generationPlan, error) { + instruction, err := resolveInstruction(f.instruction, f.instructionFile) + if err != nil { + return generationPlan{}, err + } + + plan := generationPlan{ + Name: name, + Agent: firstNonEmpty(f.target, declaredTarget(f.path)), + Model: f.model, + Instruction: instruction, + BaseDir: f.path, + OutputDir: firstNonEmpty(f.outputDir, "./"+defaultOutputDir), + } + if plan.Model == "" { + return plan, fmt.Errorf( + "a model deployment is required to generate: pass --generation-model") + } + return plan, nil +} + +// prepareGeneration builds the client and settles the one input that needs it: +// the agent's published instructions, which only the service can supply. func prepareGeneration( cmd *cobra.Command, f *generateFlags, plan generationPlan, - declared genEntry, ) (*evalContext, generationPlan, error) { ctx := cmd.Context() ec, err := newEvalContext(ctx, f.endpoint) @@ -71,8 +100,7 @@ func prepareGeneration( } plan.Instruction, err = ec.resolveGenerationInstruction( - ctx, plan.Instruction, declared.instructions, f.configPath, plan.Agent, - cmd.OutOrStdout(), isJSON(cmd), + ctx, plan.Instruction, plan.Agent, cmd.OutOrStdout(), isJSON(cmd), ) if err != nil { ec.Close() @@ -81,69 +109,21 @@ func prepareGeneration( return ec, plan, nil } -// resolvePlan settles every input that does not need the network. -// -// Resolution order is the one the spec fixes for every input: flags, then the -// generation spec, then what can be detected from the eval configuration. -// Doing it before the client is built means a missing model or an out-of-range -// sample count is refused without an authentication round trip. -// -// The instruction file is read here rather than later so that an input the -// caller named and got wrong is reported ahead of one they simply left out. -func resolvePlan( - f *generateFlags, - cfg *project.GenerateConfig, - name string, - declared genEntry, -) (generationPlan, error) { - instruction, err := resolveInstruction(f.instruction, f.instructionFile) - if err != nil { - return generationPlan{}, err - } - - plan := generationPlan{ - Name: name, - Agent: firstNonEmpty(f.target, declared.deriveFrom, evalTarget(f)), - Model: firstNonEmpty(f.model, cfg.GenerationModel), - Instruction: instruction, - BaseDir: filepath.Dir(f.configPath), - OutputDir: firstNonEmpty(f.outputDir, declared.outputDir), - SampleSize: declared.sampleSize, - TraceDays: declared.traceDays, - } - if plan.Model == "" { - return plan, fmt.Errorf( - "a model deployment is required to generate: pass --generation-model, " + - "or set `generationModel` in the generation spec") - } - return plan, nil -} - -// genEntry is the subset of a generation spec entry both commands share, so -// resolvePlan does not need to know which one it is serving. -type genEntry struct { - outputDir string - deriveFrom string - instructions string - sampleSize int - traceDays int -} - -// evalTarget reads the agent from the eval configuration, which is where the -// target is already declared, so `generate` does not need it repeated. -// -// Best effort: generation runs from the instruction alone when there is no -// eval config to read, which is the case in a bare directory. -func evalTarget(f *generateFlags) string { - path, err := project.ResolveEvalConfigPath(filepath.Dir(f.configPath), f.evalName) - if err != nil { +// declaredTarget reads the agent from the evaluation configuration, which is +// where the target is already declared, so `generate` does not need it +// repeated. Best effort: generation runs from the instruction alone when there +// is no configuration to read, which is the case in a bare directory. +func declaredTarget(evalDir string) string { + cfg, err := project.OpenEvalConfig(evalDir) + if err != nil || cfg == nil { return "" } - cfg, err := project.LoadEvalConfig(path) - if err != nil || cfg.Target == nil { - return "" + for _, eval := range cfg.Evals { + if eval.Target != nil && eval.Target.Name != "" { + return eval.Target.Name + } } - return cfg.Target.Name + return "" } func firstNonEmpty(values ...string) string { @@ -155,74 +135,63 @@ func firstNonEmpty(values ...string) string { return "" } -// datasetGenEntry reads one dataset's settings out of the generation spec, -// applying the flag override and the default row count. -func datasetGenEntry(cfg *project.GenerateConfig, name string, maxSamples int) genEntry { - spec, _ := cfg.DatasetSpec(name) - entry := genEntry{ - outputDir: firstNonEmpty(spec.OutputDir, "./"+project.DefaultDatasetsDir), - deriveFrom: spec.DeriveFrom, - instructions: spec.Instructions, - sampleSize: spec.SampleSize, - traceDays: spec.TraceDays, - } - if maxSamples > 0 { - entry.sampleSize = maxSamples - } - if entry.sampleSize == 0 { - entry.sampleSize = project.DefaultSampleSize - } - return entry -} - -// evaluatorGenEntry reads one evaluator's settings out of the generation spec, -// applying the flag override. -func evaluatorGenEntry(cfg *project.GenerateConfig, name string, traceDays int) genEntry { - spec, _ := cfg.EvaluatorSpec(name) - entry := genEntry{ - outputDir: firstNonEmpty(spec.OutputDir, "./"+project.DefaultEvaluatorsDir), - deriveFrom: spec.DeriveFrom, - instructions: spec.Instructions, - traceDays: spec.TraceDays, +// refuseExistingArtifact stops a generation that would overwrite a checked-in +// file, because the job is billed and the diff is what the author reviews. +func refuseExistingArtifact(path string, force bool) error { + if force { + return nil } - if traceDays > 0 { - entry.traceDays = traceDays + if _, err := os.Stat(path); err == nil { + return fmt.Errorf( + "%s already exists; pass --force to overwrite it, or --output-dir to write elsewhere", + filepath.ToSlash(path)) } - return entry + return nil } func newDatasetGenerateCommand() *cobra.Command { var ( flags generateFlags maxSamples int + from string ) cmd := &cobra.Command{ Use: "generate ", Short: "Generate a dataset and download it.", - Args: cobra.ExactArgs(1), + Long: "Generate a dataset and download it.\n\n" + + "--from selects one of the four sources the service accepts. " + + "Generating from the agent's own definition is a preference rather " + + "than a fallback: it covers cases no user has hit yet, and it can " + + "supply reference answers, which a transcript cannot.", + Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { name := args[0] - if err := project.ValidateSampleSize(maxSamples); err != nil { + if err := project.ValidateGenerateSource(from); err != nil { return err } - - cfg, err := project.LoadGenerateConfig(flags.configPath) - if err != nil { + if err := project.ValidateSampleSize(maxSamples); err != nil { return err } - declared := datasetGenEntry(cfg, name, maxSamples) - plan, err := resolvePlan(&flags, cfg, name, declared) + plan, err := resolvePlan(&flags, name, project.DefaultDatasetsDir) if err != nil { return err } - if err := project.ValidateSampleSize(plan.SampleSize); err != nil { + plan.From = from + plan.SampleSize = maxSamples + if plan.SampleSize == 0 { + plan.SampleSize = project.DefaultSampleSize + } + if err := refuseExistingArtifact( + project.ArtifactPath(plan.BaseDir, plan.OutputDir, name, ".jsonl"), + flags.force, + ); err != nil { return err } - ec, plan, err := prepareGeneration(cmd, &flags, plan, declared) + ec, plan, err := prepareGeneration(cmd, &flags, plan) if err != nil { return err } @@ -232,12 +201,20 @@ func newDatasetGenerateCommand() *cobra.Command { if err != nil { return err } + if err := addDatasetToCatalog(cmd, flags.path, ref); err != nil { + return err + } return reportGenerated(cmd, ref, flags.noWait) }, } cmd.Flags().IntVar(&maxSamples, "max-samples", 0, - fmt.Sprintf("Rows to synthesize (%d-%d).", project.MinSampleSize, project.MaxSampleSize)) + fmt.Sprintf("Rows to synthesize (%d-%d). Defaults to %d.", + project.MinSampleSize, project.MaxSampleSize, project.DefaultSampleSize)) + cmd.Flags().StringVar(&from, "from", "", + fmt.Sprintf("Where rows come from: %s. Defaults to traces when the project "+ + "has Application Insights connected, otherwise agent.", + strings.Join(project.GenerateSources, ", "))) addGenerateFlags(cmd, &flags) return cmd } @@ -255,18 +232,19 @@ func newEvaluatorGenerateCommand() *cobra.Command { RunE: func(cmd *cobra.Command, args []string) error { name := args[0] - cfg, err := project.LoadGenerateConfig(flags.configPath) + plan, err := resolvePlan(&flags, name, project.DefaultEvaluatorsDir) if err != nil { return err } - declared := evaluatorGenEntry(cfg, name, traceDays) - - plan, err := resolvePlan(&flags, cfg, name, declared) - if err != nil { + plan.TraceDays = traceDays + if err := refuseExistingArtifact( + project.ArtifactPath(plan.BaseDir, plan.OutputDir, name, ".json"), + flags.force, + ); err != nil { return err } - ec, plan, err := prepareGeneration(cmd, &flags, plan, declared) + ec, plan, err := prepareGeneration(cmd, &flags, plan) if err != nil { return err } @@ -276,6 +254,9 @@ func newEvaluatorGenerateCommand() *cobra.Command { if err != nil { return err } + if err := addEvaluatorToCatalog(cmd, flags.path, ref); err != nil { + return err + } return reportGenerated(cmd, ref, flags.noWait) }, } @@ -289,7 +270,7 @@ func newEvaluatorGenerateCommand() *cobra.Command { // reportGenerated closes out either command. // // With --no-wait nothing was downloaded and there is no ref, which is success: -// reportSubmitted has already said how to reattach. +// the submission message has already said how to reattach. func reportGenerated(cmd *cobra.Command, ref *project.ArtifactRef, noWait bool) error { out := cmd.OutOrStdout() if ref == nil { @@ -301,6 +282,5 @@ func reportGenerated(cmd *cobra.Command, ref *project.ArtifactRef, noWait bool) if isJSON(cmd) { return emitJSON(out, ref) } - fmt.Fprintf(out, "\nReference it from your eval config as: %s\n", ref.Source) return nil } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_plan_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_plan_test.go index 9e4f9bd3e5d..275e194044d 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_plan_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_plan_test.go @@ -8,65 +8,77 @@ import ( "path/filepath" "testing" + "azureaieval/internal/pkg/evalcore" "azureaieval/internal/project" "github.com/stretchr/testify/require" ) // `generate` decides what to submit before it touches the network, so the plan -// it builds — which agent, what model, where the artifact lands, at what sample -// size — is checkable without paying for a generation job. These are the parts -// that cannot be observed afterwards: once the job is submitted, a wrong -// default is indistinguishable from an intended one. +// it builds — which agent, what model, where the artifact lands — is checkable +// without paying for a generation job. These are the parts that cannot be +// observed afterwards: once the job is submitted, a wrong default is +// indistinguishable from an intended one. -// evalsDir writes a generation spec and returns the flags pointing at it. -func evalsDir(t *testing.T, generateBody string, files map[string]string) *generateFlags { +// evalsDir returns flags pointing at an empty eval directory. +func evalsDir(t *testing.T) *generateFlags { t.Helper() - dir := t.TempDir() - configPath := filepath.Join(dir, "generate.yaml") - if generateBody != "" { - require.NoError(t, os.WriteFile(configPath, []byte(generateBody), 0o600)) - } - for name, body := range files { - require.NoError(t, os.WriteFile(filepath.Join(dir, name), []byte(body), 0o600)) - } - return &generateFlags{configPath: configPath} + return &generateFlags{path: t.TempDir()} } -func loadSpec(t *testing.T, f *generateFlags) *project.GenerateConfig { +// withEvals writes a configuration into the flags' directory. +func withEvals(t *testing.T, f *generateFlags, evals ...project.Eval) { t.Helper() - cfg, err := project.LoadGenerateConfig(f.configPath) - require.NoError(t, err) - return cfg + require.NoError(t, project.SaveEvalConfig(f.path, &project.EvalConfig{Evals: evals})) } -// A spec is optional, so flags alone are what most callers actually run with. +// Generation settings are flags only: there is no generate.yaml, because the +// artifact is checked in and regeneration usually wants different settings. func TestResolvePlan_FromFlagsAlone(t *testing.T) { - f := evalsDir(t, "", nil) + f := evalsDir(t) f.target = "shop-agent" f.model = "gpt-4o-mini" - plan, err := resolvePlan(f, loadSpec(t, f), "shop-golden", - datasetGenEntry(loadSpec(t, f), "shop-golden", 0)) + plan, err := resolvePlan(f, "shop-golden", project.DefaultDatasetsDir) require.NoError(t, err) require.Equal(t, "shop-golden", plan.Name) require.Equal(t, "shop-agent", plan.Agent) require.Equal(t, "gpt-4o-mini", plan.Model) require.Equal(t, "./"+project.DefaultDatasetsDir, plan.OutputDir) - require.Equal(t, project.DefaultSampleSize, plan.SampleSize) + require.Equal(t, f.path, plan.BaseDir) +} + +// Each generate has its own default output directory, so a rubric never lands +// in the datasets folder. +func TestResolvePlan_OutputDirDefaultsPerArtifact(t *testing.T) { + f := evalsDir(t) + f.target = "shop-agent" + f.model = "gpt-4o-mini" + + ds, err := resolvePlan(f, "d", project.DefaultDatasetsDir) + require.NoError(t, err) + require.Equal(t, "./"+project.DefaultDatasetsDir, ds.OutputDir) + + ev, err := resolvePlan(f, "r", project.DefaultEvaluatorsDir) + require.NoError(t, err) + require.Equal(t, "./"+project.DefaultEvaluatorsDir, ev.OutputDir) + + f.outputDir = "./from-flag" + override, err := resolvePlan(f, "d", project.DefaultDatasetsDir) + require.NoError(t, err) + require.Equal(t, "./from-flag", override.OutputDir) } // Without a model there is nothing to bill the job against, and the refusal has -// to name both ways of supplying one. +// to name the flag that supplies one. func TestResolvePlan_RequiresAGenerationModel(t *testing.T) { - f := evalsDir(t, "", nil) + f := evalsDir(t) f.target = "shop-agent" - _, err := resolvePlan(f, loadSpec(t, f), "d", genEntry{}) + _, err := resolvePlan(f, "d", project.DefaultDatasetsDir) require.Error(t, err) require.Contains(t, err.Error(), "--generation-model") - require.Contains(t, err.Error(), "generationModel") } // An input the caller named and got wrong is reported ahead of one they simply @@ -74,112 +86,51 @@ func TestResolvePlan_RequiresAGenerationModel(t *testing.T) { // sees is the order they run in — and a missing instruction file is a typo the // caller can act on, while the model has a documented default path. func TestResolvePlan_ReportsABadExplicitInputFirst(t *testing.T) { - f := evalsDir(t, "", nil) + f := evalsDir(t) f.target = "shop-agent" f.instructionFile = filepath.Join(t.TempDir(), "absent.md") - _, err := resolvePlan(f, loadSpec(t, f), "d", genEntry{}) + _, err := resolvePlan(f, "d", project.DefaultDatasetsDir) require.Error(t, err) require.Contains(t, err.Error(), "--agent-instruction-file", "the flag the caller got wrong must win over the one they omitted") } -// The spec is read per artifact name, so generating one artifact never picks up -// the other's settings. -func TestResolvePlan_ReadsTheNamedSpecEntry(t *testing.T) { - f := evalsDir(t, ` -generationModel: gpt-4o -dataset: - spec-dataset: - sampleSize: 200 - outputDir: ./custom-datasets - deriveFrom: from-spec -evaluator: - spec-rubric: - outputDir: ./custom-evaluators - deriveFrom: rubric-agent -`, nil) - - cfg := loadSpec(t, f) - - ds, err := resolvePlan(f, cfg, "spec-dataset", datasetGenEntry(cfg, "spec-dataset", 0)) - require.NoError(t, err) - require.Equal(t, "gpt-4o", ds.Model) - require.Equal(t, 200, ds.SampleSize) - require.Equal(t, "./custom-datasets", ds.OutputDir) - require.Equal(t, "from-spec", ds.Agent) - - ev, err := resolvePlan(f, cfg, "spec-rubric", evaluatorGenEntry(cfg, "spec-rubric", 0)) - require.NoError(t, err) - require.Equal(t, "./custom-evaluators", ev.OutputDir) - require.Equal(t, "rubric-agent", ev.Agent) +// The target is already declared on an eval, so `generate` does not need it +// repeated on every invocation. +func TestResolvePlan_FallsBackToTheDeclaredTarget(t *testing.T) { + f := evalsDir(t) + f.model = "gpt-4o" + withEvals(t, f, project.Eval{ + Name: "support-agent-eval", + Evaluators: evalcore.EvaluatorList{{Evaluator: "builtin.relevance"}}, + Target: &project.Target{Type: project.TargetTypeAgent, Name: "support-agent"}, + }) - // An artifact the spec says nothing about still generates, on the defaults. - other, err := resolvePlan(f, cfg, "unlisted", datasetGenEntry(cfg, "unlisted", 0)) + plan, err := resolvePlan(f, "d", project.DefaultDatasetsDir) require.NoError(t, err) - require.Equal(t, project.DefaultSampleSize, other.SampleSize) - require.Equal(t, "./"+project.DefaultDatasetsDir, other.OutputDir) -} + require.Equal(t, "support-agent", plan.Agent, + "the declared target is the agent to generate from") -// Flags win over the spec, which is what makes a one-off run possible without -// editing a file that is checked in. -func TestResolvePlan_LayersFlagsOverTheSpec(t *testing.T) { - f := evalsDir(t, ` -generationModel: gpt-4o -dataset: - spec-dataset: - sampleSize: 200 - outputDir: ./custom-datasets - deriveFrom: from-spec -`, nil) + // An explicit flag still wins, which is what makes a one-off run possible + // without editing a file that is checked in. f.target = "from-flag" - f.model = "gpt-4o-mini" - f.outputDir = "./from-flag-dir" - - cfg := loadSpec(t, f) - plan, err := resolvePlan(f, cfg, "spec-dataset", datasetGenEntry(cfg, "spec-dataset", 500)) + plan, err = resolvePlan(f, "d", project.DefaultDatasetsDir) require.NoError(t, err) - require.Equal(t, "from-flag", plan.Agent) - require.Equal(t, "gpt-4o-mini", plan.Model) - require.Equal(t, "./from-flag-dir", plan.OutputDir) - require.Equal(t, 500, plan.SampleSize) } -// The target is already declared on the eval, so `generate` does not need it -// repeated on every invocation. -func TestResolvePlan_FallsBackToTheEvalTarget(t *testing.T) { - f := evalsDir(t, "generationModel: gpt-4o\n", map[string]string{ - "support-agent-smoke.yaml": "evaluators: [builtin.relevance]\n" + - "target:\n type: agent\n name: support-agent\n", - }) +// With no configuration at all, generation runs from the instruction alone. +// This is the golden path: both generates precede init. +func TestResolvePlan_NoConfigurationYet(t *testing.T) { + f := evalsDir(t) + f.model = "gpt-4o" + f.instruction = "test refunds and returns" - cfg := loadSpec(t, f) - plan, err := resolvePlan(f, cfg, "support-agent-smoke", - datasetGenEntry(cfg, "support-agent-smoke", 0)) - require.NoError(t, err) - require.Equal(t, "support-agent", plan.Agent, - "the eval's declared target is the agent to generate from") -} - -// With more than one eval the target is ambiguous, so nothing is guessed: -// generation falls back to the instruction alone rather than picking one. -func TestResolvePlan_AmbiguousEvalTargetIsNotGuessed(t *testing.T) { - f := evalsDir(t, "generationModel: gpt-4o\n", map[string]string{ - "a.yaml": "target:\n type: agent\n name: agent-a\n", - "b.yaml": "target:\n type: agent\n name: agent-b\n", - }) - - cfg := loadSpec(t, f) - plan, err := resolvePlan(f, cfg, "d", datasetGenEntry(cfg, "d", 0)) + plan, err := resolvePlan(f, "d", project.DefaultDatasetsDir) require.NoError(t, err) require.Empty(t, plan.Agent) - - // Naming one resolves it. - f.evalName = "b" - plan, err = resolvePlan(f, cfg, "d", datasetGenEntry(cfg, "d", 0)) - require.NoError(t, err) - require.Equal(t, "agent-b", plan.Agent) + require.Equal(t, "test refunds and returns", plan.Instruction) } // The bounds are the service's, and the boundaries themselves have to be @@ -206,16 +157,6 @@ func TestGenerateSampleSizeBounds(t *testing.T) { } } -// Trace days come from the spec, and the flag overrides them. -func TestEvaluatorGenEntry_TraceDays(t *testing.T) { - cfg := &project.GenerateConfig{ - Evaluator: map[string]project.EvaluatorGenSpec{"r": {TraceDays: 7}}, - } - require.Equal(t, 7, evaluatorGenEntry(cfg, "r", 0).traceDays) - require.Equal(t, 30, evaluatorGenEntry(cfg, "r", 30).traceDays) - require.Zero(t, evaluatorGenEntry(cfg, "absent", 0).traceDays) -} - func TestResolveInstruction(t *testing.T) { dir := t.TempDir() filled := filepath.Join(dir, "instruction.md") diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go index 2bdc4f4477f..cd80a2ac7d5 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go @@ -7,6 +7,7 @@ import ( "context" "errors" "fmt" + "maps" "os" "path/filepath" "strings" @@ -20,16 +21,28 @@ import ( "google.golang.org/protobuf/types/known/structpb" ) +// Data sources `init` can point an eval at. +const ( + initSourceDataset = "dataset" + initSourceTraces = "traces" +) + // newInitCommand scaffolds the eval configuration. It makes no service calls at // all, so it works offline and unauthenticated. +// +// It only ever adds. A name already declared is refused rather than +// overwritten, because the settings a reader tunes by hand — thresholds, judge +// model, data mapping — live nowhere but that entry and `init` cannot +// reproduce them. Editing an eval is a file edit. func newInitCommand() *cobra.Command { var ( evalName string target string + source string dataset string evaluators []string - genModel string - outputDir string + judgeModel string + path string force bool ) @@ -42,91 +55,117 @@ func newInitCommand() *cobra.Command { if target == "" { return requireFlag("target") } - if outputDir == "" { - outputDir = project.DefaultEvalDir + switch source { + case "", initSourceDataset, initSourceTraces: + default: + return fmt.Errorf( + "--source %q is not a data source; use %q or %q", + source, initSourceDataset, initSourceTraces) + } + if source == initSourceTraces && dataset != "" { + return errors.New("--source traces reads production traces, so it takes no --dataset") + } + if source == "" { + source = initSourceDataset + } + if path == "" { + path = project.DefaultEvalDir } if evalName == "" { - evalName = target + "-smoke" + evalName = defaultEvalName(target, source) } - evalPath := project.EvalConfigPath(outputDir, evalName) - genPath := filepath.Join(outputDir, "generate.yaml") - - for _, p := range []string{evalPath, genPath} { - if _, err := os.Stat(p); err == nil && !force { - return fmt.Errorf("%s already exists; pass --force to overwrite", p) - } + // Asked before anything is written: the project is the one thing + // init cannot supply for itself, and failing after creating + // directories leaves a half-scaffolded tree behind. + azdProject, err := readAzdProject(cmd.Context()) + if err != nil { + return err + } + if judgeModel == "" { + judgeModel = detectModelDeployment(azdProject) } - // Asked before anything is written: the project is the one thing init - // cannot supply for itself, and failing after creating directories - // leaves a half-scaffolded tree behind for the user to clean up. - azdProject, err := readAzdProject(cmd.Context()) + configPath := project.EvalConfigPath(path) + cfg, err := project.OpenEvalConfig(path) if err != nil { return err } - if genModel == "" { - genModel = detectModelDeployment(azdProject) + if cfg == nil { + cfg = &project.EvalConfig{} + } + if cfg.HasEval(evalName) { + if !force { + return fmt.Errorf( + "an eval named %q already exists in %s; choose another name with --name, "+ + "or pass --force to replace it. `init` only adds: editing an eval is a file edit", + evalName, filepath.ToSlash(configPath)) + } + cfg.RemoveEval(evalName) } - if err := os.MkdirAll(filepath.Join(outputDir, project.DefaultDatasetsDir), 0o750); err != nil { + if err := os.MkdirAll(filepath.Join(path, project.DefaultDatasetsDir), 0o750); err != nil { return fmt.Errorf("creating the datasets directory: %w", err) } - if err := os.MkdirAll(filepath.Join(outputDir, project.DefaultEvaluatorsDir), 0o750); err != nil { + if err := os.MkdirAll(filepath.Join(path, project.DefaultEvaluatorsDir), 0o750); err != nil { return fmt.Errorf("creating the evaluators directory: %w", err) } - rubricName := target + "-quality" - - plan := planScaffold(evalName, target, rubricName, dataset, evaluators, genModel, outputDir) - - if err := writeYAML(evalPath, plan.eval); err != nil { - return err - } - if err := writeYAML(genPath, plan.generate); err != nil { + plan := planScaffold(scaffoldInput{ + evalName: evalName, + target: target, + source: source, + dataset: dataset, + evaluators: evaluators, + judgeModel: judgeModel, + rubricName: target + "-quality", + evalDir: path, + cfg: cfg, + }) + + if err := project.SaveEvalConfig(path, cfg); err != nil { return err } // Scaffolding a config azd cannot see is half a step: the eval // service has to be referenced from the root config before any of // `azd up`, `azd deploy` or `azd ai eval run` will act on it. - // Printing the block and leaving the edit to the reader was enough - // to make the documented flow stop working between `init` and - // `azd up`. - rootWiring, err := ensureRootEvalService(cmd.Context(), evalName, target, evalPath) + serviceName := target + "-evals" + rootWiring, err := ensureRootEvalService(cmd.Context(), serviceName, target, configPath) if err != nil { return err } if isJSON(cmd) { return emitJSON(out, map[string]any{ - "eval": evalName, - "evalConfig": evalPath, - "generateConfig": genPath, - "datasetsDir": filepath.Join(outputDir, project.DefaultDatasetsDir), - "evaluatorsDir": filepath.Join(outputDir, project.DefaultEvaluatorsDir), - "rootConfig": rootWiring, - "target": target, - "generationModel": genModel, - "evaluators": plan.evaluatorNames(), + "eval": evalName, + "evalConfig": configPath, + "service": serviceName, + "datasetsDir": filepath.Join(path, project.DefaultDatasetsDir), + "evaluatorsDir": filepath.Join(path, project.DefaultEvaluatorsDir), + "rootConfig": rootWiring, + "target": target, + "source": source, + "judgeModel": judgeModel, + "evaluators": plan.evaluatorNames(), }) } fmt.Fprintf(out, "%s Detected agent target: %s\n", doneMark, target) - if genModel != "" { - fmt.Fprintf(out, "%s Detected model deployment: %s\n", doneMark, genModel) + if source == initSourceTraces { + fmt.Fprintf(out, "%s Using data source: traces (Application Insights)\n", doneMark) + } + if judgeModel != "" { + fmt.Fprintf(out, "%s Judge model deployment: %s\n", doneMark, judgeModel) } - fmt.Fprintf(out, "%s Planned evaluators: %s\n", doneMark, plan.evaluatorSummary()) fmt.Fprintln(out, "\nCreated") - fmt.Fprintf(out, " %-33s eval definition\n", filepath.ToSlash(evalPath)) - fmt.Fprintf(out, " %-33s generation settings (%d samples, %d rubric)\n", - filepath.ToSlash(genPath), project.DefaultSampleSize, plan.rubricCount()) + fmt.Fprintf(out, " %-33s evaluation configuration\n", filepath.ToSlash(configPath)) switch rootWiring { case wiringAdded: - fmt.Fprintf(out, " %-33s added service '%s'\n", rootConfigName, evalName) + fmt.Fprintf(out, " %-33s added service '%s'\n", rootConfigName, serviceName) case wiringPresent: - fmt.Fprintf(out, " %-33s already declares service '%s'\n", rootConfigName, evalName) + fmt.Fprintf(out, " %-33s already declares service '%s'\n", rootConfigName, serviceName) } // Only what was actually scheduled is offered. Suggesting @@ -141,20 +180,241 @@ func newInitCommand() *cobra.Command { }, } - cmd.Flags().StringVar(&evalName, "name", "", "Name of the eval. Defaults to -smoke.") + cmd.Flags().StringVar(&evalName, "name", "", + "Name of the eval. Defaults to -eval, or -trace-eval under --source traces.") cmd.Flags().StringVar(&target, "target", "", "Name of the agent to evaluate.") - cmd.Flags().StringVar(&dataset, "dataset", "", "Path to a local .jsonl, or the name of a registered dataset.") + cmd.Flags().StringVar(&source, "source", "", + "Where rows come from: dataset or traces. Defaults to dataset.") + cmd.Flags().StringVar(&dataset, "dataset", "", + "Path to a local .jsonl, or the name of a registered dataset.") cmd.Flags().StringArrayVar(&evaluators, "evaluator", nil, "Evaluator reference, repeatable. Use builtin. for a built-in. "+ "Passing this replaces the defaults, so it also opts out of rubric generation.") - cmd.Flags().StringVar(&genModel, "generation-model", "", - "Model deployment that generates and judges. Detected from the project when omitted.") - cmd.Flags().StringVar(&outputDir, "output-dir", project.DefaultEvalDir, - "Directory to write the config into. Used verbatim, never re-rooted.") - cmd.Flags().BoolVar(&force, "force", false, "Overwrite existing files.") + cmd.Flags().StringVar(&judgeModel, "judge-model", "", + "Model deployment the graders judge with. Detected from the project when omitted.") + cmd.Flags().StringVar(&path, "path", project.DefaultEvalDir, + "Directory to write the configuration into. Used verbatim, never re-rooted.") + cmd.Flags().BoolVar(&force, "force", false, + "Replace an eval of the same name instead of failing.") return cmd } +// defaultEvalName names an eval after what it evaluates and what it reads. +func defaultEvalName(target, source string) string { + if source == initSourceTraces { + return target + "-trace-eval" + } + return target + "-eval" +} + +// scaffoldInput is everything planScaffold needs, gathered so the signature +// does not grow a seventh positional string. +type scaffoldInput struct { + evalName string + target string + source string + dataset string + evaluators []string + judgeModel string + rubricName string + evalDir string + cfg *project.EvalConfig +} + +// scaffold is what `init` added, and what it should suggest doing next. +type scaffold struct { + eval *project.Eval + datasetName string + rubricName string + generateDataset bool + generateRubric bool +} + +// planScaffold appends one eval to the configuration, adding any catalog +// entries it needs. +// +// The default evaluator set is a built-in plus a generated rubric: the built-in +// alone would be generic, and the rubric is what makes the baseline about this +// agent. Passing --evaluator replaces both, which is how a caller opts out of +// rubric generation. +func planScaffold(in scaffoldInput) scaffold { + cfg := in.cfg + out := scaffold{rubricName: in.rubricName} + + eval := project.Eval{ + Name: in.evalName, + Description: fmt.Sprintf("Basic quality evaluation for %s", in.target), + EvaluationLevel: project.EvaluationLevelTurn, + Target: &project.Target{ + Type: project.TargetTypeAgent, + Name: in.target, + }, + } + + if in.source == initSourceTraces { + // A trace-backed eval filters by agent rather than invoking one: the + // conversations already happened. + eval.Target = nil + eval.Source = &project.SourceDecl{ + Type: project.SourceTypeTraces, + AgentName: in.target, + } + } else { + datasetName := in.evalName + datasetSource := "" + out.generateDataset = true + if in.dataset != "" { + out.generateDataset = false + if looksLikeLocalDataset(in.dataset) { + // --dataset is given relative to where the user is standing, + // but source: resolves relative to the config, so the path has + // to be rebased or the deploy looks for it inside evals/. + datasetSource = relativeToConfig(in.dataset, in.evalDir) + datasetName = strings.TrimSuffix( + filepath.Base(in.dataset), filepath.Ext(in.dataset)) + } else { + // A bare name references an already-registered dataset. + datasetName = in.dataset + } + } else { + datasetSource = fmt.Sprintf("./%s/%s.jsonl", project.DefaultDatasetsDir, datasetName) + } + eval.Dataset = datasetName + out.datasetName = datasetName + addDatasetDecl(cfg, project.DatasetDecl{Name: datasetName, Source: datasetSource}) + } + + // Every evaluator carries the judge deployment, because that is where the + // service reads it from: judging built-ins declare it as required, so an + // eval that leaves it off is rejected before it runs. The binding step + // drops it again for a rule-based evaluator that declares no judge. + initParams := map[string]any{} + if in.judgeModel != "" { + initParams["model"] = in.judgeModel + } + withModel := func(ref evalcore.EvaluatorRef) evalcore.EvaluatorRef { + if len(initParams) == 0 { + return ref + } + params := make(map[string]any, len(initParams)) + maps.Copy(params, initParams) + ref.InitializationParameters = params + return ref + } + + refs := evalcore.EvaluatorList{} + if len(in.evaluators) == 0 { + refs = append(refs, + withModel(evalcore.EvaluatorRef{ + Evaluator: evalcore.BuiltinPrefix + "task_adherence", + })) + if in.source != initSourceTraces { + refs = append(refs, withModel(evalcore.EvaluatorRef{Evaluator: in.rubricName})) + addEvaluatorDecl(cfg, project.EvaluatorDecl{ + Name: in.rubricName, + Source: fmt.Sprintf("./%s/%s.json", project.DefaultEvaluatorsDir, in.rubricName), + }) + out.generateRubric = true + } + } else { + for _, e := range in.evaluators { + ref := evalcore.EvaluatorRef{Evaluator: e} + refs = append(refs, withModel(ref)) + if ref.IsBuiltin() { + continue + } + addEvaluatorDecl(cfg, project.EvaluatorDecl{ + Name: e, + Source: fmt.Sprintf("./%s/%s.json", project.DefaultEvaluatorsDir, e), + }) + } + } + eval.Evaluators = refs + + cfg.Evals = append(cfg.Evals, eval) + out.eval = &cfg.Evals[len(cfg.Evals)-1] + return out +} + +// addDatasetDecl adds a catalog entry unless the name is already declared. +func addDatasetDecl(cfg *project.EvalConfig, decl project.DatasetDecl) { + if decl.Name == "" { + return + } + // A source-less entry is still declared: it names a dataset already + // registered on the project. Skipping it left the eval referencing a + // dataset absent from the catalog, which its own validation rejects. + if _, ok := cfg.DatasetDeclaration(decl.Name); ok { + return + } + cfg.Datasets = append(cfg.Datasets, decl) +} + +// addEvaluatorDecl adds a catalog entry unless the name is already declared. +func addEvaluatorDecl(cfg *project.EvalConfig, decl project.EvaluatorDecl) { + if _, ok := cfg.EvaluatorDeclaration(decl.Name); ok { + return + } + cfg.Evaluators = append(cfg.Evaluators, decl) +} + +// evaluatorNames lists the evaluators the eval will run, in declaration order. +func (s scaffold) evaluatorNames() []string { + names := make([]string, 0, len(s.eval.Evaluators)) + for _, ref := range s.eval.Evaluators { + names = append(names, ref.Evaluator) + } + return names +} + +// nextSteps are the commands to run after `init`, and only the ones that have +// something to do. +// +// A caller who supplied both a dataset and their evaluators has nothing left to +// generate, and pointing them at a generation command would submit a billed job +// for an artifact they already have. +func (s scaffold) nextSteps() []string { + var steps []string + if s.generateDataset { + steps = append(steps, "azd ai dataset generate "+s.datasetName) + } + if s.generateRubric { + steps = append(steps, "azd ai eval evaluator generate "+s.rubricName) + } + if len(steps) == 0 { + steps = append(steps, "azd up", "azd ai eval run start") + } + return steps +} + +// relativeToConfig rewrites a path given relative to the working directory so +// it resolves from the directory holding the eval config. +func relativeToConfig(path, evalDir string) string { + if filepath.IsAbs(path) { + return path + } + + absPath, err := filepath.Abs(path) + if err != nil { + return path + } + absOut, err := filepath.Abs(evalDir) + if err != nil { + return path + } + + rel, err := filepath.Rel(absOut, absPath) + if err != nil { + return path + } + + rel = filepath.ToSlash(rel) + if !strings.HasPrefix(rel, ".") { + rel = "./" + rel + } + return rel +} + // rootConfigName is azd's project file, which the eval service is declared in. const rootConfigName = "azure.yaml" @@ -174,10 +434,6 @@ const noAzdProject = "no azd project found in this directory. Run `azd init` fir "its azure.yaml" // readAzdProject returns the project, without changing it. -// -// It is read before anything is written: the project is the one thing init -// cannot supply for itself, and it also carries the agent and model detection -// that `init` reports. func readAzdProject(ctx context.Context) (*azdext.ProjectConfig, error) { azdClient, err := azdext.NewAzdClient() if err != nil { @@ -197,12 +453,11 @@ func readAzdProject(ctx context.Context) (*azdext.ProjectConfig, error) { // call. const aiModelHost = "azure.ai.model" -// detectModelDeployment finds the deployment generation and judging run -// against, from what the project already declares. +// detectModelDeployment finds the deployment the graders judge with, from what +// the project already declares. // // `init` makes no service calls, so detection is limited to the project file. -// Coming back empty is not a failure: --generation-model supplies it, and the -// generate commands say so when it is missing. +// Coming back empty is not a failure: --judge-model supplies it. func detectModelDeployment(proj *azdext.ProjectConfig) string { for name, svc := range proj.GetServices() { if svc.GetHost() != aiModelHost { @@ -226,12 +481,10 @@ func detectModelDeployment(proj *azdext.ProjectConfig) string { // than described. It goes through azd's own Project().AddService, the same call // the agents extension uses, so azd owns the edit and the project file keeps // whatever shape azd gives it. -// -// The service key is the eval's name — one `azure.ai.eval` service per eval — -// and the eval body stays in evals/.yaml, referenced with `$ref`. -// azd carries unknown keys through AdditionalProperties untouched, which is how -// the extension gets it back at deploy time. -func ensureRootEvalService(ctx context.Context, evalName, target, evalPath string) (string, error) { +func ensureRootEvalService( + ctx context.Context, + serviceName, target, configPath string, +) (string, error) { azdClient, err := azdext.NewAzdClient() if err != nil { return "", fmt.Errorf("connecting to azd: %w", err) @@ -240,23 +493,17 @@ func ensureRootEvalService(ctx context.Context, evalName, target, evalPath strin resp, err := azdClient.Project().Get(ctx, &azdext.EmptyRequest{}) if err != nil || resp.GetProject() == nil { - // Evals attach to a project; they do not create one. Saying which - // command makes one is more use than a gRPC error. - return "", fmt.Errorf( - "no azd project found in this directory. Run `azd init` first, "+ - "or run this from the root of an existing one; the eval service is "+ - "added to its %s", rootConfigName) - } - - // A service already declaring this eval is left alone: re-adding it would - // deploy the same eval twice. A differently-named eval service is not a - // conflict, because one service is one eval. - if svc, ok := resp.GetProject().GetServices()[evalName]; ok && svc.GetHost() == project.EvalHost { + return "", errors.New(noAzdProject) + } + + // A service already pointing at this configuration is left alone: + // re-adding it would deploy the same evals twice. + if svc, ok := resp.GetProject().GetServices()[serviceName]; ok && svc.GetHost() == project.EvalHost { return wiringPresent, nil } props, err := structpb.NewStruct(map[string]any{ - "$ref": "./" + filepath.ToSlash(evalPath), + "$ref": "./" + filepath.ToSlash(configPath), }) if err != nil { return "", fmt.Errorf("building the eval service entry: %w", err) @@ -264,7 +511,7 @@ func ensureRootEvalService(ctx context.Context, evalName, target, evalPath strin _, err = azdClient.Project().AddService(ctx, &azdext.AddServiceRequest{ Service: &azdext.ServiceConfig{ - Name: evalName, + Name: serviceName, Host: project.EvalHost, Uses: evalServiceUses(resp.GetProject(), target), AdditionalProperties: props, @@ -281,8 +528,11 @@ func ensureRootEvalService(ctx context.Context, evalName, target, evalPath strin // It is conditional for the same reason the agents extension makes it // conditional: naming a service the project does not declare is a broken // reference, and an eval config can perfectly well sit in a repo that reaches -// an existing Foundry project by endpoint and an agent that is deployed -// elsewhere. +// an existing Foundry project by endpoint and an agent deployed elsewhere. +// +// Catalog entries need no ordering of their own — datasets, evaluators and +// evals are reconciled in a fixed order inside one deploy, forced by the +// contract rather than chosen. func evalServiceUses(proj *azdext.ProjectConfig, target string) []string { var uses []string for name, svc := range proj.GetServices() { @@ -297,217 +547,6 @@ func evalServiceUses(proj *azdext.ProjectConfig, target string) []string { return uses } -// scaffold is what `init` writes: one eval body and the generation settings -// that fill in the artifacts it references. -type scaffold struct { - eval *project.EvalConfig - generate *project.GenerateConfig - datasetName string - rubricName string -} - -// evaluatorNames lists the evaluators the eval will run, in declaration order. -func (s scaffold) evaluatorNames() []string { - names := make([]string, 0, len(s.eval.Evaluators)) - for _, ref := range s.eval.Evaluators { - names = append(names, ref.Name) - } - return names -} - -// evaluatorSummary is the one-line form `init` reports, marking the evaluator -// that still has to be generated. -func (s scaffold) evaluatorSummary() string { - parts := make([]string, 0, len(s.eval.Evaluators)) - for _, ref := range s.eval.Evaluators { - if ref.Name == s.rubricName && s.rubricCount() > 0 { - parts = append(parts, ref.Name+" (rubric)") - continue - } - parts = append(parts, ref.Name) - } - return strings.Join(parts, ", ") -} - -// rubricCount is the number of evaluators `init` expects to be generated. -func (s scaffold) rubricCount() int { - if s.generate == nil { - return 0 - } - return len(s.generate.Evaluator) -} - -// nextSteps are the commands to run after `init`, and only the ones that have -// something to do. -// -// A caller who supplied both a dataset and their evaluators has nothing left to -// generate, and pointing them at a generation command would submit a billed job -// for an artifact they already have. With everything in place the next step is -// to deploy it. -func (s scaffold) nextSteps() []string { - var steps []string - if s.generate != nil && len(s.generate.Dataset) > 0 { - steps = append(steps, "azd ai eval dataset generate "+s.datasetName) - } - if s.rubricCount() > 0 { - steps = append(steps, "azd ai eval evaluator generate "+s.rubricName) - } - if len(steps) == 0 { - steps = append(steps, "azd up", "azd ai eval run start") - } - return steps -} - -// relativeToConfig rewrites a path given relative to the working directory so -// it resolves from the directory holding the eval config. -// -// `--dataset ./tests/golden.jsonl` means "relative to where I am", but -// `source:` is resolved relative to the config file, so writing the path -// through unchanged sends the deploy looking inside evals/. An absolute path is -// left alone, and forward slashes are kept so the config reads the same on -// every platform. -func relativeToConfig(path, outputDir string) string { - if filepath.IsAbs(path) { - return path - } - - absPath, err := filepath.Abs(path) - if err != nil { - return path - } - absOut, err := filepath.Abs(outputDir) - if err != nil { - return path - } - - rel, err := filepath.Rel(absOut, absPath) - if err != nil { - return path - } - - rel = filepath.ToSlash(rel) - if !strings.HasPrefix(rel, ".") { - rel = "./" + rel - } - return rel -} - -// planScaffold builds both files `init` writes. -// -// The default evaluator set is a built-in plus a generated rubric: the built-in -// alone would be generic, and the rubric is what makes the baseline about this -// agent. Passing --evaluator replaces both, which is how a caller opts out of -// rubric generation. -func planScaffold( - evalName, target, rubricName, dataset string, - evaluators []string, - genModel string, - outputDir string, -) scaffold { - cfg := &project.EvalConfig{ - Description: fmt.Sprintf("Basic quality evaluation for %s", target), - } - - datasetName := evalName - datasetSource := "" - generateDataset := true - if dataset != "" { - if looksLikeLocalDataset(dataset) { - // --dataset is given relative to where the user is standing, but - // source: is resolved relative to the config, so the path has to be - // rebased or the deploy looks for it inside evals/. - datasetSource = relativeToConfig(dataset, outputDir) - datasetName = strings.TrimSuffix(filepath.Base(dataset), filepath.Ext(dataset)) - } else { - // A bare name references an already-registered dataset. - datasetName = dataset - } - generateDataset = false - } else { - datasetSource = fmt.Sprintf("./%s/%s.jsonl", project.DefaultDatasetsDir, datasetName) - } - cfg.Dataset = &project.DatasetDecl{ - Name: datasetName, - Source: datasetSource, - } - - // Every evaluator carries the judge deployment, because that is where the - // service reads it from: built-ins declare `deployment_name` as required, - // so an eval that leaves it off is rejected before it runs. - initParams := map[string]any{} - if genModel != "" { - initParams["deployment_name"] = genModel - } - withModel := func(ref evalcore.EvaluatorRef) evalcore.EvaluatorRef { - if len(initParams) == 0 { - return ref - } - params := make(map[string]any, len(initParams)) - for k, v := range initParams { - params[k] = v - } - ref.InitializationParameters = params - return ref - } - - refs := evalcore.EvaluatorList{} - generateRubric := false - if len(evaluators) == 0 { - refs = append(refs, - withModel(evalcore.EvaluatorRef{Name: evalcore.BuiltinPrefix + "task_adherence"}), - withModel(evalcore.EvaluatorRef{ - Name: rubricName, - Source: fmt.Sprintf("./%s/%s.json", project.DefaultEvaluatorsDir, rubricName), - }), - ) - generateRubric = true - } else { - for _, e := range evaluators { - ref := evalcore.EvaluatorRef{Name: e} - if !ref.IsBuiltin() { - ref.Source = fmt.Sprintf("./%s/%s.json", project.DefaultEvaluatorsDir, e) - } - refs = append(refs, withModel(ref)) - } - } - cfg.Evaluators = refs - - cfg.Target = &project.Target{ - Type: project.TargetTypeAgent, - Name: target, - } - cfg.Options = &project.Options{ - MaxSamples: project.DefaultSampleSize, - EvaluationLevel: project.EvaluationLevelTurn, - } - - gen := &project.GenerateConfig{GenerationModel: genModel} - if generateDataset { - gen.Dataset = map[string]project.DatasetGenSpec{ - datasetName: { - SampleSize: project.DefaultSampleSize, - OutputDir: "./" + project.DefaultDatasetsDir, - DeriveFrom: target, - }, - } - } - if generateRubric { - gen.Evaluator = map[string]project.EvaluatorGenSpec{ - rubricName: { - OutputDir: "./" + project.DefaultEvaluatorsDir, - DeriveFrom: target, - }, - } - } - - return scaffold{ - eval: cfg, - generate: gen, - datasetName: datasetName, - rubricName: rubricName, - } -} - // looksLikeLocalDataset distinguishes a path from a registered dataset name. func looksLikeLocalDataset(v string) bool { if strings.ContainsAny(v, `/\`) { diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_test.go index b391dd908c0..c0f2549fbdf 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_test.go @@ -13,56 +13,109 @@ import ( "github.com/stretchr/testify/require" ) -// The scaffold must load and validate cleanly, otherwise `azd up` fails on a +// scaffoldFor runs planScaffold against a fresh configuration, which is what +// `init` does on a project that has never been initialized. +func scaffoldFor(t *testing.T, in scaffoldInput) (scaffold, *project.EvalConfig) { + t.Helper() + if in.cfg == nil { + in.cfg = &project.EvalConfig{} + } + if in.evalDir == "" { + in.evalDir = project.DefaultEvalDir + } + if in.rubricName == "" { + in.rubricName = in.target + "-quality" + } + return planScaffold(in), in.cfg +} + +// The scaffold must round-trip and validate, otherwise `azd up` fails on a // config the tool itself produced. func TestScaffold_RoundTripsAndValidates(t *testing.T) { dir := t.TempDir() - evalPath := filepath.Join(dir, "support-agent-smoke.yaml") - - plan := planScaffold("support-agent-smoke", "support-agent", "support-agent-quality", - "", nil, "gpt-4.1-nano", project.DefaultEvalDir) - require.NoError(t, writeYAML(evalPath, plan.eval)) + _, cfg := scaffoldFor(t, scaffoldInput{ + evalName: "support-agent-smoke", + target: "support-agent", + judgeModel: "gpt-4.1-nano", + evalDir: dir, + }) - loaded, err := project.LoadEvalConfig(evalPath) + require.NoError(t, project.SaveEvalConfig(dir, cfg)) + loaded, err := project.OpenEvalConfig(dir) require.NoError(t, err) require.NoError(t, loaded.Validate(), "the generated scaffold must be valid") - require.Equal(t, project.TargetTypeAgent, loaded.Target.Type) - require.Equal(t, "support-agent", loaded.Target.Name) - require.Equal(t, project.DefaultSampleSize, loaded.Options.MaxSamples) + eval, err := loaded.Eval("support-agent-smoke") + require.NoError(t, err) + require.Equal(t, project.TargetTypeAgent, eval.Target.Type) + require.Equal(t, "support-agent", eval.Target.Name) + require.Equal(t, project.EvaluationLevelTurn, eval.EvaluationLevel) +} - // The eval takes its name from the file, which is the azure.yaml service key. - require.Equal(t, "support-agent-smoke", loaded.Eval("support-agent-smoke").Name) +// Re-running init appends rather than replacing, so one file ends up holding +// every eval for the target. +func TestScaffold_AppendsToAnExistingConfiguration(t *testing.T) { + dir := t.TempDir() + _, cfg := scaffoldFor(t, scaffoldInput{ + evalName: "first", target: "support-agent", judgeModel: "m", evalDir: dir, + }) + _, cfg = scaffoldFor(t, scaffoldInput{ + evalName: "second", target: "support-agent", judgeModel: "m", evalDir: dir, cfg: cfg, + }) + + require.Equal(t, []string{"first", "second"}, cfg.EvalNames()) + require.NoError(t, project.SaveEvalConfig(dir, cfg)) + loaded, err := project.OpenEvalConfig(dir) + require.NoError(t, err) + require.NoError(t, loaded.Validate()) +} + +// A trace-backed eval invokes nothing, so agent_name filters instead of +// targeting, and the service default caps how many traces are read. +func TestScaffold_TraceSourceHasNoTarget(t *testing.T) { + plan, _ := scaffoldFor(t, scaffoldInput{ + evalName: "support-agent-trace-eval", + target: "support-agent", + source: initSourceTraces, + }) + + require.Nil(t, plan.eval.Target) + require.NotNil(t, plan.eval.Source) + require.Equal(t, project.SourceTypeTraces, plan.eval.Source.Type) + require.Equal(t, "support-agent", plan.eval.Source.AgentName) } // The default set is a built-in plus a generated rubric: the built-in alone // would be generic, and the rubric is what makes the baseline about this agent. func TestScaffold_DefaultEvaluators(t *testing.T) { - plan := planScaffold("support-agent-smoke", "support-agent", "support-agent-quality", - "", nil, "gpt-5.6-luna", project.DefaultEvalDir) + plan, _ := scaffoldFor(t, scaffoldInput{ + evalName: "support-agent-smoke", target: "support-agent", judgeModel: "gpt-5.6-luna", + }) require.Equal(t, []string{"builtin.task_adherence", "support-agent-quality"}, plan.evaluatorNames()) - require.Contains(t, plan.evaluatorSummary(), "support-agent-quality (rubric)") - // Every evaluator carries the judge deployment, because built-ins declare - // deployment_name as required and an eval that leaves it off is rejected. + // Every evaluator carries the judge deployment, because the judging + // built-ins declare it and an eval that leaves it off is rejected. for _, ref := range plan.eval.Evaluators { - require.Equal(t, "gpt-5.6-luna", ref.InitializationParameters["deployment_name"], - "%s must name a judge deployment", ref.Name) + require.Equal(t, "gpt-5.6-luna", ref.InitializationParameters["model"], + "%s must name a judge deployment", ref.Evaluator) } } // Passing --evaluator replaces the defaults, which is how a caller opts out of // rubric generation. func TestScaffold_ExplicitEvaluatorsOptOutOfGeneration(t *testing.T) { - plan := planScaffold("smoke", "support-agent", "support-agent-quality", "", - []string{"builtin.task_adherence"}, "m", project.DefaultEvalDir) + plan, _ := scaffoldFor(t, scaffoldInput{ + evalName: "smoke", + target: "support-agent", + evaluators: []string{"builtin.task_adherence"}, + judgeModel: "m", + }) require.Equal(t, []string{"builtin.task_adherence"}, plan.evaluatorNames()) - require.Zero(t, plan.rubricCount(), "no rubric is generated when evaluators are given") - require.Empty(t, plan.generate.Evaluator) + require.False(t, plan.generateRubric, "no rubric is generated when evaluators are given") } // `init` closes by naming what to run next, and only what has something to do. @@ -70,105 +123,104 @@ func TestScaffold_ExplicitEvaluatorsOptOutOfGeneration(t *testing.T) { // would submit a billed job for something they already have. func TestScaffold_NextStepsOfferOnlyWhatIsScheduled(t *testing.T) { t.Run("nothing supplied", func(t *testing.T) { - plan := planScaffold("support-agent-smoke", "support-agent", "support-agent-quality", - "", nil, "m", project.DefaultEvalDir) + plan, _ := scaffoldFor(t, scaffoldInput{ + evalName: "support-agent-smoke", target: "support-agent", judgeModel: "m", + }) require.Equal(t, []string{ - "azd ai eval dataset generate support-agent-smoke", + "azd ai dataset generate support-agent-smoke", "azd ai eval evaluator generate support-agent-quality", }, plan.nextSteps()) }) t.Run("dataset supplied", func(t *testing.T) { - plan := planScaffold("smoke", "support-agent", "support-agent-quality", - "prod-golden", nil, "m", project.DefaultEvalDir) + plan, _ := scaffoldFor(t, scaffoldInput{ + evalName: "smoke", target: "support-agent", dataset: "prod-golden", judgeModel: "m", + }) require.Equal(t, []string{"azd ai eval evaluator generate support-agent-quality"}, plan.nextSteps()) }) t.Run("everything supplied", func(t *testing.T) { - plan := planScaffold("smoke", "support-agent", "support-agent-quality", - "prod-golden", []string{"builtin.task_adherence"}, "m", project.DefaultEvalDir) + plan, _ := scaffoldFor(t, scaffoldInput{ + evalName: "smoke", + target: "support-agent", + dataset: "prod-golden", + evaluators: []string{"builtin.task_adherence"}, + judgeModel: "m", + }) require.Equal(t, []string{"azd up", "azd ai eval run start"}, plan.nextSteps(), "with every artifact in place the next step is to deploy") }) } -func TestGenerateScaffold_RoundTripsAndValidates(t *testing.T) { +// Built-ins are referenced but never declared, so the scaffold must not give +// one a catalog entry to publish. +func TestScaffold_BuiltinEvaluatorsGetNoCatalogEntry(t *testing.T) { dir := t.TempDir() - genPath := filepath.Join(dir, "generate.yaml") - - plan := planScaffold("support-agent-smoke", "support-agent", "support-agent-quality", - "", nil, "gpt-4.1-nano", project.DefaultEvalDir) - require.NoError(t, writeYAML(genPath, plan.generate)) - - loaded, err := project.LoadGenerateConfig(genPath) - require.NoError(t, err) - require.Equal(t, "gpt-4.1-nano", loaded.GenerationModel) - - ds, ok := loaded.DatasetSpec("support-agent-smoke") - require.True(t, ok, "the generation spec is keyed by artifact name") - require.Equal(t, project.DefaultSampleSize, ds.SampleSize) - require.Equal(t, "support-agent", ds.DeriveFrom) - - ev, ok := loaded.EvaluatorSpec("support-agent-quality") - require.True(t, ok) - require.Equal(t, "./"+project.DefaultEvaluatorsDir, ev.OutputDir) -} - -// Built-ins are referenced but never published, so the scaffold must not give -// one a local source to upload. -func TestScaffold_BuiltinEvaluatorsHaveNoSource(t *testing.T) { - plan := planScaffold("smoke", "support-agent", "unused", "", - []string{"builtin.task_adherence", "my-custom"}, "", project.DefaultEvalDir) - cfg := plan.eval + plan, cfg := scaffoldFor(t, scaffoldInput{ + evalName: "smoke", + target: "support-agent", + evaluators: []string{"builtin.task_adherence", "my-custom"}, + judgeModel: "m", + evalDir: dir, + }) - require.Len(t, cfg.Evaluators, 2) - require.True(t, cfg.Evaluators[0].IsBuiltin()) - require.Empty(t, cfg.Evaluators[0].Source) - require.False(t, cfg.Evaluators[1].IsBuiltin()) - require.NotEmpty(t, cfg.Evaluators[1].Source) + require.Len(t, plan.eval.Evaluators, 2) + require.True(t, plan.eval.Evaluators[0].IsBuiltin()) + require.False(t, plan.eval.Evaluators[1].IsBuiltin()) + require.Len(t, cfg.Evaluators, 1, "only the custom evaluator is declared") + require.Equal(t, "my-custom", cfg.Evaluators[0].Name) require.Len(t, cfg.CustomEvaluators(), 1, "only the custom evaluator is this config's to publish") - path := filepath.Join(t.TempDir(), "smoke.yaml") - require.NoError(t, writeYAML(path, cfg)) - loaded, err := project.LoadEvalConfig(path) + require.NoError(t, project.SaveEvalConfig(dir, cfg)) + loaded, err := project.OpenEvalConfig(dir) require.NoError(t, err) require.NoError(t, loaded.Validate()) } // A bare name means an already-registered dataset; a path means a local file. // Either way the dataset was supplied, so nothing is scheduled to generate it — -// only a missing --dataset produces a generation entry. +// only a missing --dataset produces a generation step. func TestScaffold_DatasetReferenceForms(t *testing.T) { t.Run("local path becomes a source", func(t *testing.T) { // --dataset is relative to the working directory, but source: is // resolved relative to the eval config, so it has to be rebased. - plan := planScaffold("smoke", "a", "r", "./tests/golden.jsonl", nil, "", "evals") - require.Equal(t, "../tests/golden.jsonl", plan.eval.Dataset.Source, + plan, cfg := scaffoldFor(t, scaffoldInput{ + evalName: "smoke", target: "a", dataset: "./tests/golden.jsonl", evalDir: "evals", + }) + decl, ok := cfg.DatasetDeclaration("golden") + require.True(t, ok) + require.Equal(t, "../tests/golden.jsonl", decl.Source, "a dataset outside the eval dir must be reached with ..") - require.Equal(t, "golden", plan.eval.Dataset.Name) - require.Empty(t, plan.generate.Dataset, + require.Equal(t, "golden", plan.eval.Dataset) + require.False(t, plan.generateDataset, "a supplied dataset must not be scheduled for generation") }) t.Run("bare name references a registered dataset", func(t *testing.T) { - plan := planScaffold("smoke", "a", "r", "prod-sample", nil, "", project.DefaultEvalDir) - require.Equal(t, "prod-sample", plan.eval.Dataset.Name) - require.Empty(t, plan.eval.Dataset.Source, - "a registered dataset must not get a local source") - require.Empty(t, plan.generate.Dataset) + plan, cfg := scaffoldFor(t, scaffoldInput{ + evalName: "smoke", target: "a", dataset: "prod-sample", + }) + decl, ok := cfg.DatasetDeclaration("prod-sample") + require.True(t, ok) + require.Empty(t, decl.Source, "a registered dataset must not get a local source") + require.Equal(t, "prod-sample", plan.eval.Dataset) + require.False(t, plan.generateDataset) }) - t.Run("no dataset flag scaffolds a local path and a generation entry", func(t *testing.T) { - plan := planScaffold("support-agent-smoke", "support-agent", "r", "", - nil, "", project.DefaultEvalDir) - require.Equal(t, "support-agent-smoke", plan.eval.Dataset.Name, + t.Run("no dataset flag scaffolds a local path and a generation step", func(t *testing.T) { + plan, cfg := scaffoldFor(t, scaffoldInput{ + evalName: "support-agent-smoke", target: "support-agent", + }) + require.Equal(t, "support-agent-smoke", plan.eval.Dataset, "the dataset is named after the eval") - require.Contains(t, plan.eval.Dataset.Source, "support-agent-smoke.jsonl") - require.Contains(t, plan.generate.Dataset, "support-agent-smoke") + decl, ok := cfg.DatasetDeclaration("support-agent-smoke") + require.True(t, ok) + require.Contains(t, decl.Source, "support-agent-smoke.jsonl") + require.True(t, plan.generateDataset) }) } @@ -181,15 +233,15 @@ func TestLooksLikeLocalDataset(t *testing.T) { // Paths are used verbatim relative to the working directory; the doubling bug // in the agent-scoped command must not reappear. -func TestWriteYAML_UsesPathVerbatim(t *testing.T) { +func TestSaveEvalConfig_UsesPathVerbatim(t *testing.T) { dir := t.TempDir() - nested := filepath.Join(dir, "evals", "smoke.yaml") + nested := filepath.Join(dir, "evals") - require.NoError(t, writeYAML(nested, &project.EvalConfig{})) - _, err := os.Stat(nested) + require.NoError(t, project.SaveEvalConfig(nested, &project.EvalConfig{})) + _, err := os.Stat(project.EvalConfigPath(nested)) require.NoError(t, err, "the file must land exactly at the requested path") - doubled := filepath.Join(dir, "evals", "evals", "smoke.yaml") + doubled := filepath.Join(dir, "evals", "evals") _, err = os.Stat(doubled) require.Error(t, err, "the path must not be re-rooted under itself") } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/resolution_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/resolution_test.go index 9e20a857312..ae0a60152a3 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/resolution_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/resolution_test.go @@ -15,7 +15,7 @@ import ( // is silent. options.max_samples was parsed and dropped once already, which is // what these lock down. func TestResolveMaxSamples_Precedence(t *testing.T) { - withOptions := &project.Eval{Options: &project.Options{MaxSamples: 25}} + withOptions := &project.Eval{MaxSamples: 25} assert.Equal(t, 5, resolveMaxSamples(5, withOptions), "the flag wins over the config") assert.Equal(t, 25, resolveMaxSamples(0, withOptions), "the config is used when no flag is given") @@ -25,12 +25,12 @@ func TestResolveMaxSamples_Precedence(t *testing.T) { // Zero in config is absent, not a cap of zero: a cap of zero would send // nothing at all. - assert.Equal(t, 0, resolveMaxSamples(0, &project.Eval{Options: &project.Options{MaxSamples: 0}})) + assert.Equal(t, 0, resolveMaxSamples(0, &project.Eval{MaxSamples: 0})) } func TestResolveLevel_Precedence(t *testing.T) { withOptions := &project.Eval{ - Options: &project.Options{EvaluationLevel: project.EvaluationLevelConversation}, + EvaluationLevel: project.EvaluationLevelConversation, } assert.Equal(t, project.EvaluationLevelTurn, resolveLevel(project.EvaluationLevelTurn, withOptions), diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go index 9042c0f1486..1bb13de25bf 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go @@ -101,38 +101,23 @@ func buildRunCommand(use, short string) *cobra.Command { } defer ec.Close() - // --eval-id bypasses the config entirely. - var group *project.Eval - configPath := "" - if evalID == "" { - configPath, err = project.ResolveEvalConfigPath(project.DefaultEvalDir, groupName) - if err != nil { - return err - } - cfg, err := project.LoadEvalConfig(configPath) - if err != nil { - return err - } - if err := cfg.Validate(); err != nil { - return err - } - resolved := cfg.Eval(evalNameFromPath(configPath)) - group = &resolved - - if err := ec.checkDatasetRegistered(ctx, cfg, group, configPath); err != nil { - return err - } + // One flag takes a name or an id. A declared name also brings the + // declaration, which is what says where rows come from; a bare id + // has none, so the pairing comes from the eval's previous run. + ref, err := ec.resolveEvalRef(ctx, project.DefaultEvalDir, groupName) + if err != nil { + return err + } + evalID := ref.ID + group := ref.Eval + configPath := ref.ConfigPath - evalID, err = ec.resolveEvalIDFromConfig( - ctx, group, configPath, resolveLevel(level, group), - out, isJSON(cmd)) - if err != nil { + if ref.Declared() { + if err := ec.checkDatasetRegistered(ctx, ref.Config, group, configPath); err != nil { return err } } - // With --eval-id there is no config to read, so the pairing of - // target and dataset comes from the group's previous run. var dataSource *eval_api.EvalRunDataSource switch { case group == nil: @@ -268,16 +253,11 @@ func (ec *evalContext) resolveEvalIDFromConfig( fmt.Fprintf(out, "Creating eval %q...\n", group.Name) } - // The level from the flag wins over the group's own options, so it has to - // reach the criteria that accept evaluation_level. + // The level from the flag wins over the eval's own declaration, so it has + // to reach the criteria that accept evaluation_level. effective := *group if level != "" { - opts := project.Options{} - if group.Options != nil { - opts = *group.Options - } - opts.EvaluationLevel = level - effective.Options = &opts + effective.EvaluationLevel = level } req, err := buildEvalRequest( @@ -305,12 +285,13 @@ func (ec *evalContext) resolveEvalIDFromConfig( // The per-name entry is what the extension writes. EVAL_ID is also the // documented way to point a config at an eval that already exists, created in // the portal or by another tool, so it stays readable — but only when the -// project declares a single eval. With more than one there is no way to tell -// which eval a shared entry refers to, and reading it anyway is what let a +// configuration declares a single eval. With more than one there is no way to +// tell which eval a shared entry refers to, and reading it anyway is what let a // second eval adopt the first one's id. func evalIDKeys(name, evalDir string) []string { keys := []string{idKey("eval", name)} - if names, err := project.EvalNamesIn(evalDir); err == nil && len(names) == 1 { + if cfg, err := project.OpenEvalConfig(evalDir); err == nil && + cfg != nil && len(cfg.Evals) == 1 { keys = append(keys, envKeyEvalID) } return keys @@ -338,8 +319,8 @@ func (ec *evalContext) checkDatasetRegistered( return nil } - decl := cfg.Dataset - if decl == nil { + decl, ok := cfg.DatasetDeclaration(group.Dataset) + if !ok { return nil } @@ -513,11 +494,11 @@ func datasetColumnsFromPath(localPath string) map[string]bool { // file, returning empty when the dataset is registered rather than local. func localDatasetPath(configPath string, group *project.Eval) string { cfg, err := project.LoadEvalConfig(configPath) - if err != nil { + if err != nil || group == nil { return "" } - decl := cfg.Dataset - if decl == nil || decl.Source == "" { + decl, ok := cfg.DatasetDeclaration(group.Dataset) + if !ok || decl.Source == "" { return "" } if filepath.IsAbs(decl.Source) { @@ -526,13 +507,6 @@ func localDatasetPath(configPath string, group *project.Eval) string { return filepath.Join(filepath.Dir(configPath), decl.Source) } -// evalNameFromPath is the eval's name: one config file is one eval, and the -// file is named after it, matching the azure.yaml service key that $refs it. -func evalNameFromPath(configPath string) string { - base := filepath.Base(configPath) - return strings.TrimSuffix(base, filepath.Ext(base)) -} - // readJSONL reads newline-delimited JSON, optionally truncating to limit rows. func readJSONL(path string, limit int) ([]map[string]any, error) { f, err := os.Open(path) @@ -582,29 +556,29 @@ func scanJSONL(r io.Reader, limit int) ([]map[string]any, error) { return items, nil } -// resolveLevel prefers the flag, then the group's options. +// resolveLevel prefers the flag, then the eval's own declaration. func resolveLevel(flag string, group *project.Eval) string { if flag != "" { return flag } - if group != nil && group.Options != nil { - return group.Options.EvaluationLevel + if group != nil { + return group.EvaluationLevel } return "" } -// resolveMaxSamples prefers the flag, then the group's options, matching how -// the evaluation level resolves. +// resolveMaxSamples prefers the flag, then the eval's own declaration, matching +// how the evaluation level resolves. // -// Without this, options.max_samples parsed and did nothing: a group that caps -// its sample count in config would send the whole dataset, and only a flag on -// every invocation would honour the cap. +// Without this, max_samples parsed and did nothing: an eval that caps its +// sample count in config would send the whole dataset, and only a flag on every +// invocation would honour the cap. func resolveMaxSamples(flag int, group *project.Eval) int { if flag > 0 { return flag } - if group != nil && group.Options != nil && group.Options.MaxSamples > 0 { - return group.Options.MaxSamples + if group != nil && group.MaxSamples > 0 { + return group.MaxSamples } return 0 } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/surface_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/surface_test.go index 3e3f9e035be..0bfe4b2514f 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/surface_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/surface_test.go @@ -105,7 +105,6 @@ func TestFlagVocabularyIsShared(t *testing.T) { "--out-dir": "--output-dir", "--file": "--from-file", "--rubric": "--from-file", - "--judge-model": "--generation-model, declared per evaluator instead", "--from-traces": "deferred to M2", "--response-id": "deferred to M2", "--no-target": "deferred to M2", @@ -118,7 +117,6 @@ func TestFlagVocabularyIsShared(t *testing.T) { "--data-schema": "deferred to M2", "--metrics": "deferred to M2", "--trace-window": "deferred to M2", - "--max-traces": "deferred to M2", "--max-turns": "deferred to M2", } @@ -172,8 +170,8 @@ func TestInitFlagsMatchTheSpec(t *testing.T) { }) assert.ElementsMatch(t, []string{ - "--name", "--target", "--dataset", "--evaluator", - "--generation-model", "--output-dir", "--force", + "--name", "--target", "--source", "--dataset", "--evaluator", + "--judge-model", "--path", "--force", }, got, "init's flags are a table in the spec; change both together") } @@ -240,7 +238,7 @@ func TestSuggestedCommandsExist(t *testing.T) { return err } - for _, line := range strings.Split(string(body), "\n") { + for line := range strings.SplitSeq(string(body), "\n") { // Comments explain the surface; only what reaches a terminal has // to resolve. if strings.HasPrefix(strings.TrimSpace(line), "//") { diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/operations.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/operations.go index c2283fceb68..427f8c6af00 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/operations.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/operations.go @@ -112,7 +112,7 @@ func (c *DatasetClient) UploadNextVersion( } var err error - for attempt := 0; attempt < versionConflictAttempts; attempt++ { + for range versionConflictAttempts { var ds *Dataset ds, err = c.UploadNewVersion(ctx, name, currentVersion, localDir, apiVersion) if err == nil || !IsVersionConflict(err) { diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/models.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/models.go index 6441568db72..122828750de 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/models.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/models.go @@ -265,6 +265,7 @@ type TestingCriterion struct { Type string `json:"type"` Name string `json:"name"` EvaluatorName string `json:"evaluator_name"` + EvaluatorVersion string `json:"evaluator_version,omitempty"` InitializationParameters map[string]any `json:"initialization_parameters,omitempty"` DataMapping map[string]string `json:"data_mapping,omitempty"` } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/evaluator.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/evaluator.go index df6aa791b7d..75bc7f8a251 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/evaluator.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/evaluator.go @@ -16,46 +16,67 @@ import ( // stripped before the name is sent as testing_criteria[].evaluator_name. const BuiltinPrefix = "builtin." -// EvaluatorRef references an evaluator from an eval. It accepts either a -// bare string or a mapping carrying the rest of the declaration: +// EvaluatorRef is one entry in an eval's `evaluators:` list. Every entry is a +// map keyed `evaluator:`; what to publish lives on the catalog entry instead, +// so a reference only names an evaluator and says how to run it: // // evaluators: -// - builtin.task_adherence -// - name: support-quality -// source: ./evaluators/support-quality.json +// - evaluator: builtin.task_adherence // initialization_parameters: -// deployment_name: gpt-5.6-luna -// -// A built-in needs nothing but its name. One with a Source is the project's -// own, and is published before the eval that references it is created. +// model: gpt-5.6-luna +// threshold: 3 +// - evaluator: support-agent-quality +// name: quality_strict +// version: "2" +// data_mapping: +// query: "{{item.customer_message}}" type EvaluatorRef struct { - Name string `yaml:"name" json:"name"` + // Evaluator is the evaluator to run: a catalog name or builtin.. + Evaluator string `yaml:"evaluator" json:"evaluator"` + // Name labels the criterion in results. Empty means the evaluator's name. + Name string `yaml:"name,omitempty" json:"name,omitempty"` + // Version pins this reference. Pinning belongs to one eval's reference + // rather than to the asset, matching evaluator_version on the criterion. Version string `yaml:"version,omitempty" json:"version,omitempty"` - // Source names a local rubric JSON file. Empty means the evaluator is - // already registered, or is a built-in. - Source string `yaml:"source,omitempty" json:"source,omitempty"` - // InitializationParameters are passed through to the testing criterion, - // which is where a judge deployment is named. + // InitializationParameters carry the judge deployment and a built-in's + // numeric threshold. They are bound against the evaluator's published + // contract rather than forwarded as written. InitializationParameters map[string]any `yaml:"initialization_parameters,omitempty" json:"initialization_parameters,omitempty"` - // Threshold maps to testing_criteria[].initialization_parameters.threshold. - Threshold *float64 `yaml:"threshold,omitempty" json:"threshold,omitempty"` + // DataMapping binds evaluator inputs to dataset columns, and is written + // only when the inference from declared inputs and columns gets it wrong. + DataMapping map[string]string `yaml:"data_mapping,omitempty" json:"data_mapping,omitempty"` } // IsBuiltin reports whether the reference names a platform evaluator, which -// needs no declaration and is never uploaded. +// needs no catalog entry and is never uploaded. func (e EvaluatorRef) IsBuiltin() bool { - return strings.HasPrefix(e.Name, BuiltinPrefix) + return strings.HasPrefix(e.Evaluator, BuiltinPrefix) } // APIName is the name the service expects, with the builtin prefix removed. func (e EvaluatorRef) APIName() string { - return strings.TrimPrefix(e.Name, BuiltinPrefix) + return strings.TrimPrefix(e.Evaluator, BuiltinPrefix) +} + +// CriterionName labels this criterion in results. +func (e EvaluatorRef) CriterionName() string { + if e.Name != "" { + return e.Name + } + return e.APIName() } -// EvaluatorList is a sequence of EvaluatorRef supporting mixed string and -// mapping entries. +// EvaluatorList is a sequence of EvaluatorRef. +// +// A bare string is refused rather than accepted quietly. Every other collection +// in the file is a list of named maps, and a bare string would have to mean the +// evaluator while reading as the criterion's own name — a different key this +// same entry also carries. type EvaluatorList []EvaluatorRef +const bareEvaluatorRemedy = "an evaluator entry is a mapping, not a bare string: " + + "write `- evaluator: %s`" + func (el *EvaluatorList) UnmarshalYAML(value *yaml.Node) error { if value.Kind != yaml.SequenceNode { return fmt.Errorf("evaluators must be a sequence, got %v", value.Kind) @@ -69,18 +90,18 @@ func (el *EvaluatorList) UnmarshalYAML(value *yaml.Node) error { if err := node.Decode(&name); err != nil { return fmt.Errorf("decoding evaluator name: %w", err) } - result = append(result, EvaluatorRef{Name: name}) + return fmt.Errorf(bareEvaluatorRemedy, name) case yaml.MappingNode: var ref EvaluatorRef if err := node.Decode(&ref); err != nil { return fmt.Errorf("decoding evaluator: %w", err) } - if ref.Name == "" { - return fmt.Errorf("evaluator entry is missing 'name'") + if ref.Evaluator == "" { + return fmt.Errorf("evaluator entry is missing 'evaluator'") } result = append(result, ref) default: - return fmt.Errorf("evaluator entry must be a string or a mapping, got %v", node.Kind) + return fmt.Errorf("evaluator entry must be a mapping, got %v", node.Kind) } } @@ -88,26 +109,12 @@ func (el *EvaluatorList) UnmarshalYAML(value *yaml.Node) error { return nil } -// MarshalYAML emits the compact string form when an entry carries nothing but a -// name, so round-tripping a hand-written config does not rewrite it. -func (el EvaluatorList) MarshalYAML() (any, error) { - out := make([]any, 0, len(el)) - for _, ref := range el { - if ref.isBareName() { - out = append(out, ref.Name) - continue - } - out = append(out, ref) - } - return out, nil -} - -// UnmarshalJSON accepts the same mixed string-or-mapping form as the YAML -// decoder. +// UnmarshalJSON accepts the same mapping-only form as the YAML decoder. // // This matters for the service-target provider: azd hands the service entry to -// the extension as JSON, so a config written as `- builtin.task_adherence` -// arrives as a bare string and would otherwise fail to decode. +// the extension as JSON, so a config written the old way arrives here as a bare +// string and has to be refused with the same remedy rather than with a +// decoder's own type error. func (el *EvaluatorList) UnmarshalJSON(data []byte) error { var entries []json.RawMessage if err := json.Unmarshal(data, &entries); err != nil { @@ -122,16 +129,15 @@ func (el *EvaluatorList) UnmarshalJSON(data []byte) error { if err := json.Unmarshal(trimmed, &name); err != nil { return fmt.Errorf("decoding evaluator name: %w", err) } - result = append(result, EvaluatorRef{Name: name}) - continue + return fmt.Errorf(bareEvaluatorRemedy, name) } var ref EvaluatorRef if err := json.Unmarshal(trimmed, &ref); err != nil { return fmt.Errorf("decoding evaluator: %w", err) } - if ref.Name == "" { - return fmt.Errorf("evaluator entry is missing 'name'") + if ref.Evaluator == "" { + return fmt.Errorf("evaluator entry is missing 'evaluator'") } result = append(result, ref) } @@ -140,30 +146,19 @@ func (el *EvaluatorList) UnmarshalJSON(data []byte) error { return nil } -// MarshalJSON mirrors MarshalYAML's compact form. +// MarshalJSON is the default list encoding, defined so a compact form cannot +// creep back in through the encoder. // -// Everything the reference carries has to survive the round trip, including -// the source and the initialization parameters: the eval fingerprint is taken -// over this encoding, so a field dropped here is a change the reconciler -// cannot see. +// Everything the reference carries has to survive the round trip: the eval +// fingerprint is taken over this encoding, so a field dropped here is a change +// the reconciler cannot see. func (el EvaluatorList) MarshalJSON() ([]byte, error) { // Aliased so the element encoder does not recurse through this method. type ref = EvaluatorRef out := make([]any, 0, len(el)) for _, r := range el { - if r.isBareName() { - out = append(out, r.Name) - continue - } out = append(out, ref(r)) } return json.Marshal(out) } - -// isBareName reports whether the reference carries nothing but its name, in -// which case both encoders emit the compact string form. -func (e EvaluatorRef) isBareName() bool { - return e.Threshold == nil && e.Version == "" && e.Source == "" && - len(e.InitializationParameters) == 0 -} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/evaluator_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/evaluator_test.go index 5408776bec7..b185414e8ed 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/evaluator_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/evaluator_test.go @@ -11,28 +11,34 @@ import ( "go.yaml.in/yaml/v3" ) -// The service-target provider receives the config as JSON, not YAML, so the -// mixed string-or-mapping form has to decode through both paths. Supporting -// only YAML made `azd deploy` fail on a config the CLI itself writes. -func TestEvaluatorListDecodesMixedEntriesFromJSON(t *testing.T) { +// The service-target provider receives the config as JSON, not YAML, so both +// paths have to decode the mapping form identically. Supporting only YAML made +// `azd deploy` fail on a config the CLI itself writes. +func TestEvaluatorListDecodesEntriesFromJSON(t *testing.T) { const payload = `[ - "builtin.task_adherence", - {"name": "support-quality", "threshold": 4.0}, - {"name": "pinned", "version": "3"} + {"evaluator": "builtin.task_adherence"}, + {"evaluator": "support-quality", "name": "quality_strict", + "initialization_parameters": {"model": "gpt-5.6-luna", "threshold": 4}}, + {"evaluator": "pinned", "version": "3"} ]` var list EvaluatorList require.NoError(t, json.Unmarshal([]byte(payload), &list)) require.Len(t, list, 3) - require.Equal(t, "builtin.task_adherence", list[0].Name) - require.Nil(t, list[0].Threshold) + require.Equal(t, "builtin.task_adherence", list[0].Evaluator) + require.True(t, list[0].IsBuiltin()) + require.Equal(t, "task_adherence", list[0].APIName()) + require.Equal(t, "task_adherence", list[0].CriterionName()) + require.Nil(t, list[0].InitializationParameters) - require.Equal(t, "support-quality", list[1].Name) - require.NotNil(t, list[1].Threshold) - require.InDelta(t, 4.0, *list[1].Threshold, 0.0001) + require.Equal(t, "support-quality", list[1].Evaluator) + require.Equal(t, "quality_strict", list[1].CriterionName()) + require.False(t, list[1].IsBuiltin()) + require.Equal(t, "gpt-5.6-luna", list[1].InitializationParameters["model"]) + require.EqualValues(t, 4, list[1].InitializationParameters["threshold"]) - require.Equal(t, "pinned", list[2].Name) + require.Equal(t, "pinned", list[2].Evaluator) require.Equal(t, "3", list[2].Version) } @@ -40,8 +46,13 @@ func TestEvaluatorListDecodesMixedEntriesFromJSON(t *testing.T) { // through the CLI and another through `azd up`. func TestEvaluatorListJSONMatchesYAML(t *testing.T) { const doc = ` -- builtin.task_adherence -- { name: support-quality, threshold: 4.0 } +- evaluator: builtin.task_adherence +- evaluator: support-quality + name: quality_strict + initialization_parameters: + model: gpt-5.6-luna + data_mapping: + query: "{{item.customer_message}}" ` var fromYAML EvaluatorList require.NoError(t, yaml.Unmarshal([]byte(doc), &fromYAML)) @@ -54,9 +65,56 @@ func TestEvaluatorListJSONMatchesYAML(t *testing.T) { require.Equal(t, fromYAML, fromJSON) } -func TestEvaluatorListRejectsEntryWithoutName(t *testing.T) { - var list EvaluatorList - err := json.Unmarshal([]byte(`[{"threshold": 4.0}]`), &list) - require.Error(t, err) - require.Contains(t, err.Error(), "name") +// A bare string is the old shorthand. It has to be refused with the remedy +// rather than a decoder type error, through both decoders, because the +// service-target provider only ever sees JSON. +func TestEvaluatorListRefusesBareString(t *testing.T) { + t.Run("yaml", func(t *testing.T) { + var list EvaluatorList + err := yaml.Unmarshal([]byte("- builtin.task_adherence\n"), &list) + require.Error(t, err) + require.Contains(t, err.Error(), "- evaluator: builtin.task_adherence") + }) + + t.Run("json", func(t *testing.T) { + var list EvaluatorList + err := json.Unmarshal([]byte(`["builtin.task_adherence"]`), &list) + require.Error(t, err) + require.Contains(t, err.Error(), "- evaluator: builtin.task_adherence") + }) +} + +func TestEvaluatorListRejectsEntryWithoutEvaluator(t *testing.T) { + t.Run("yaml", func(t *testing.T) { + var list EvaluatorList + err := yaml.Unmarshal([]byte("- name: quality_strict\n"), &list) + require.Error(t, err) + require.Contains(t, err.Error(), "evaluator") + }) + + t.Run("json", func(t *testing.T) { + var list EvaluatorList + err := json.Unmarshal([]byte(`[{"name": "quality_strict"}]`), &list) + require.Error(t, err) + require.Contains(t, err.Error(), "evaluator") + }) +} + +// The eval fingerprint is taken over this encoding, so a field the encoder +// drops is a change the reconciler cannot see. +func TestEvaluatorListMarshalKeepsEveryField(t *testing.T) { + list := EvaluatorList{{ + Evaluator: "support-quality", + Name: "quality_strict", + Version: "2", + InitializationParameters: map[string]any{"model": "gpt-5.6-luna"}, + DataMapping: map[string]string{"query": "{{item.customer_message}}"}, + }} + + encoded, err := json.Marshal(list) + require.NoError(t, err) + + var round EvaluatorList + require.NoError(t, json.Unmarshal(encoded, &round)) + require.Equal(t, list, round) } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/artifacts.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/artifacts.go new file mode 100644 index 00000000000..9f5a6320684 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/artifacts.go @@ -0,0 +1,99 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package project + +import ( + "fmt" + "path/filepath" + "strings" +) + +// Conventional artifact locations, relative to the eval directory. +const ( + DefaultDatasetsDir = "datasets" + DefaultEvaluatorsDir = "evaluators" +) + +// ArtifactRef is the name/source pair a generation run produces, so the +// command can tell the developer how to reference it. +type ArtifactRef struct { + Name string `json:"name"` + Source string `json:"source"` +} + +// Sample-count bounds enforced by the generation service. +const ( + MinSampleSize = 15 + MaxSampleSize = 1000 + DefaultSampleSize = 15 +) + +// Sources a dataset can be generated from. +const ( + GenerateFromTraces = "traces" + GenerateFromAgent = "agent" + GenerateFromPrompt = "prompt" + GenerateFromFile = "file" +) + +// GenerateSources is what --from accepts, in help order. +var GenerateSources = []string{ + GenerateFromTraces, GenerateFromAgent, GenerateFromPrompt, GenerateFromFile, +} + +// ValidateGenerateSource rejects a --from value the service has no path for. +func ValidateGenerateSource(from string) error { + switch from { + case "", GenerateFromTraces, GenerateFromAgent, GenerateFromPrompt, GenerateFromFile: + return nil + default: + return fmt.Errorf( + "--from %q is not a source; use one of %s", + from, strings.Join(GenerateSources, ", ")) + } +} + +// ValidateSampleSize rejects a row count the service would reject, before a +// generation job is submitted and billed. +func ValidateSampleSize(n int) error { + if n != 0 && (n < MinSampleSize || n > MaxSampleSize) { + return fmt.Errorf( + "sample size must be between %d and %d, got %d", + MinSampleSize, MaxSampleSize, n) + } + return nil +} + +// ArtifactPath resolves an output directory against baseDir. The value may be a +// directory, in which case the file name is derived from resourceName and ext, +// or an explicit file path, which is used as-is. +func ArtifactPath(baseDir, outputDir, resourceName, ext string) string { + if outputDir == "" { + return filepath.Join(baseDir, resourceName+ext) + } + candidate := outputDir + if !filepath.IsAbs(candidate) { + candidate = filepath.Join(baseDir, candidate) + } + if looksLikeFile(outputDir, ext) { + return candidate + } + return filepath.Join(candidate, resourceName+ext) +} + +// looksLikeFile treats a trailing recognized extension as an explicit file path. +func looksLikeFile(p, ext string) bool { + got := strings.ToLower(filepath.Ext(p)) + if got == "" { + return false + } + if got == strings.ToLower(ext) { + return true + } + switch got { + case ".json", ".jsonl", ".yaml", ".yml": + return true + } + return false +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go index ec58282de06..c9bea4324a0 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go @@ -7,31 +7,26 @@ package project import ( "fmt" - "os" - "path/filepath" - "sort" "strings" "azureaieval/internal/pkg/evalcore" - - "go.yaml.in/yaml/v3" ) -// EvalConfig is one eval — the body of a single `azure.ai.eval` service entry, -// kept in evals/.yaml and pulled in with $ref. +// EvalConfig is one evaluation configuration: the catalogs of reusable assets, +// and every eval defined over them. +// +// It is the body of a single `azure.ai.eval` service entry, pulled in with +// $ref. One file rather than one per eval, because the catalogs are shared: +// two evals over the same dataset should name it once. // -// The eval's name is the service key in azure.yaml and is not repeated here. -// One service per eval is what lets the azd dependency graph order an eval -// after the agent it evaluates. +// How it is stored lives in eval_config_store.go. type EvalConfig struct { - Description string `yaml:"description,omitempty" json:"description,omitempty"` - Dataset *DatasetDecl `yaml:"dataset,omitempty" json:"dataset,omitempty"` - Evaluators evalcore.EvaluatorList `yaml:"evaluators,omitempty" json:"evaluators,omitempty"` - Target *Target `yaml:"target,omitempty" json:"target,omitempty"` - Options *Options `yaml:"options,omitempty" json:"options,omitempty"` + Datasets []DatasetDecl `yaml:"datasets,omitempty" json:"datasets,omitempty"` + Evaluators []EvaluatorDecl `yaml:"evaluators,omitempty" json:"evaluators,omitempty"` + Evals []Eval `yaml:"evals,omitempty" json:"evals,omitempty"` } -// DatasetDecl declares a dataset. A local Source is uploaded on deploy; without +// DatasetDecl is a catalog entry. A local Source is uploaded on deploy; without // one the name must already resolve to a registered dataset. type DatasetDecl struct { Name string `yaml:"name" json:"name"` @@ -39,8 +34,8 @@ type DatasetDecl struct { Version string `yaml:"version,omitempty" json:"version,omitempty"` } -// EvaluatorDecl declares a custom evaluator. Built-ins are referenced directly -// from an eval and never declared here. +// EvaluatorDecl is a catalog entry for a custom evaluator. Built-ins are +// referenced straight from an eval and never declared here. // // Source names a `.json` file holding a rubric: a list of weighted scoring // dimensions. @@ -50,34 +45,51 @@ type EvaluatorDecl struct { Version string `yaml:"version,omitempty" json:"version,omitempty"` } -// Eval is a run definition: evaluators plus options, bound to a dataset. +// Eval is one evaluation defined over the catalogs. +// +// Dataset and Source are alternatives: rows come from a catalog dataset, or +// from a source such as production traces. Target is what gets invoked, and is +// a separate axis — an eval can read traces and invoke nothing. type Eval struct { - Name string `yaml:"name" json:"name"` - ID string `yaml:"id,omitempty" json:"id,omitempty"` - Description string `yaml:"description,omitempty" json:"description,omitempty"` - Dataset string `yaml:"dataset,omitempty" json:"dataset,omitempty"` - Evaluators evalcore.EvaluatorList `yaml:"evaluators,omitempty" json:"evaluators,omitempty"` - Target *Target `yaml:"target,omitempty" json:"target,omitempty"` - Options *Options `yaml:"options,omitempty" json:"options,omitempty"` + Name string `yaml:"name" json:"name"` + ID string `yaml:"id,omitempty" json:"id,omitempty"` + Description string `yaml:"description,omitempty" json:"description,omitempty"` + Dataset string `yaml:"dataset,omitempty" json:"dataset,omitempty"` + Source *SourceDecl `yaml:"source,omitempty" json:"source,omitempty"` + EvaluationLevel string `yaml:"evaluation_level,omitempty" json:"evaluation_level,omitempty"` + MaxSamples int `yaml:"max_samples,omitempty" json:"max_samples,omitempty"` + Evaluators evalcore.EvaluatorList `yaml:"evaluators,omitempty" json:"evaluators,omitempty"` + Target *Target `yaml:"target,omitempty" json:"target,omitempty"` } -// Target names what the run invokes. Only type "agent" is supported today. +// SourceDecl says where an eval's rows come from when they are not a dataset. +type SourceDecl struct { + Type string `yaml:"type" json:"type"` + LookbackHours int `yaml:"lookback_hours,omitempty" json:"lookback_hours,omitempty"` + MaxTraces int `yaml:"max_traces,omitempty" json:"max_traces,omitempty"` + AgentName string `yaml:"agent_name,omitempty" json:"agent_name,omitempty"` + ResponseIDs []string `yaml:"response_ids,omitempty" json:"response_ids,omitempty"` + MaxTurns int `yaml:"max_turns,omitempty" json:"max_turns,omitempty"` +} + +// Source types an eval can read rows from. +const ( + SourceTypeTraces = "traces" + SourceTypeResponses = "responses" +) + +// Target names what the run invokes. type Target struct { Type string `yaml:"type" json:"type"` Name string `yaml:"name" json:"name"` } -const TargetTypeAgent = "agent" - -// Options are run settings carried on the eval. -// -// There is deliberately no judge-model option. A judge deployment is a testing -// criterion's `initialization_parameters.deployment_name`, which differs per -// evaluator, so it is declared on the evaluator reference instead. -type Options struct { - MaxSamples int `yaml:"max_samples,omitempty" json:"max_samples,omitempty"` - EvaluationLevel string `yaml:"evaluation_level,omitempty" json:"evaluation_level,omitempty"` -} +// Target types the extension can invoke. Absent means nothing is invoked and +// the dataset already carries the answers. +const ( + TargetTypeAgent = "agent" + TargetTypeModel = "model" +) // Evaluation levels accepted by the service. The service default is turn. const ( @@ -85,92 +97,149 @@ const ( EvaluationLevelConversation = "conversation" ) -// LoadEvalConfig reads an eval body from disk. The path is used verbatim, -// relative to the process working directory — never re-rooted. -func LoadEvalConfig(path string) (*EvalConfig, error) { - data, err := os.ReadFile(path) - if err != nil { - return nil, fmt.Errorf("reading eval config %q: %w", path, err) +// EvalNames lists the declared evals in declaration order. +func (c *EvalConfig) EvalNames() []string { + names := make([]string, 0, len(c.Evals)) + for _, e := range c.Evals { + names = append(names, e.Name) } - - var cfg EvalConfig - if err := yaml.Unmarshal(data, &cfg); err != nil { - return nil, fmt.Errorf("parsing eval config %q: %w", path, err) - } - return &cfg, nil + return names } -// EvalNamesIn lists the evals declared under evalDir, in sorted order. +// Eval returns the named eval. // -// One file is one eval, named after it. The generation spec shares the -// directory and is not one, so it is excluded by name. -func EvalNamesIn(evalDir string) ([]string, error) { - entries, err := os.ReadDir(evalDir) - if err != nil { - return nil, err +// An empty name is only answered when the file declares exactly one, because +// guessing which eval a command meant is the kind of mistake that is noticed +// only after it has run. +func (c *EvalConfig) Eval(name string) (*Eval, error) { + if name == "" { + switch len(c.Evals) { + case 0: + return nil, fmt.Errorf("no evals are declared") + case 1: + return &c.Evals[0], nil + default: + return nil, fmt.Errorf( + "this configuration declares %d evals (%s); choose one with --eval", + len(c.Evals), strings.Join(c.EvalNames(), ", ")) + } } - var names []string - for _, e := range entries { - if e.IsDir() { - continue + for i := range c.Evals { + if c.Evals[i].Name == name { + return &c.Evals[i], nil } - ext := strings.ToLower(filepath.Ext(e.Name())) - if ext != ".yaml" && ext != ".yml" { - continue + } + return nil, fmt.Errorf( + "eval %q is not declared; this configuration has %s", + name, strings.Join(c.EvalNames(), ", ")) +} + +// HasEval reports whether the named eval is declared. Unlike Eval it never +// falls back to "the only one", so callers checking for a collision cannot +// match a differently named entry. +func (c *EvalConfig) HasEval(name string) bool { + for i := range c.Evals { + if c.Evals[i].Name == name { + return true } - name := strings.TrimSuffix(e.Name(), filepath.Ext(e.Name())) - if name == generateConfigBase { - continue + } + return false +} + +// RemoveEval drops the named eval, reporting whether it was there. +func (c *EvalConfig) RemoveEval(name string) bool { + for i := range c.Evals { + if c.Evals[i].Name == name { + c.Evals = append(c.Evals[:i], c.Evals[i+1:]...) + return true } - names = append(names, name) } - sort.Strings(names) - return names, nil + return false } -// generateConfigBase is the reserved file name in the evals directory. -const generateConfigBase = "generate" +// DatasetDeclaration returns the catalog entry an eval's `dataset:` names. +func (c *EvalConfig) DatasetDeclaration(name string) (*DatasetDecl, bool) { + for i := range c.Datasets { + if c.Datasets[i].Name == name { + return &c.Datasets[i], true + } + } + return nil, false +} -// ResolveEvalConfigPath finds the config file holding one eval's body. -// -// A named eval is evals/.yaml. With no name the directory must hold -// exactly one eval, and anything else names the candidates rather than -// picking one, because guessing which eval a command meant is the kind of -// mistake that is only noticed after it has run. -func ResolveEvalConfigPath(evalDir, evalName string) (string, error) { - if evalName != "" { - path := EvalConfigPath(evalDir, evalName) - if _, err := os.Stat(path); err != nil { - return "", fmt.Errorf("eval %q is not declared in %s", evalName, evalDir) +// EvaluatorDeclaration returns the catalog entry an evaluator reference names. +func (c *EvalConfig) EvaluatorDeclaration(name string) (*EvaluatorDecl, bool) { + for i := range c.Evaluators { + if c.Evaluators[i].Name == name { + return &c.Evaluators[i], true } - return path, nil } + return nil, false +} - names, err := EvalNamesIn(evalDir) - if err != nil { - return "", fmt.Errorf("reading %s: %w", evalDir, err) +// CustomEvaluators are the catalog entries this configuration owns — the ones +// carrying a local source, published before the evals that name them. +func (c *EvalConfig) CustomEvaluators() []EvaluatorDecl { + var owned []EvaluatorDecl + for _, decl := range c.Evaluators { + if decl.Source == "" { + continue + } + owned = append(owned, decl) } - switch len(names) { - case 0: - return "", fmt.Errorf("no evals are declared in %s", evalDir) - case 1: - return EvalConfigPath(evalDir, names[0]), nil - default: - return "", fmt.Errorf( - "%s declares %d evals (%s); choose one with --eval", - evalDir, len(names), strings.Join(names, ", ")) + return owned +} + +// LocalDatasets are the catalog entries carrying a file to upload. +func (c *EvalConfig) LocalDatasets() []DatasetDecl { + var owned []DatasetDecl + for _, decl := range c.Datasets { + if decl.Source == "" { + continue + } + owned = append(owned, decl) } + return owned } // Validate checks the invariants the provider relies on before it calls the // service, so failures surface as config errors rather than opaque 4xx. func (c *EvalConfig) Validate() error { - if c.Dataset != nil && c.Dataset.Name == "" { - return fmt.Errorf("dataset: 'name' is required") + if err := c.validateCatalogs(); err != nil { + return err + } + if len(c.Evals) == 0 { + return fmt.Errorf("at least one eval is required") + } + + seen := map[string]bool{} + for i, eval := range c.Evals { + if eval.Name == "" { + return fmt.Errorf("evals[%d]: 'name' is required", i) + } + if seen[eval.Name] { + return fmt.Errorf("evals[%d]: duplicate eval name %q", i, eval.Name) + } + seen[eval.Name] = true + + if err := c.validateEval(i, eval); err != nil { + return err + } } - if len(c.Evaluators) == 0 { - return fmt.Errorf("at least one evaluator is required") + return nil +} + +func (c *EvalConfig) validateCatalogs() error { + datasets := map[string]bool{} + for i, d := range c.Datasets { + if d.Name == "" { + return fmt.Errorf("datasets[%d]: 'name' is required", i) + } + if datasets[d.Name] { + return fmt.Errorf("datasets[%d]: duplicate dataset name %q", i, d.Name) + } + datasets[d.Name] = true } evaluators := map[string]bool{} @@ -183,13 +252,10 @@ func (c *EvalConfig) Validate() error { } evaluators[e.Name] = true - if e.IsBuiltin() { - if e.Source != "" { - return fmt.Errorf( - "evaluators[%d] (%s): a built-in evaluator has no source to publish", - i, e.Name) - } - continue + if strings.HasPrefix(e.Name, evalcore.BuiltinPrefix) { + return fmt.Errorf( + "evaluators[%d] (%s): a built-in needs no catalog entry; reference it "+ + "straight from an eval", i, e.Name) } // The service assigns an evaluator's version on publish, so a declared // one cannot be honoured alongside a source: the upload lands on @@ -203,54 +269,74 @@ func (c *EvalConfig) Validate() error { "on the project", i, e.Name) } } + return nil +} - if c.Target != nil && c.Target.Type != "" && c.Target.Type != TargetTypeAgent { +func (c *EvalConfig) validateEval(i int, eval Eval) error { + if eval.Dataset != "" && eval.Source != nil { return fmt.Errorf( - "target.type %q is not supported; use %q", c.Target.Type, TargetTypeAgent) + "evals[%d] (%s): `dataset` and `source` both say where rows come from; "+ + "declare one", i, eval.Name) + } + if eval.Dataset != "" { + if _, ok := c.DatasetDeclaration(eval.Dataset); !ok { + return fmt.Errorf( + "evals[%d] (%s): dataset %q is not in the datasets catalog", + i, eval.Name, eval.Dataset) + } } - if c.Options != nil { - switch c.Options.EvaluationLevel { - case "", EvaluationLevelTurn, EvaluationLevelConversation: + if eval.Source != nil { + switch eval.Source.Type { + case SourceTypeTraces, SourceTypeResponses: + case "": + return fmt.Errorf("evals[%d] (%s): source.type is required", i, eval.Name) default: return fmt.Errorf( - "options.evaluation_level %q is invalid; expected %q or %q", - c.Options.EvaluationLevel, EvaluationLevelTurn, EvaluationLevelConversation) + "evals[%d] (%s): source.type %q is not supported; use %q or %q", + i, eval.Name, eval.Source.Type, SourceTypeTraces, SourceTypeResponses) } } - return nil -} - -// Eval resolves the config into the eval the reconciler publishes, taking its -// name from the service entry that pulled the file in. -func (c *EvalConfig) Eval(name string) Eval { - resolved := Eval{ - Name: name, - Description: c.Description, - Evaluators: c.Evaluators, - Target: c.Target, - Options: c.Options, + if len(eval.Evaluators) == 0 { + return fmt.Errorf("evals[%d] (%s): at least one evaluator is required", i, eval.Name) } - if c.Dataset != nil { - resolved.Dataset = c.Dataset.Name - } - return resolved -} + criteria := map[string]bool{} + for j, ref := range eval.Evaluators { + if ref.Evaluator == "" { + return fmt.Errorf("evals[%d].evaluators[%d]: 'evaluator' is required", i, j) + } + // The criterion name is what identifies a result row, so two rows that + // cannot be told apart are refused here rather than in the results. + criterion := ref.CriterionName() + if criteria[criterion] { + return fmt.Errorf( + "evals[%d].evaluators[%d]: duplicate criterion %q; give one a `name`", + i, j, criterion) + } + criteria[criterion] = true -// CustomEvaluators are the evaluators this config owns — the referenced ones -// carrying a local source, which are published before the eval that names them. -// A built-in needs nothing, and one without a source is already registered. -func (c *EvalConfig) CustomEvaluators() []EvaluatorDecl { - var owned []EvaluatorDecl - for _, ref := range c.Evaluators { - if ref.IsBuiltin() || ref.Source == "" { + if ref.IsBuiltin() { continue } - owned = append(owned, EvaluatorDecl{ - Name: ref.Name, - Source: ref.Source, - Version: ref.Version, - }) + if _, ok := c.EvaluatorDeclaration(ref.Evaluator); !ok { + return fmt.Errorf( + "evals[%d].evaluators[%d]: evaluator %q is not in the evaluators catalog", + i, j, ref.Evaluator) + } } - return owned + + if eval.Target != nil && eval.Target.Type != "" && + eval.Target.Type != TargetTypeAgent && eval.Target.Type != TargetTypeModel { + return fmt.Errorf( + "evals[%d] (%s): target.type %q is not supported; use %q or %q", + i, eval.Name, eval.Target.Type, TargetTypeAgent, TargetTypeModel) + } + switch eval.EvaluationLevel { + case "", EvaluationLevelTurn, EvaluationLevelConversation: + default: + return fmt.Errorf( + "evals[%d] (%s): evaluation_level %q is invalid; expected %q or %q", + i, eval.Name, eval.EvaluationLevel, EvaluationLevelTurn, EvaluationLevelConversation) + } + return nil } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_store.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_store.go new file mode 100644 index 00000000000..2b2b587c4f2 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_store.go @@ -0,0 +1,81 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package project + +import ( + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" + + "go.yaml.in/yaml/v3" +) + +// This file is the only place that knows how the configuration is stored: the +// directory it lives in, what the file is called, and how it is parsed and +// serialized. Everything else works with *EvalConfig, so changing the on-disk +// shape stays a local edit. + +// DefaultEvalDir is where init writes the configuration and its artifacts. +const DefaultEvalDir = "evals" + +// EvalConfigBase is the single configuration file inside that directory. +const EvalConfigBase = "eval.yaml" + +// EvalConfigPath is the configuration file inside an eval directory. It is +// exported for error messages and for the azure.yaml $ref; readers should +// prefer OpenEvalConfig. +func EvalConfigPath(evalDir string) string { + return filepath.Join(evalDir, EvalConfigBase) +} + +// OpenEvalConfig reads the configuration under evalDir. +// +// A missing file returns (nil, nil): generate runs before init, so "no +// configuration yet" is an ordinary state rather than a failure. +func OpenEvalConfig(evalDir string) (*EvalConfig, error) { + cfg, err := LoadEvalConfig(EvalConfigPath(evalDir)) + if errors.Is(err, fs.ErrNotExist) { + return nil, nil + } + return cfg, err +} + +// LoadEvalConfig reads a configuration from an explicit path. The path is used +// verbatim, relative to the process working directory — never re-rooted. +func LoadEvalConfig(path string) (*EvalConfig, error) { + data, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("reading eval config %q: %w", path, err) + } + + var cfg EvalConfig + if err := yaml.Unmarshal(data, &cfg); err != nil { + return nil, fmt.Errorf("parsing eval config %q: %w", path, err) + } + return &cfg, nil +} + +// SaveEvalConfig writes cfg as the configuration under evalDir, creating the +// directory when it does not exist yet. +func SaveEvalConfig(evalDir string, cfg *EvalConfig) error { + if err := os.MkdirAll(evalDir, 0o750); err != nil { + return fmt.Errorf("creating %q: %w", evalDir, err) + } + return SaveEvalConfigTo(EvalConfigPath(evalDir), cfg) +} + +// SaveEvalConfigTo writes cfg over an explicit path, for callers that already +// resolved one. +func SaveEvalConfigTo(path string, cfg *EvalConfig) error { + body, err := yaml.Marshal(cfg) + if err != nil { + return fmt.Errorf("serializing eval config: %w", err) + } + if err := os.WriteFile(path, body, 0o600); err != nil { + return fmt.Errorf("writing eval config %q: %w", path, err) + } + return nil +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_test.go index 03e58c5cf35..b5cece4f058 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_test.go @@ -8,149 +8,198 @@ import ( "path/filepath" "testing" - "azureaieval/internal/pkg/evalcore" - "github.com/stretchr/testify/require" - "go.yaml.in/yaml/v3" ) -// sampleEvalConfig is the shape the spec documents for evals/.yaml. +// sampleEvalConfig is the shape the spec documents for evals/eval.yaml: two +// catalogs, then the evals defined over them. const sampleEvalConfig = ` -description: Quality gate for the support agent - -dataset: - name: support-golden - source: ./datasets/support-golden.jsonl - version: "1" +datasets: + - name: support-golden + source: ./datasets/support-golden.jsonl + version: "1" + - name: prod-registered evaluators: - - builtin.task_adherence - name: support-quality source: ./evaluators/support-quality.json - threshold: 4.0 - initialization_parameters: - deployment_name: gpt-4.1-nano - - safety-check - -target: - type: agent - name: support-agent - -options: - max_samples: 100 - evaluation_level: conversation + +evals: + - name: support-agent-smoke + description: Quality gate for the support agent + dataset: support-golden + evaluation_level: conversation + max_samples: 100 + evaluators: + - evaluator: builtin.task_adherence + - evaluator: support-quality + name: quality_strict + initialization_parameters: + deployment_name: gpt-4.1-nano + target: + type: agent + name: support-agent + + - name: support-agent-trace-eval + source: + type: traces + agent_name: support-agent + max_traces: 20 + evaluators: + - evaluator: builtin.task_adherence ` func loadFromString(t *testing.T, body string) *EvalConfig { t.Helper() - path := filepath.Join(t.TempDir(), "support-agent-smoke.yaml") - require.NoError(t, os.WriteFile(path, []byte(body), 0o600)) - cfg, err := LoadEvalConfig(path) + dir := t.TempDir() + require.NoError(t, os.WriteFile(EvalConfigPath(dir), []byte(body), 0o600)) + cfg, err := OpenEvalConfig(dir) require.NoError(t, err) + require.NotNil(t, cfg) return cfg } func TestLoadEvalConfig_ParsesAllSections(t *testing.T) { cfg := loadFromString(t, sampleEvalConfig) - require.NotNil(t, cfg.Dataset) - require.Equal(t, "support-golden", cfg.Dataset.Name) - require.Equal(t, "./datasets/support-golden.jsonl", cfg.Dataset.Source) - require.Equal(t, "1", cfg.Dataset.Version) + require.Len(t, cfg.Datasets, 2) + require.Equal(t, "support-golden", cfg.Datasets[0].Name) + require.Equal(t, "./datasets/support-golden.jsonl", cfg.Datasets[0].Source) + require.Equal(t, "1", cfg.Datasets[0].Version) + + require.Len(t, cfg.Evaluators, 1) + require.Equal(t, "support-quality", cfg.Evaluators[0].Name) - require.Len(t, cfg.Evaluators, 3) - require.Equal(t, TargetTypeAgent, cfg.Target.Type) - require.Equal(t, "support-agent", cfg.Target.Name) - require.Equal(t, EvaluationLevelConversation, cfg.Options.EvaluationLevel) - require.Equal(t, 100, cfg.Options.MaxSamples) + require.Equal(t, []string{"support-agent-smoke", "support-agent-trace-eval"}, cfg.EvalNames()) } -// The eval takes its name from the service entry that pulled the file in, so -// the body never repeats it. -func TestEval_TakesNameFromTheService(t *testing.T) { +// One file holds many evals, and each is selected by its own name. +func TestEval_SelectsByName(t *testing.T) { cfg := loadFromString(t, sampleEvalConfig) - eval := cfg.Eval("support-agent-smoke") - require.Equal(t, "support-agent-smoke", eval.Name) + eval, err := cfg.Eval("support-agent-smoke") + require.NoError(t, err) require.Equal(t, "support-golden", eval.Dataset) require.Equal(t, "Quality gate for the support agent", eval.Description) - require.Len(t, eval.Evaluators, 3) - require.Same(t, cfg.Target, eval.Target) + require.Equal(t, EvaluationLevelConversation, eval.EvaluationLevel) + require.Equal(t, 100, eval.MaxSamples) + require.Len(t, eval.Evaluators, 2) + require.Equal(t, TargetTypeAgent, eval.Target.Type) + require.Equal(t, "support-agent", eval.Target.Name) +} + +// A trace-backed eval invokes nothing, so agent_name filters rather than targets. +func TestEval_TraceSourceHasNoTarget(t *testing.T) { + cfg := loadFromString(t, sampleEvalConfig) + + eval, err := cfg.Eval("support-agent-trace-eval") + require.NoError(t, err) + require.Nil(t, eval.Target) + require.Equal(t, SourceTypeTraces, eval.Source.Type) + require.Equal(t, "support-agent", eval.Source.AgentName) + require.Equal(t, 20, eval.Source.MaxTraces) +} + +// An unnamed selection is only answered when the file declares exactly one, +// because guessing which eval a command meant is noticed only after it runs. +func TestEval_UnnamedIsAmbiguousWithSeveral(t *testing.T) { + cfg := loadFromString(t, sampleEvalConfig) + + _, err := cfg.Eval("") + require.ErrorContains(t, err, "--eval") + require.ErrorContains(t, err, "support-agent-trace-eval") + + single := loadFromString(t, "evals:\n - name: only\n evaluators:\n - evaluator: builtin.relevance\n") + eval, err := single.Eval("") + require.NoError(t, err) + require.Equal(t, "only", eval.Name) +} + +func TestEval_UnknownNameNamesWhatIsDeclared(t *testing.T) { + cfg := loadFromString(t, sampleEvalConfig) + + _, err := cfg.Eval("nope") + require.ErrorContains(t, err, "is not declared") + require.ErrorContains(t, err, "support-agent-smoke") } -// Only the referenced evaluators carrying a local source are this config's to -// publish. A built-in needs nothing, and one without a source already exists. -func TestCustomEvaluators_OnlyOwnsLocalSources(t *testing.T) { +// HasEval never falls back to "the only one", so a collision check cannot match +// a differently named entry. +func TestHasEvalAndRemoveEval(t *testing.T) { + cfg := loadFromString(t, sampleEvalConfig) + + require.True(t, cfg.HasEval("support-agent-smoke")) + require.False(t, cfg.HasEval("nope")) + require.False(t, cfg.HasEval("")) + + require.True(t, cfg.RemoveEval("support-agent-smoke")) + require.False(t, cfg.HasEval("support-agent-smoke")) + require.Equal(t, []string{"support-agent-trace-eval"}, cfg.EvalNames()) + require.False(t, cfg.RemoveEval("support-agent-smoke")) +} + +// Only catalog entries carrying a local source are this config's to publish. +// One without a source already exists on the project. +func TestCustomEvaluatorsAndLocalDatasets_OnlyOwnLocalSources(t *testing.T) { cfg := loadFromString(t, sampleEvalConfig) owned := cfg.CustomEvaluators() require.Len(t, owned, 1) require.Equal(t, "support-quality", owned[0].Name) require.Equal(t, "./evaluators/support-quality.json", owned[0].Source) + + local := cfg.LocalDatasets() + require.Len(t, local, 1) + require.Equal(t, "support-golden", local[0].Name, + "prod-registered has no source, so it is already on the project") } -// Evaluator entries accept a bare string or a mapping carrying the rest of the -// declaration. -func TestEvaluatorList_MixedForms(t *testing.T) { +func TestDeclarationLookups(t *testing.T) { cfg := loadFromString(t, sampleEvalConfig) - require.Len(t, cfg.Evaluators, 3) - - require.Equal(t, "builtin.task_adherence", cfg.Evaluators[0].Name) - require.True(t, cfg.Evaluators[0].IsBuiltin()) - require.Equal(t, "task_adherence", cfg.Evaluators[0].APIName(), - "the builtin prefix must be stripped before it reaches the service") - require.Nil(t, cfg.Evaluators[0].Threshold) - - require.Equal(t, "support-quality", cfg.Evaluators[1].Name) - require.False(t, cfg.Evaluators[1].IsBuiltin()) - require.NotNil(t, cfg.Evaluators[1].Threshold) - require.InDelta(t, 4.0, *cfg.Evaluators[1].Threshold, 0.0001) - require.Equal(t, "gpt-4.1-nano", - cfg.Evaluators[1].InitializationParameters["deployment_name"], - "the judge deployment is declared per evaluator, not once per eval") - - require.Equal(t, "safety-check", cfg.Evaluators[2].Name) - require.Nil(t, cfg.Evaluators[2].Threshold) + + ds, ok := cfg.DatasetDeclaration("support-golden") + require.True(t, ok) + require.Equal(t, "./datasets/support-golden.jsonl", ds.Source) + + _, ok = cfg.DatasetDeclaration("missing") + require.False(t, ok) + + ev, ok := cfg.EvaluatorDeclaration("support-quality") + require.True(t, ok) + require.Equal(t, "./evaluators/support-quality.json", ev.Source) } -// Round-tripping must not rewrite bare names into mappings. -func TestEvaluatorList_RoundTripKeepsCompactForm(t *testing.T) { - threshold := 4.0 - list := evalcore.EvaluatorList{ - {Name: "builtin.relevance"}, - {Name: "support-quality", Threshold: &threshold}, - } +// The configuration must survive a write/read cycle, because init and generate +// both append to a file they just read. +func TestEvalConfig_RoundTripsThroughTheStore(t *testing.T) { + dir := t.TempDir() + cfg := loadFromString(t, sampleEvalConfig) - out, err := yaml.Marshal(list) + require.NoError(t, SaveEvalConfig(dir, cfg)) + back, err := OpenEvalConfig(dir) require.NoError(t, err) + require.Equal(t, cfg, back) +} - var back evalcore.EvaluatorList - require.NoError(t, yaml.Unmarshal(out, &back)) - require.Len(t, back, 2) - require.Equal(t, "builtin.relevance", back[0].Name) - require.Nil(t, back[0].Threshold) - require.NotNil(t, back[1].Threshold) - require.Contains(t, string(out), "- builtin.relevance", - "an evaluator with only a name should stay a plain string") +// A missing file is an ordinary state: generate runs before init. +func TestOpenEvalConfig_MissingIsNotAnError(t *testing.T) { + cfg, err := OpenEvalConfig(t.TempDir()) + require.NoError(t, err) + require.Nil(t, cfg) } -// An evaluator carrying a source must not be flattened to its name, or the -// declaration that says what to publish is lost on the next write. -func TestEvaluatorList_RoundTripKeepsSource(t *testing.T) { - list := evalcore.EvaluatorList{ - {Name: "support-quality", Source: "./evaluators/support-quality.json"}, - {Name: "builtin.task_adherence", - InitializationParameters: map[string]any{"deployment_name": "gpt-4.1-nano"}}, - } +// SaveEvalConfig creates the directory, so generate can record an artifact in a +// project that has never run init. +func TestSaveEvalConfig_CreatesTheDirectory(t *testing.T) { + dir := filepath.Join(t.TempDir(), "evals") + require.NoError(t, SaveEvalConfig(dir, &EvalConfig{ + Datasets: []DatasetDecl{{Name: "generated", Source: "./datasets/generated.jsonl"}}, + })) - out, err := yaml.Marshal(list) + cfg, err := OpenEvalConfig(dir) require.NoError(t, err) - - var back evalcore.EvaluatorList - require.NoError(t, yaml.Unmarshal(out, &back)) - require.Len(t, back, 2) - require.Equal(t, "./evaluators/support-quality.json", back[0].Source) - require.Equal(t, "gpt-4.1-nano", back[1].InitializationParameters["deployment_name"]) + require.Len(t, cfg.Datasets, 1) + require.Empty(t, cfg.Evals, "a generate-only file is inert until init wires an eval") } func TestValidate_Accepts(t *testing.T) { @@ -158,6 +207,8 @@ func TestValidate_Accepts(t *testing.T) { } func TestValidate_Rejects(t *testing.T) { + const oneEval = "evals:\n - name: e\n evaluators:\n - evaluator: builtin.relevance\n" + cases := []struct { name string body string @@ -165,38 +216,76 @@ func TestValidate_Rejects(t *testing.T) { }{ { name: "dataset without a name", - body: "dataset:\n source: ./d.jsonl\nevaluators: [builtin.relevance]\n", + body: "datasets:\n - source: ./d.jsonl\n" + oneEval, wantErr: "'name' is required", }, + { + name: "duplicate dataset", + body: "datasets:\n - name: d\n - name: d\n" + oneEval, + wantErr: "duplicate dataset name", + }, + { + name: "built-in declared in the catalog", + body: "evaluators:\n - name: builtin.relevance\n" + oneEval, + wantErr: "needs no catalog entry", + }, + { + name: "version pinned alongside a source", + body: "evaluators:\n - name: q\n source: ./q.json\n version: \"3\"\n" + oneEval, + wantErr: "cannot be set with `source`", + }, + { + name: "no evals", + body: "datasets:\n - name: d\n", + wantErr: "at least one eval is required", + }, + { + name: "duplicate eval", + body: oneEval + " - name: e\n evaluators:\n - evaluator: builtin.relevance\n", + wantErr: "duplicate eval name", + }, { name: "no evaluators", - body: "evaluators: []\n", + body: "evals:\n - name: e\n evaluators: []\n", wantErr: "at least one evaluator is required", }, { - name: "built-in with a source to publish", - body: "evaluators:\n - name: builtin.relevance\n source: ./x.json\n", - wantErr: "has no source to publish", + name: "dataset and source both declared", + body: "datasets:\n - name: d\nevals:\n - name: e\n dataset: d\n" + + " source:\n type: traces\n evaluators:\n - evaluator: builtin.relevance\n", + wantErr: "declare one", }, { - name: "duplicate evaluator", - body: "evaluators: [builtin.relevance, builtin.relevance]\n", - wantErr: "duplicate evaluator name", + name: "dataset not in the catalog", + body: "evals:\n - name: e\n dataset: missing\n evaluators:\n - evaluator: builtin.relevance\n", + wantErr: "not in the datasets catalog", }, { - name: "version pinned alongside a source", - body: "evaluators:\n - name: q\n source: ./q.json\n version: \"3\"\n", - wantErr: "cannot be set with `source`", + name: "evaluator not in the catalog", + body: "evals:\n - name: e\n evaluators:\n - evaluator: quality\n", + wantErr: "not in the evaluators catalog", + }, + { + name: "duplicate criterion", + body: "evals:\n - name: e\n evaluators:\n" + + " - evaluator: builtin.relevance\n - evaluator: builtin.relevance\n", + wantErr: "duplicate criterion", }, { - name: "unsupported target type", - body: "evaluators: [builtin.relevance]\ntarget:\n type: prompt\n", + name: "unsupported source type", + body: "evals:\n - name: e\n source:\n type: prompt\n evaluators:\n - evaluator: builtin.relevance\n", + wantErr: "is not supported", + }, + { + name: "unsupported target type", + body: "evals:\n - name: e\n evaluators:\n - evaluator: builtin.relevance\n" + + " target:\n type: prompt\n", wantErr: "is not supported", }, { name: "invalid evaluation level", - body: "evaluators: [builtin.relevance]\n" + - "options:\n evaluation_level: sentence\n", + body: "evals:\n - name: e\n evaluation_level: sentence\n" + + " evaluators:\n - evaluator: builtin.relevance\n", wantErr: "evaluation_level", }, } @@ -210,59 +299,6 @@ func TestValidate_Rejects(t *testing.T) { } } -// One file is one eval, named after the file, so the directory listing is the -// list of evals a project declares. -func TestResolveEvalConfigPath(t *testing.T) { - write := func(t *testing.T, dir string, names ...string) { - t.Helper() - for _, n := range names { - require.NoError(t, os.WriteFile( - filepath.Join(dir, n), []byte("evaluators: [builtin.relevance]\n"), 0o600)) - } - } - - t.Run("the only eval is used when unnamed", func(t *testing.T) { - dir := t.TempDir() - write(t, dir, "pr-gate.yaml", "generate.yaml") - - path, err := ResolveEvalConfigPath(dir, "") - require.NoError(t, err) - require.Equal(t, filepath.Join(dir, "pr-gate.yaml"), path, - "the generation spec shares the directory and is not an eval") - }) - - t.Run("named eval", func(t *testing.T) { - dir := t.TempDir() - write(t, dir, "pr-gate.yaml", "nightly.yaml") - - path, err := ResolveEvalConfigPath(dir, "nightly") - require.NoError(t, err) - require.Equal(t, filepath.Join(dir, "nightly.yaml"), path) - }) - - t.Run("unknown name is an error", func(t *testing.T) { - dir := t.TempDir() - write(t, dir, "pr-gate.yaml") - - _, err := ResolveEvalConfigPath(dir, "nope") - require.ErrorContains(t, err, "is not declared") - }) - - t.Run("ambiguous without a name", func(t *testing.T) { - dir := t.TempDir() - write(t, dir, "pr-gate.yaml", "nightly.yaml") - - _, err := ResolveEvalConfigPath(dir, "") - require.ErrorContains(t, err, "--eval") - require.ErrorContains(t, err, "nightly") - }) - - t.Run("empty directory", func(t *testing.T) { - _, err := ResolveEvalConfigPath(t.TempDir(), "") - require.ErrorContains(t, err, "no evals") - }) -} - // outputDir accepts a directory or an explicit file path. func TestArtifactPath(t *testing.T) { cases := []struct { diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/generate_config.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/generate_config.go deleted file mode 100644 index 12fdb1cf6ff..00000000000 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/generate_config.go +++ /dev/null @@ -1,175 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package project - -import ( - "fmt" - "os" - "path/filepath" - "strings" - - "go.yaml.in/yaml/v3" -) - -// Conventional locations. All are relative to the working directory and are -// used verbatim — never re-rooted under the agent or project directory. -const ( - DefaultEvalDir = "evals" - DefaultGenerateConfig = "evals/generate.yaml" - DefaultDatasetsDir = "datasets" - DefaultEvaluatorsDir = "evaluators" -) - -// EvalConfigPath is where the body of the eval named by a service entry lives. -func EvalConfigPath(evalDir, evalName string) string { - return filepath.Join(evalDir, evalName+".yaml") -} - -// GenerateConfig says how the local dataset and evaluator artifacts referenced -// by an eval are produced. It is never deployed. -// -// generationModel: gpt-5.6-luna -// dataset: -// support-agent-smoke: -// sampleSize: 15 -// outputDir: ./datasets -// evaluator: -// support-quality: -// outputDir: ./evaluators -// deriveFrom: support-agent -// -// The maps are keyed by artifact name so `dataset generate ` and -// `evaluator generate ` each look up exactly the entry they were asked -// for, and generating one artifact never reads the other's settings. -type GenerateConfig struct { - GenerationModel string `yaml:"generationModel,omitempty" json:"generationModel,omitempty"` - Dataset map[string]DatasetGenSpec `yaml:"dataset,omitempty" json:"dataset,omitempty"` - Evaluator map[string]EvaluatorGenSpec `yaml:"evaluator,omitempty" json:"evaluator,omitempty"` -} - -// DatasetGenSpec configures synthetic dataset generation for one dataset. -type DatasetGenSpec struct { - SampleSize int `yaml:"sampleSize,omitempty" json:"sampleSize,omitempty"` - OutputDir string `yaml:"outputDir,omitempty" json:"outputDir,omitempty"` - // DeriveFrom names the agent whose context seeds generation. Optional: the - // eval's target supplies it, and --target overrides both. - DeriveFrom string `yaml:"deriveFrom,omitempty" json:"deriveFrom,omitempty"` - // Instructions points at a local file whose contents stand in for the - // agent's published instructions for this generation only. - Instructions string `yaml:"instructions,omitempty" json:"instructions,omitempty"` - // TraceDays seeds generation from that many days of recent traces. Zero - // disables it. Traces are a generation input only; they cannot be a run's - // data source. - TraceDays int `yaml:"traceDays,omitempty" json:"traceDays,omitempty"` -} - -// EvaluatorGenSpec configures rubric generation for one evaluator. -type EvaluatorGenSpec struct { - OutputDir string `yaml:"outputDir,omitempty" json:"outputDir,omitempty"` - // DeriveFrom names the agent the rubric is written against. - DeriveFrom string `yaml:"deriveFrom,omitempty" json:"deriveFrom,omitempty"` - // Instructions points at a local file whose contents stand in for the - // agent's published instructions for this generation only. - Instructions string `yaml:"instructions,omitempty" json:"instructions,omitempty"` - // TraceDays seeds generation from that many days of recent traces. Zero - // disables it. - TraceDays int `yaml:"traceDays,omitempty" json:"traceDays,omitempty"` -} - -// ArtifactRef is the name/source pair a generation run produces, so the -// command can tell the developer how to reference it. -// -// Generation writes artifacts only and never edits azure.yaml or the eval -// config: `init` declares the paths and `generate` fills them in, which is what -// keeps a generation run a data-file-only diff. -type ArtifactRef struct { - Name string `json:"name"` - Source string `json:"source"` -} - -// Sample-count bounds enforced by the generation service. -const ( - MinSampleSize = 15 - MaxSampleSize = 1000 - DefaultSampleSize = 15 -) - -// LoadGenerateConfig reads a generation spec from disk. -// -// A missing file is not an error. Generation is optional — a developer with -// hand-authored data and evaluators never writes one — and every setting it -// carries can be given on the command line instead. -func LoadGenerateConfig(path string) (*GenerateConfig, error) { - data, err := os.ReadFile(path) - if err != nil { - if os.IsNotExist(err) { - return &GenerateConfig{}, nil - } - return nil, fmt.Errorf("reading generation config %q: %w", path, err) - } - - var cfg GenerateConfig - if err := yaml.Unmarshal(data, &cfg); err != nil { - return nil, fmt.Errorf("parsing generation config %q: %w", path, err) - } - return &cfg, nil -} - -// DatasetSpec returns the settings for one dataset, and whether the config -// declared them. -func (c *GenerateConfig) DatasetSpec(name string) (DatasetGenSpec, bool) { - spec, ok := c.Dataset[name] - return spec, ok -} - -// EvaluatorSpec returns the settings for one evaluator, and whether the config -// declared them. -func (c *GenerateConfig) EvaluatorSpec(name string) (EvaluatorGenSpec, bool) { - spec, ok := c.Evaluator[name] - return spec, ok -} - -// ValidateSampleSize rejects a row count the service would reject, before a -// generation job is submitted and billed. -func ValidateSampleSize(n int) error { - if n != 0 && (n < MinSampleSize || n > MaxSampleSize) { - return fmt.Errorf( - "sample size must be between %d and %d, got %d", - MinSampleSize, MaxSampleSize, n) - } - return nil -} - -// ArtifactPath resolves an outputDir value against baseDir. The value may be a -// directory, in which case the file name is derived from resourceName and ext, -// or an explicit file path, which is used as-is. -func ArtifactPath(baseDir, outputDir, resourceName, ext string) string { - if outputDir == "" { - return filepath.Join(baseDir, resourceName+ext) - } - candidate := outputDir - if !filepath.IsAbs(candidate) { - candidate = filepath.Join(baseDir, candidate) - } - if looksLikeFile(outputDir, ext) { - return candidate - } - return filepath.Join(candidate, resourceName+ext) -} - -// looksLikeFile treats a trailing recognized extension as an explicit file path. -func looksLikeFile(p, ext string) bool { - got := strings.ToLower(filepath.Ext(p)) - if got == "" { - return false - } - if got == strings.ToLower(ext) { - return true - } - switch got { - case ".json", ".jsonl", ".yaml", ".yml": - return true - } - return false -} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/generate_config_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/generate_config_test.go deleted file mode 100644 index 5746b2286b6..00000000000 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/generate_config_test.go +++ /dev/null @@ -1,68 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package project - -import ( - "os" - "path/filepath" - "testing" - - "github.com/stretchr/testify/require" -) - -// The generation spec is keyed by artifact name, which is what makes -// `dataset generate ` and `evaluator generate ` able to look up -// exactly the entry they were asked for. -func TestLoadGenerateConfig_ParsesTheDocumentedShape(t *testing.T) { - body := ` -generationModel: gpt-5.6-luna -dataset: - support-agent-smoke: - sampleSize: 15 - outputDir: ./datasets -evaluator: - support-quality: - outputDir: ./evaluators - deriveFrom: support-agent -` - path := filepath.Join(t.TempDir(), "generate.yaml") - require.NoError(t, os.WriteFile(path, []byte(body), 0o600)) - - cfg, err := LoadGenerateConfig(path) - require.NoError(t, err) - require.Equal(t, "gpt-5.6-luna", cfg.GenerationModel) - - ds, ok := cfg.DatasetSpec("support-agent-smoke") - require.True(t, ok) - require.Equal(t, 15, ds.SampleSize) - require.Equal(t, "./datasets", ds.OutputDir) - - ev, ok := cfg.EvaluatorSpec("support-quality") - require.True(t, ok) - require.Equal(t, "./evaluators", ev.OutputDir) - require.Equal(t, "support-agent", ev.DeriveFrom) -} - -// Generation is optional: a developer with hand-authored data and evaluators -// never writes a spec, and the generate commands still run from flags alone. -func TestLoadGenerateConfig_MissingFileIsNotAnError(t *testing.T) { - cfg, err := LoadGenerateConfig(filepath.Join(t.TempDir(), "generate.yaml")) - require.NoError(t, err) - require.Empty(t, cfg.GenerationModel) - require.Empty(t, cfg.Dataset) - - _, ok := cfg.DatasetSpec("anything") - require.False(t, ok) -} - -// A row count the service would reject costs a billed job to find out about, -// so it is refused at the flag that carried it. -func TestValidateSampleSize(t *testing.T) { - require.NoError(t, ValidateSampleSize(0), "unset means the default applies") - require.NoError(t, ValidateSampleSize(MinSampleSize)) - require.NoError(t, ValidateSampleSize(MaxSampleSize)) - - require.ErrorContains(t, ValidateSampleSize(MinSampleSize-1), "must be between") - require.ErrorContains(t, ValidateSampleSize(MaxSampleSize+1), "must be between") -} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go index 6fd01d06680..2097621ed6d 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go @@ -145,26 +145,27 @@ func (p *EvalServiceTargetProvider) Deploy( baseDir := serviceRelativeDir(serviceConfig) - // The eval takes its name from the service entry that pulled this config - // in, which is what makes one service per eval work. - eval := cfg.Eval(serviceConfig.Name) - - // 1. Dataset. + // 1. Datasets the configuration owns. Paths are kept so an eval that names + // one can derive its columns without reading the blob back. anyChanged := false - datasetPath := "" - if cfg.Dataset != nil { - report(progress, fmt.Sprintf("Reconciling dataset %s", cfg.Dataset.Name)) - datasetPath = resolveSource(baseDir, cfg.Dataset.Source) - version, changed, err := reconciler.EnsureDataset(ctx, *cfg.Dataset, datasetPath) + datasetPaths := map[string]string{} + for _, decl := range cfg.Datasets { + if decl.Source == "" { + continue + } + report(progress, fmt.Sprintf("Reconciling dataset %s", decl.Name)) + localPath := resolveSource(baseDir, decl.Source) + datasetPaths[decl.Name] = localPath + version, changed, err := reconciler.EnsureDataset(ctx, decl, localPath) if err != nil { - return nil, fmt.Errorf("dataset %q: %w", cfg.Dataset.Name, err) + return nil, fmt.Errorf("dataset %q: %w", decl.Name, err) } anyChanged = anyChanged || changed - report(progress, describeResult("dataset", cfg.Dataset.Name, version, changed)) + report(progress, describeResult("dataset", decl.Name, version, changed)) } - // 2. Evaluators this config owns. Built-ins and already-registered ones - // need no publish. + // 2. Evaluators this configuration owns. Built-ins and already-registered + // ones need no publish. for _, decl := range cfg.CustomEvaluators() { report(progress, fmt.Sprintf("Reconciling evaluator %s", decl.Name)) localPath := resolveSource(baseDir, decl.Source) @@ -176,14 +177,17 @@ func (p *EvalServiceTargetProvider) Deploy( report(progress, describeResult("evaluator", decl.Name, version, changed)) } - // 3. The eval. Evals are immutable, so a change upstream means a new one + // 3. The evals. Evals are immutable, so a change upstream means a new one // must be created and the stored id replaced. - report(progress, fmt.Sprintf("Reconciling eval %s", eval.Name)) - id, err := reconciler.EnsureEval(ctx, eval, datasetPath, anyChanged) - if err != nil { - return nil, fmt.Errorf("eval %q: %w", eval.Name, err) + for i := range cfg.Evals { + eval := cfg.Evals[i] + report(progress, fmt.Sprintf("Reconciling eval %s", eval.Name)) + id, err := reconciler.EnsureEval(ctx, eval, datasetPaths[eval.Dataset], anyChanged) + if err != nil { + return nil, fmt.Errorf("eval %q: %w", eval.Name, err) + } + report(progress, fmt.Sprintf("Eval %s is %s", eval.Name, id)) } - report(progress, fmt.Sprintf("Eval %s is %s", eval.Name, id)) return &azdext.ServiceDeployResult{}, nil } @@ -322,18 +326,19 @@ func Fingerprint(path string) (string, error) { // evaluators are untouched. Without this a retargeted group keeps running // against the old definition. func FingerprintGroup(group Eval) (string, error) { - // The id is server-assigned. The description is carried in the group's - // metadata, so editing it does change the request, but recreating an - // immutable group over a reworded description would cost the group id and - // break comparison against earlier runs. It is documentation, not - // evaluation semantics, so an edit lands the next time the group is - // recreated for a reason that matters. + // Only substance is hashed. The id is server-assigned; name and description + // are what UpdateEvalParametersBody reaches, so an edit confined to them is + // pushed in place and must not cost the eval its id and its run history. + // Everything else — dataset, source, evaluators, target, level — is fixed at + // creation, so a change there is a new eval. + name := group.Name group.ID = "" + group.Name = "" group.Description = "" data, err := json.Marshal(group) if err != nil { - return "", fmt.Errorf("hashing eval %q: %w", group.Name, err) + return "", fmt.Errorf("hashing eval %q: %w", name, err) } sum := sha256.Sum256(data) return hex.EncodeToString(sum[:]), nil diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval_test.go index 987b2815c44..e20ecd49009 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval_test.go @@ -68,25 +68,37 @@ func TestServiceRelativeDirDefaultsToProjectRoot(t *testing.T) { // includes against. func TestEvalConfigFromServiceReadsInlineConfig(t *testing.T) { svc := &azdext.ServiceConfig{ - Name: "support-agent-smoke", + Name: "support-agent-evals", AdditionalProperties: propsFrom(t, map[string]any{ - "dataset": map[string]any{"name": "golden", "source": "./datasets/golden.jsonl"}, - "evaluators": []any{"builtin.task_adherence"}, - "target": map[string]any{"type": "agent", "name": "my-agent"}, + "datasets": []any{ + map[string]any{"name": "golden", "source": "./datasets/golden.jsonl"}, + }, + "evals": []any{ + map[string]any{ + "name": "support-agent-smoke", + "dataset": "golden", + "evaluators": []any{ + map[string]any{"evaluator": "builtin.task_adherence"}, + }, + "target": map[string]any{"type": "agent", "name": "my-agent"}, + }, + }, }), } cfg, err := EvalConfigFromService(svc, "") require.NoError(t, err) - require.NotNil(t, cfg.Dataset) - require.Equal(t, "golden", cfg.Dataset.Name) - require.Len(t, cfg.Evaluators, 1) - require.Equal(t, "builtin.task_adherence", cfg.Evaluators[0].Name) - require.Equal(t, "my-agent", cfg.Target.Name) - - // The eval's name is the service key, which is what makes one service per - // eval work without the body repeating it. - require.Equal(t, "support-agent-smoke", cfg.Eval(svc.Name).Name) + require.Len(t, cfg.Datasets, 1) + require.Equal(t, "golden", cfg.Datasets[0].Name) + + // One service covers every eval in the file it pulled in, so the eval is + // selected by its own name rather than by the service key. + eval, err := cfg.Eval("support-agent-smoke") + require.NoError(t, err) + require.Equal(t, "golden", eval.Dataset) + require.Len(t, eval.Evaluators, 1) + require.Equal(t, "builtin.task_adherence", eval.Evaluators[0].Evaluator) + require.Equal(t, "my-agent", eval.Target.Name) } func TestEvalConfigFromServiceRejectsEmptyService(t *testing.T) { @@ -95,16 +107,16 @@ func TestEvalConfigFromServiceRejectsEmptyService(t *testing.T) { require.Contains(t, err.Error(), "no eval configuration") } -// Evals are immutable, so a change to the eval's own declaration has to be +// Evals are immutable, so a change to an eval's own declaration has to be // detectable. Upstream artifact fingerprints do not cover it: retargeting an // eval at a different agent leaves the dataset and evaluators untouched. func TestFingerprintGroupTracksMeaningfulChanges(t *testing.T) { base := Eval{ - Name: "quality", - Dataset: "golden", - Evaluators: evalcore.EvaluatorList{{Name: "builtin.task_adherence"}}, - Target: &Target{Type: "agent", Name: "agent-a"}, - Options: &Options{EvaluationLevel: EvaluationLevelTurn}, + Name: "quality", + Dataset: "golden", + Evaluators: evalcore.EvaluatorList{{Evaluator: "builtin.task_adherence"}}, + Target: &Target{Type: "agent", Name: "agent-a"}, + EvaluationLevel: EvaluationLevelTurn, } original, err := FingerprintGroup(base) @@ -117,16 +129,25 @@ func TestFingerprintGroupTracksMeaningfulChanges(t *testing.T) { cases := map[string]func(g *Eval){ "target": func(g *Eval) { g.Target = &Target{Type: "agent", Name: "agent-b"} }, "evaluators": func(g *Eval) { - g.Evaluators = append(g.Evaluators, evalcore.EvaluatorRef{Name: "builtin.similarity"}) + g.Evaluators = append(g.Evaluators, evalcore.EvaluatorRef{Evaluator: "builtin.similarity"}) }, "judge deployment": func(g *Eval) { g.Evaluators = evalcore.EvaluatorList{{ - Name: "builtin.task_adherence", + Evaluator: "builtin.task_adherence", InitializationParameters: map[string]any{"deployment_name": "gpt-4o-mini"}, }} }, - "options": func(g *Eval) { g.Options = &Options{EvaluationLevel: EvaluationLevelConversation} }, - "dataset": func(g *Eval) { g.Dataset = "other" }, + "version pin": func(g *Eval) { + g.Evaluators = evalcore.EvaluatorList{{ + Evaluator: "builtin.task_adherence", Version: "2", + }} + }, + "evaluation level": func(g *Eval) { g.EvaluationLevel = EvaluationLevelConversation }, + "dataset": func(g *Eval) { g.Dataset = "other" }, + "source": func(g *Eval) { + g.Dataset = "" + g.Source = &SourceDecl{Type: SourceTypeTraces, AgentName: "agent-a"} + }, } for name, mutate := range cases { t.Run(name, func(t *testing.T) { @@ -141,21 +162,48 @@ func TestFingerprintGroupTracksMeaningfulChanges(t *testing.T) { } } -// Server-assigned and cosmetic fields must not force a recreate. -func TestFingerprintGroupIgnoresIdAndDescription(t *testing.T) { +// The fingerprint covers substance only. The id is server-assigned, and name +// and description are what UpdateEvalParametersBody reaches — an edit confined +// to those is pushed in place, so it must not fork the run history. +func TestFingerprintGroupIgnoresIdNameAndDescription(t *testing.T) { base := Eval{ Name: "quality", Dataset: "golden", - Evaluators: evalcore.EvaluatorList{{Name: "builtin.task_adherence"}}, + Evaluators: evalcore.EvaluatorList{{Evaluator: "builtin.task_adherence"}}, } original, err := FingerprintGroup(base) require.NoError(t, err) noisy := base noisy.ID = "eval_abc123" + noisy.Name = "quality-renamed" noisy.Description = "reworded" digest, err := FingerprintGroup(noisy) require.NoError(t, err) require.Equal(t, original, digest) } + +// Editing one eval must not recreate its siblings: the unit compared is the +// eval's own subtree, never the file. +func TestFingerprintGroupIsScopedToOneEval(t *testing.T) { + gate := Eval{ + Name: "support-agent-gate", + Dataset: "prod-golden", + Evaluators: evalcore.EvaluatorList{{Evaluator: "builtin.task_adherence"}}, + } + regression := Eval{ + Name: "support-agent-regression-eval", + Dataset: "support-agent-regression", + Evaluators: evalcore.EvaluatorList{{Evaluator: "builtin.task_adherence"}}, + } + + before, err := FingerprintGroup(regression) + require.NoError(t, err) + + gate.Evaluators = append(gate.Evaluators, evalcore.EvaluatorRef{Evaluator: "builtin.similarity"}) + + after, err := FingerprintGroup(regression) + require.NoError(t, err) + require.Equal(t, before, after, "editing a sibling must leave this eval alone") +} From 4591118281d9297c6ca5974847a9d9ebec82d657 Mon Sep 17 00:00:00 2001 From: mohessie Date: Thu, 6 Aug 2026 02:13:45 +0300 Subject: [PATCH 104/320] Keep an eval across a rename, and bound the first trace run init gains --max-traces, writing max_traces on a trace-backed eval so a first run is bounded rather than taking the service's default of 1000. Deleting the key by hand restores that default, so the flag is not a ceiling. Passing it without --source traces is refused rather than silently ignored. A rename is now an update, not a fork. Excluding the name from the fingerprint was not enough on its own: the environment keys the fingerprint by name, so a renamed declaration found nothing recorded and created a second eval, stranding every earlier run behind an id. The environment now records the id against the substance as well, which is what recognises the rename, and the new name is pushed with UpdateEvalParametersBody. That lookup is only unambiguous while two evals cannot share a substance, so validation refuses a pair differing solely by name and description, naming both. The alternative is a rename adopting whichever of them deployed last. --- .../azure.ai.evaluations/internal/cmd/init.go | 13 ++++ .../internal/cmd/init_test.go | 25 ++++++-- .../internal/cmd/reconciler.go | 60 +++++++++++++++++-- .../internal/cmd/surface_test.go | 4 +- .../internal/pkg/eval_api/models.go | 9 +++ .../internal/pkg/eval_api/operations.go | 16 +++++ .../internal/project/eval_config.go | 22 +++++++ .../internal/project/eval_config_test.go | 6 ++ 8 files changed, 144 insertions(+), 11 deletions(-) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go index cd80a2ac7d5..46ec0c0047c 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go @@ -40,6 +40,7 @@ func newInitCommand() *cobra.Command { target string source string dataset string + maxTraces int evaluators []string judgeModel string path string @@ -65,6 +66,12 @@ func newInitCommand() *cobra.Command { if source == initSourceTraces && dataset != "" { return errors.New("--source traces reads production traces, so it takes no --dataset") } + if cmd.Flags().Changed("max-traces") && source != initSourceTraces { + return errors.New("--max-traces caps a trace-backed eval; pass --source traces") + } + if maxTraces < 0 { + return errors.New("--max-traces must be positive") + } if source == "" { source = initSourceDataset } @@ -116,6 +123,7 @@ func newInitCommand() *cobra.Command { target: target, source: source, dataset: dataset, + maxTraces: maxTraces, evaluators: evaluators, judgeModel: judgeModel, rubricName: target + "-quality", @@ -187,6 +195,9 @@ func newInitCommand() *cobra.Command { "Where rows come from: dataset or traces. Defaults to dataset.") cmd.Flags().StringVar(&dataset, "dataset", "", "Path to a local .jsonl, or the name of a registered dataset.") + cmd.Flags().IntVar(&maxTraces, "max-traces", project.DefaultScaffoldMaxTraces, + "Cap on traces read by a --source traces eval. Delete max_traces from the "+ + "file to take the service default instead.") cmd.Flags().StringArrayVar(&evaluators, "evaluator", nil, "Evaluator reference, repeatable. Use builtin. for a built-in. "+ "Passing this replaces the defaults, so it also opts out of rubric generation.") @@ -214,6 +225,7 @@ type scaffoldInput struct { target string source string dataset string + maxTraces int evaluators []string judgeModel string rubricName string @@ -258,6 +270,7 @@ func planScaffold(in scaffoldInput) scaffold { eval.Source = &project.SourceDecl{ Type: project.SourceTypeTraces, AgentName: in.target, + MaxTraces: in.maxTraces, } } else { datasetName := in.evalName diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_test.go index c0f2549fbdf..4f714bfcf03 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_test.go @@ -11,6 +11,7 @@ import ( "azureaieval/internal/project" "github.com/stretchr/testify/require" + "go.yaml.in/yaml/v3" ) // scaffoldFor runs planScaffold against a fresh configuration, which is what @@ -71,18 +72,34 @@ func TestScaffold_AppendsToAnExistingConfiguration(t *testing.T) { } // A trace-backed eval invokes nothing, so agent_name filters instead of -// targeting, and the service default caps how many traces are read. +// targeting, and a scaffolded cap keeps the first run bounded rather than +// taking the service's default of 1000. func TestScaffold_TraceSourceHasNoTarget(t *testing.T) { plan, _ := scaffoldFor(t, scaffoldInput{ - evalName: "support-agent-trace-eval", - target: "support-agent", - source: initSourceTraces, + evalName: "support-agent-trace-eval", + target: "support-agent", + source: initSourceTraces, + maxTraces: project.DefaultScaffoldMaxTraces, }) require.Nil(t, plan.eval.Target) require.NotNil(t, plan.eval.Source) require.Equal(t, project.SourceTypeTraces, plan.eval.Source.Type) require.Equal(t, "support-agent", plan.eval.Source.AgentName) + require.Equal(t, 20, plan.eval.Source.MaxTraces) +} + +// Omitting the cap leaves the key out, which is how the service default is +// taken — writing a zero would send one. +func TestScaffold_TraceCapIsOmittedWhenZero(t *testing.T) { + plan, _ := scaffoldFor(t, scaffoldInput{ + evalName: "t", target: "a", source: initSourceTraces, + }) + require.Zero(t, plan.eval.Source.MaxTraces) + + body, err := yaml.Marshal(plan.eval) + require.NoError(t, err) + require.NotContains(t, string(body), "max_traces") } // The default set is a built-in plus a generated rubric: the built-in alone diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go index c770f26ea7b..0e0affef613 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go @@ -16,6 +16,7 @@ import ( "time" "azureaieval/internal/pkg/dataset_api" + "azureaieval/internal/pkg/eval_api" "azureaieval/internal/project" ) @@ -428,9 +429,10 @@ func (r *evalReconciler) EnsureEval( return group.ID, nil } - // Groups are immutable, so a change to the group's own declaration — - // evaluators, target, or options — needs a new group just as much as a - // change to an upstream artifact does. + // Evals are immutable, so a change to the eval's own substance — evaluators, + // dataset, source, target, level — needs a new eval just as much as a change + // to an upstream artifact does. Name and description are excluded from the + // digest and pushed in place instead. digest, err := project.FingerprintGroup(group) if err != nil { return "", err @@ -441,12 +443,23 @@ func (r *evalReconciler) EnsureEval( } cached := r.ec.getEnvValue(ctx, idKey("eval", group.Name)) + if cached == "" && !recreate { + // Nothing recorded under this name, but the substance may already be + // deployed under the name it had before. The environment records the id + // against the digest as well, which is what recognises a rename rather + // than reading it as a delete plus an add. + if adopted := r.adoptRenamed(ctx, group, digest); adopted != "" { + cached = adopted + } + } if cached != "" && !recreate { if _, err := r.ec.evalClient.GetOpenAIEval(ctx, cached); err == nil { - // Record the digest on reuse as well, otherwise a group deployed + // Record the digest on reuse as well, otherwise an eval deployed // before fingerprinting existed never establishes a baseline and // later edits go undetected. _ = r.ec.setEnvValue(ctx, key, digest) + _ = r.ec.setEnvValue(ctx, idKey("eval", group.Name), cached) + _ = r.ec.setEnvValue(ctx, digestIDKey(digest), cached) _ = r.ec.setEnvValue(ctx, envKeyEvalID, cached) return cached, nil } @@ -466,12 +479,42 @@ func (r *evalReconciler) EnsureEval( } _ = r.ec.setEnvValue(ctx, key, digest) _ = r.ec.setEnvValue(ctx, idKey("eval", group.Name), created.ID) - // EVAL_ID stays the last-deployed group, which is what the commands + _ = r.ec.setEnvValue(ctx, digestIDKey(digest), created.ID) + // EVAL_ID stays the last-deployed eval, which is what the commands // fall back to when a config names only one. _ = r.ec.setEnvValue(ctx, envKeyEvalID, created.ID) return created.ID, nil } +// adoptRenamed reclaims the eval this declaration used to be called, so a +// rename keeps the id and every run under it rather than forking the history. +// +// The name is what UpdateEvalParametersBody reaches, so the new one is pushed +// to the service. A failure there is not fatal: the eval is still the right one +// and the declaration still resolves, it just reads under its old name in the +// portal until the next deploy. +func (r *evalReconciler) adoptRenamed( + ctx context.Context, + group project.Eval, + digest string, +) string { + id := r.ec.getEnvValue(ctx, digestIDKey(digest)) + if id == "" { + return "" + } + remote, err := r.ec.evalClient.GetOpenAIEval(ctx, id) + if err != nil { + return "" + } + if remote.Name == group.Name { + return id + } + _, _ = r.ec.evalClient.UpdateOpenAIEval(ctx, id, &eval_api.UpdateOpenAIEvalRequest{ + Name: group.Name, + }) + return id +} + // sameDefinition reports whether the locally authored definition already // matches what the service holds. // @@ -547,3 +590,10 @@ func versionKey(kind, name string) string { func idKey(kind, name string) string { return project.FingerprintKey(kind, name) + "_ID" } + +// digestIDKey records an eval's id against its substance, which is what lets a +// renamed declaration find the eval it already deployed. Keyed by a prefix of +// the digest, because the whole hash makes an unreadable environment variable. +func digestIDKey(digest string) string { + return "EVAL_SUBSTANCE_" + strings.ToUpper(digest[:16]) + "_ID" +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/surface_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/surface_test.go index 0bfe4b2514f..b0a70d6d038 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/surface_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/surface_test.go @@ -170,8 +170,8 @@ func TestInitFlagsMatchTheSpec(t *testing.T) { }) assert.ElementsMatch(t, []string{ - "--name", "--target", "--source", "--dataset", "--evaluator", - "--judge-model", "--path", "--force", + "--name", "--target", "--source", "--dataset", "--max-traces", + "--evaluator", "--judge-model", "--path", "--force", }, got, "init's flags are a table in the spec; change both together") } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/models.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/models.go index 122828750de..aa6c4f89ebb 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/models.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/models.go @@ -278,6 +278,15 @@ type CreateOpenAIEvalRequest struct { TestingCriteria []TestingCriterion `json:"testing_criteria,omitempty"` } +// UpdateOpenAIEvalRequest is UpdateEvalParametersBody: the only fields an eval +// accepts after creation. Testing criteria and the data source are fixed at +// create time, and the service drops anything else here silently rather than +// rejecting it. +type UpdateOpenAIEvalRequest struct { + Name string `json:"name,omitempty"` + Metadata map[string]string `json:"metadata,omitempty"` +} + // OpenAIEval is the response for an OpenAI eval definition. type OpenAIEval struct { ID string `json:"id"` diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/operations.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/operations.go index f28b8e81ef5..114a012f40d 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/operations.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/operations.go @@ -410,6 +410,22 @@ func (c *EvalClient) DeleteOpenAIEval(ctx context.Context, evalID string) error return err } +// UpdateOpenAIEval edits an eval in place. The route is a POST on the eval +// itself, matching how this surface spells run cancel — there is no PATCH verb +// here. +// +// Only what UpdateEvalParametersBody reaches is editable: name, metadata and +// properties. Anything else the service drops silently, so substance never +// travels through this call and an edit that touches it is a new eval. +func (c *EvalClient) UpdateOpenAIEval( + ctx context.Context, + evalID string, + request *UpdateOpenAIEvalRequest, +) (*OpenAIEval, error) { + path := pathOpenAIEvals + "/" + url.PathEscape(evalID) + return doRequestTyped[OpenAIEval](c, ctx, http.MethodPost, path, nil, request, "") +} + // CreateOpenAIEvalRun starts a run for an OpenAI eval definition. func (c *EvalClient) CreateOpenAIEvalRun( ctx context.Context, diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go index c9bea4324a0..8b8f1581194 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go @@ -78,6 +78,11 @@ const ( SourceTypeResponses = "responses" ) +// DefaultScaffoldMaxTraces is the cap init writes on a trace-backed eval, so a +// first run is bounded rather than taking the service's own default of 1000. +// Deleting max_traces from the file restores that default. +const DefaultScaffoldMaxTraces = 20 + // Target names what the run invokes. type Target struct { Type string `yaml:"type" json:"type"` @@ -214,6 +219,7 @@ func (c *EvalConfig) Validate() error { } seen := map[string]bool{} + substance := map[string]string{} for i, eval := range c.Evals { if eval.Name == "" { return fmt.Errorf("evals[%d]: 'name' is required", i) @@ -226,6 +232,22 @@ func (c *EvalConfig) Validate() error { if err := c.validateEval(i, eval); err != nil { return err } + + // Two evals that differ only by name are indistinguishable once + // deployed: the environment records an id against each eval's substance + // so a renamed declaration can find what it already deployed, and a + // shared substance makes that lookup ambiguous. + digest, err := FingerprintGroup(eval) + if err != nil { + return err + } + if first, clash := substance[digest]; clash { + return fmt.Errorf( + "evals[%d] (%s): identical to %q apart from its name and description; "+ + "give them different evaluators, datasets or settings, or declare one", + i, eval.Name, first) + } + substance[digest] = eval.Name } return nil } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_test.go index b5cece4f058..cb51446d56c 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_test.go @@ -288,6 +288,12 @@ func TestValidate_Rejects(t *testing.T) { " evaluators:\n - evaluator: builtin.relevance\n", wantErr: "evaluation_level", }, + { + name: "two evals differing only by name", + body: "evals:\n - name: a\n evaluators:\n - evaluator: builtin.relevance\n" + + " - name: b\n evaluators:\n - evaluator: builtin.relevance\n", + wantErr: "identical to", + }, } for _, tc := range cases { From 7c052ada5ee7019ea5277345d239e95884443fd4 Mon Sep 17 00:00:00 2001 From: mohessie Date: Thu, 6 Aug 2026 02:25:01 +0300 Subject: [PATCH 105/320] Say which eval on one flag, and let a run read another dataset --eval already accepted a name or an id, so the --eval-id beside it was redundant. On run start it was worse than redundant: the flag variable was shadowed by the resolved id, so the value was parsed and never read. Every run command now takes one --eval. run start drops --level. The level decides the row mapping, so overriding it per run would put two incomparable result sets under one eval's history and bypass the supported_evaluation_levels check azd up does against the declared level. A second level is a second eval. run start gains --dataset, which reads a different catalog dataset for this run without touching the declaration. It needs a declared eval to override, and the name has to be in the catalog, so a typo fails before the run is submitted rather than scoring nothing. run output export defaults to csv and adds jsonl. The results are a table, and a build artifact is normally read by a spreadsheet or a diff; jsonl is for a downstream job that would rather stream than hold the run in memory. --- .../internal/cmd/resolution_test.go | 15 +-- .../azure.ai.evaluations/internal/cmd/run.go | 43 +++++--- .../internal/cmd/run_ops.go | 8 +- .../internal/cmd/run_ops_test.go | 19 ++-- .../internal/cmd/run_output.go | 100 +++++++++++------- 5 files changed, 116 insertions(+), 69 deletions(-) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/resolution_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/resolution_test.go index ae0a60152a3..fa417ce43c3 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/resolution_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/resolution_test.go @@ -28,16 +28,17 @@ func TestResolveMaxSamples_Precedence(t *testing.T) { assert.Equal(t, 0, resolveMaxSamples(0, &project.Eval{MaxSamples: 0})) } -func TestResolveLevel_Precedence(t *testing.T) { - withOptions := &project.Eval{ +// The level is the eval's alone. A per-run override would put two incomparable +// result sets under one eval's history, and would bypass the +// supported_evaluation_levels check `azd up` does against the declared level. +func TestResolveLevel_ComesFromTheEval(t *testing.T) { + declared := &project.Eval{ EvaluationLevel: project.EvaluationLevelConversation, } - assert.Equal(t, project.EvaluationLevelTurn, resolveLevel(project.EvaluationLevelTurn, withOptions), - "the flag wins over the config") - assert.Equal(t, project.EvaluationLevelConversation, resolveLevel("", withOptions)) - assert.Empty(t, resolveLevel("", &project.Eval{}), "unset defers to the service default") - assert.Empty(t, resolveLevel("", nil)) + assert.Equal(t, project.EvaluationLevelConversation, resolveLevel(declared)) + assert.Empty(t, resolveLevel(&project.Eval{}), "unset defers to the service default") + assert.Empty(t, resolveLevel(nil)) } // A group's target decides which run-time fields its criteria can bind. Getting diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go index 1bb13de25bf..c3591d523f9 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go @@ -72,9 +72,8 @@ func newRunCommand() *cobra.Command { func buildRunCommand(use, short string) *cobra.Command { var ( groupName string - evalID string + datasetName string runName string - level string maxSamples int wait bool failOn string @@ -112,6 +111,24 @@ func buildRunCommand(use, short string) *cobra.Command { group := ref.Eval configPath := ref.ConfigPath + if datasetName != "" { + if !ref.Declared() { + return errors.New( + "--dataset overrides the dataset an eval declares, so it needs a " + + "declared eval; pass --eval with a name from the configuration") + } + if _, ok := ref.Config.DatasetDeclaration(datasetName); !ok { + return fmt.Errorf( + "dataset %q is not in the catalog in %s", + datasetName, filepath.ToSlash(configPath)) + } + // The eval keeps its own declaration; only this run reads elsewhere. + overridden := *group + overridden.Dataset = datasetName + overridden.Source = nil + group = &overridden + } + if ref.Declared() { if err := ec.checkDatasetRegistered(ctx, ref.Config, group, configPath); err != nil { return err @@ -139,7 +156,7 @@ func buildRunCommand(use, short string) *cobra.Command { } metadata := map[string]string{} - if lvl := resolveLevel(level, group); lvl != "" { + if lvl := resolveLevel(group); lvl != "" { metadata["evaluation_level"] = lvl } @@ -199,12 +216,11 @@ func buildRunCommand(use, short string) *cobra.Command { } cmd.Flags().StringVar(&groupName, "eval", "", - "Name of the eval to run. Defaults to the only one declared.") - cmd.Flags().StringVar(&evalID, "eval-id", "", - "Run against an existing eval by id, ignoring the config.") + "Name of the eval to run, or its id. Defaults to the only one declared.") + cmd.Flags().StringVar(&datasetName, "dataset", "", + "Catalog dataset to read instead of the one the eval declares. "+ + "Must satisfy the eval's column schema.") cmd.Flags().StringVar(&runName, "name", "", "Name for this run. Defaults to the eval name plus a timestamp.") - cmd.Flags().StringVar(&level, "level", "", - "Scoring granularity: turn or conversation. Defaults to the service default (turn).") cmd.Flags().IntVar(&maxSamples, "max-samples", 0, "Cap the rows sent from the dataset.") cmd.Flags().BoolVar(&wait, "wait", true, "Block until the run reaches a terminal state.") @@ -557,10 +573,13 @@ func scanJSONL(r io.Reader, limit int) ([]map[string]any, error) { } // resolveLevel prefers the flag, then the eval's own declaration. -func resolveLevel(flag string, group *project.Eval) string { - if flag != "" { - return flag - } +// resolveLevel is the eval's declared scoring granularity. +// +// There is no per-run override: the level decides the row mapping, so two +// levels under one eval would put incomparable result sets in the same history, +// and it would bypass the supported_evaluation_levels check `azd up` does +// against the declared level. A second level is a second eval. +func resolveLevel(group *project.Eval) string { if group != nil { return group.EvaluationLevel } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go index deb0d0a49e5..cea3b01bb4d 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go @@ -79,7 +79,7 @@ func newRunListCommand() *cobra.Command { []string{"RUN ID", "NAME", "STATUS", "RESULTS"}, rows) }, } - addEvalFlags(cmd, &groupName) + addEvalFlag(cmd, &groupName) cmd.Flags().IntVar(&limit, "limit", 0, "Return at most this many runs. Omit for the service default.") cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") @@ -173,7 +173,7 @@ func newRunShowCommand() *cobra.Command { cmd.Flags().BoolVar(&wait, "wait", false, "Block until the run reaches a terminal state before reporting.") addFailOnFlag(cmd, &failOn) - addEvalFlags(cmd, &groupName) + addEvalFlag(cmd, &groupName) cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") return cmd } @@ -236,7 +236,7 @@ func newRunCancelCommand() *cobra.Command { return nil }, } - addEvalFlags(cmd, &groupName) + addEvalFlag(cmd, &groupName) cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") return cmd } @@ -287,7 +287,7 @@ func newRunDeleteCommand() *cobra.Command { return nil }, } - addEvalFlags(cmd, &groupName) + addEvalFlag(cmd, &groupName) cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") return cmd } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops_test.go index 1a5bd5ac978..8259bfd6b35 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops_test.go @@ -57,16 +57,20 @@ func TestRunStartMirrorsCompositeFlags(t *testing.T) { } require.NotNil(t, start) - for _, flag := range []string{"eval-id", "eval", "name", "level", "max-samples", "wait", "no-wait"} { + for _, flag := range []string{"eval", "dataset", "name", "max-samples", "wait", "no-wait"} { require.NotNil(t, start.Flags().Lookup(flag), "run start should accept --%s", flag) } + + // The level decides the row mapping, so a per-run override would put two + // incomparable result sets under one eval. A second level is a second eval. + require.Nil(t, start.Flags().Lookup("level"), "run start must not offer --level") } -// Every command that acts on an eval takes the id the same two ways. -// `run start --eval-id` is the form the CI example uses, and `run list` used to -// reject that flag and accept only a positional, so a script that worked for -// one sibling failed on the next. -func TestEvalCommandsAcceptIDAsAFlag(t *testing.T) { +// Every command that acts on an eval says which one the same way. One flag +// takes a name from the configuration or a raw service id: an eval created +// outside a project has no declaration to name, and a second --eval-id beside +// it was accepted and silently ignored. +func TestEvalCommandsTakeOneEvalFlag(t *testing.T) { subs := map[string]*cobra.Command{} for _, sub := range newRunCommand().Commands() { subs["run "+sub.Name()] = sub @@ -83,8 +87,9 @@ func TestEvalCommandsAcceptIDAsAFlag(t *testing.T) { } { cmd := subs[name] require.NotNil(t, cmd, "%s should exist", name) - require.NotNil(t, cmd.Flags().Lookup("eval-id"), "%s should accept --eval-id", name) require.NotNil(t, cmd.Flags().Lookup("eval"), "%s should accept --eval", name) + require.Nil(t, cmd.Flags().Lookup("eval-id"), + "%s must not keep --eval-id beside --eval", name) } } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output.go index 7dacd49abb7..3c6c0e36ad7 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output.go @@ -5,6 +5,7 @@ package cmd import ( "encoding/csv" + "encoding/json" "fmt" "io" "os" @@ -12,6 +13,7 @@ import ( "strings" "azureaieval/internal/pkg/eval_api" + "azureaieval/internal/project" "github.com/spf13/cobra" ) @@ -111,7 +113,7 @@ func newRunOutputListCommand() *cobra.Command { cmd.Flags().BoolVar(&failedOnly, "failed-only", false, "Show only the rows that failed.") cmd.Flags().StringVar(&outFile, "output-file", "", "Write JSON results to this path.") - addEvalFlags(cmd, &groupName) + addEvalFlag(cmd, &groupName) cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") return cmd } @@ -166,7 +168,7 @@ func newRunOutputShowCommand() *cobra.Command { } cmd.Flags().StringVar(&runID, "run", "", "Run the item belongs to. Defaults to the most recent run.") - addEvalFlags(cmd, &groupName) + addEvalFlag(cmd, &groupName) cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") return cmd } @@ -217,26 +219,36 @@ func newRunOutputExportCommand() *cobra.Command { w = f } - if format == "json" { + switch format { + case formatCSV: + return writeResultsCSV(w, run) + case formatJSON: return emitJSON(w, run) + case formatJSONL: + return writeResultsJSONL(w, run) + default: + return fmt.Errorf( + "--format %q is not supported; use %s, %s or %s", + format, formatCSV, formatJSON, formatJSONL) } - return writeResultsCSV(w, run) }, } - cmd.Flags().StringVar(&format, "format", "json", "Output format: json or csv.") + cmd.Flags().StringVar(&format, "format", formatCSV, + fmt.Sprintf("Output format: %s, %s or %s.", formatCSV, formatJSON, formatJSONL)) cmd.Flags().StringVar(&outFile, "output-file", "", "Write to this path instead of stdout.") - addEvalFlags(cmd, &groupName) + addEvalFlag(cmd, &groupName) cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") return cmd } -// resolveEvalID takes the eval id from the argument, from a group named -// with --eval, or from the id cached in the azd environment. +// resolveEvalID takes the eval id from the argument, from --eval, or from the +// id cached in the azd environment. // -// The cached id is the last group deployed, which is unambiguous only while a -// config declares one. --eval is how the others are reached without -// having to know their service ids. +// --eval accepts a name or a raw id on the one flag: an eval created outside a +// project has no declaration to name, and the environment records one id per +// name, so editing a declaration leaves every run of the previous eval +// reachable only by id. func resolveEvalID( cmd *cobra.Command, ec *evalContext, @@ -247,46 +259,27 @@ func resolveEvalID( return args[0], nil } - if flag, err := cmd.Flags().GetString("eval-id"); err == nil && flag != "" { - return flag, nil - } - if groupName != "" { - if id := ec.getEnvValue(cmd.Context(), idKey("eval", groupName)); id != "" { - return id, nil + ref, err := ec.resolveEvalRef(cmd.Context(), project.DefaultEvalDir, groupName) + if err != nil { + return "", err } - return "", fmt.Errorf( - "eval %q has no id recorded in this environment; deploy it first, "+ - "or pass its id directly", groupName) + return ref.ID, nil } if cached := ec.getEnvValue(cmd.Context(), envKeyEvalID); cached != "" { return cached, nil } return "", fmt.Errorf( - "no eval id given; pass it as an argument, name one with --eval, "+ - "or set %s in the azd environment", - envKeyEvalID) + "no eval given; pass its id as an argument, or name one with --eval") } -// addEvalFlag registers the flag that names an eval from the config, for -// commands that never take a raw service id. +// addEvalFlag registers the flag that says which eval a command acts on. It +// takes a name from the configuration or a raw service id, which is why there +// is no second --eval-id beside it. func addEvalFlag(cmd *cobra.Command, target *string) { cmd.Flags().StringVar(target, "eval", "", - "Name of the eval declared in azure.yaml.") -} - -// addEvalFlags registers the two ways to say which eval a command acts -// on: --eval names one from the config, --eval-id gives its service id. -// -// The id is also accepted as a positional argument. The flag exists because -// `run start --eval-id` already spells it that way, and a script that learned -// it there should not have to find out that the sibling commands take only a -// positional. -func addEvalFlags(cmd *cobra.Command, target *string) { - addEvalFlag(cmd, target) - cmd.Flags().String("eval-id", "", - "Id of the eval. Same as passing the id as an argument.") + "Name of the eval declared in the configuration, or its id.") } // latestOrNamedRun returns the named run, or the most recent one for the eval. @@ -453,3 +446,32 @@ func writeResultsCSV(w io.Writer, run *eval_api.OpenAIEvalRun) error { } return nil } + +// Export formats. csv is the default because the results are a table and a +// build artifact is normally read by a spreadsheet or a diff. +const ( + formatCSV = "csv" + formatJSON = "json" + formatJSONL = "jsonl" +) + +// writeResultsJSONL emits one criterion per line, which is what a downstream +// job can stream without holding the whole run in memory. +func writeResultsJSONL(w io.Writer, run *eval_api.OpenAIEvalRun) error { + enc := json.NewEncoder(w) + if len(run.PerTestingCriteria) == 0 { + return enc.Encode(map[string]any{"run_id": run.ID, "status": run.Status}) + } + for _, cr := range run.PerTestingCriteria { + if err := enc.Encode(map[string]any{ + "run_id": run.ID, + "status": run.Status, + "testing_criteria": cr.TestingCriteria, + "passed": cr.Passed, + "failed": cr.Failed, + }); err != nil { + return err + } + } + return nil +} From 7da5c1b2ad70406935e9708f05a3659d8bd4b615 Mon Sep 17 00:00:00 2001 From: mohessie Date: Thu, 6 Aug 2026 02:41:10 +0300 Subject: [PATCH 106/320] Nest generation jobs under what produced them, and add eval create The job group was top level and every command under it tried both generation resources, reporting whichever answered. The two share no collection, so that meant guessing the endpoint from an id prefix that is not a documented contract. Each resource now owns its own group -- dataset job and evaluator job -- and a lookup that misses names the sibling group, because the ids look alike and reaching for the wrong one is the likely mistake. Both groups gain delete. The job record is what is discarded; the artifact it produced is already registered as its own version and is untouched. eval create makes one declared eval without deploying the rest, for a project that is not deployed as a whole or, with --from-file, for no project at all. It runs the same reconcile path azd up uses rather than a second creation route, so there is never a second definition to maintain. The reattach hints printed by --no-wait now name the group that owns the job. A test that scans the source for suggested commands caught them still pointing at the old top-level path. --- .../internal/cmd/dataset.go | 1 + .../internal/cmd/eval_group.go | 84 ++++++- .../internal/cmd/evaluator.go | 1 + .../internal/cmd/generate.go | 11 +- .../azure.ai.evaluations/internal/cmd/job.go | 208 +++++++++++------- .../azure.ai.evaluations/internal/cmd/root.go | 2 +- .../internal/cmd/surface_test.go | 15 +- .../internal/pkg/eval_api/operations.go | 29 +++ 8 files changed, 260 insertions(+), 91 deletions(-) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset.go index f4ac919306b..05a42c354e1 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset.go @@ -33,6 +33,7 @@ func newDatasetCommand() *cobra.Command { newDatasetShowCommand(), newDatasetDeleteCommand(), newDatasetVersionsCommand(), + newJobCommand(datasetJobs), ) return cmd } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/eval_group.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/eval_group.go index 8f7566c00cf..3082153d4b4 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/eval_group.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/eval_group.go @@ -5,16 +5,92 @@ package cmd import ( "fmt" + "path/filepath" "azureaieval/internal/pkg/eval_api" + "azureaieval/internal/project" "github.com/spf13/cobra" ) -// The eval group is read and delete only. Creation belongs to `azd up`, which -// owns reconciliation: a second creation path would drift from the declared -// config, and reconciliation could not then tell whether to adopt an eval it -// found or replace it. +// Creation normally belongs to `azd up`, which owns reconciliation. `create` +// is the same path for a single eval outside a project, and takes the +// configuration rather than a wall of flags so there is never a second +// definition to maintain. + +// newEvalCreateCommand creates one declared eval without deploying the rest. +func newEvalCreateCommand() *cobra.Command { + var ( + fromFile string + evalDir string + endpointFlg string + ) + + cmd := &cobra.Command{ + Use: "create [name]", + Short: "Create one eval declared in the configuration.", + Long: "Create one eval declared in the configuration.\n\n" + + "`azd up` reconciles every eval in the file. This creates a single one, " + + "for a project that is not deployed as a whole — or, with --from-file, " + + "for no project at all.\n\n" + + "The name is optional while the configuration declares exactly one eval.", + Args: cobra.MaximumNArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + ctx := cmd.Context() + + path := fromFile + if path == "" { + path = project.EvalConfigPath(evalDir) + } + cfg, err := project.LoadEvalConfig(path) + if err != nil { + return err + } + if err := cfg.Validate(); err != nil { + return err + } + + eval, err := cfg.Eval(firstArg(args)) + if err != nil { + return err + } + + ec, err := newEvalContext(ctx, endpointFlg) + if err != nil { + return err + } + defer ec.Close() + + // Local sources resolve against the file, not the working directory, + // so the columns are read from where the declaration points. + datasetPath := "" + if decl, ok := cfg.DatasetDeclaration(eval.Dataset); ok && decl.Source != "" { + datasetPath = filepath.Join(filepath.Dir(path), decl.Source) + } + + reconciler := &evalReconciler{ec: ec} + id, err := reconciler.EnsureEval(ctx, *eval, datasetPath, false) + if err != nil { + return err + } + + if isJSON(cmd) { + return emitJSON(cmd.OutOrStdout(), map[string]string{ + "id": id, "name": eval.Name, + }) + } + fmt.Fprintf(cmd.OutOrStdout(), "%s Created eval: %s (%s)\n", doneMark, eval.Name, id) + return nil + }, + } + + cmd.Flags().StringVar(&fromFile, "from-file", "", + "Read the configuration from this path instead of the eval directory.") + cmd.Flags().StringVar(&evalDir, "path", project.DefaultEvalDir, + "Directory holding the evaluation configuration.") + cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") + return cmd +} func newEvalListCommand() *cobra.Command { var ( diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator.go index c5f86a19d15..48aeb48a477 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator.go @@ -26,6 +26,7 @@ func newEvaluatorCommand() *cobra.Command { newEvaluatorShowCommand(), newEvaluatorDeleteCommand(), newEvaluatorVersionsCommand(), + newJobCommand(evaluatorJobs), ) return cmd } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go index 4dd77505f0d..0c27d6702da 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go @@ -164,7 +164,7 @@ func (ec *evalContext) generateRubric( return nil, fmt.Errorf("submitting the rubric generation job: %w", err) } if noWait { - reportSubmitted(out, job.ID) + reportSubmitted(out, "azd ai eval evaluator", job.ID) return nil, nil } @@ -187,10 +187,11 @@ func (ec *evalContext) generateRubric( // // The job id goes into the command rather than being left as a placeholder: // --no-wait exists so the caller can walk away, and the line they walk away -// with has to be the one they can paste when they come back. -func reportSubmitted(out io.Writer, jobID string) { +// with has to be the one they can paste when they come back. The group is named +// too, because the two job types share no collection. +func reportSubmitted(out io.Writer, group, jobID string) { fmt.Fprintf(out, " submitted job %s\n", jobID) - fmt.Fprintf(out, "\nReattach with: azd ai eval job show %s\n", jobID) + fmt.Fprintf(out, "\nReattach with: %s job show %s\n", group, jobID) } // generateDataset submits the data generation job and downloads the result. @@ -212,7 +213,7 @@ func (ec *evalContext) generateDataset( return nil, fmt.Errorf("submitting the data generation job: %w", err) } if noWait { - reportSubmitted(out, job.ID) + reportSubmitted(out, "azd ai dataset", job.ID) return nil, nil } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/job.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/job.go index 1c094e20c77..c4a9d48aa23 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/job.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/job.go @@ -13,37 +13,89 @@ import ( ) // Generation runs as two independent long-running resources — one for datasets, -// one for evaluators — and a job id does not say which it came from. Rather -// than make the caller remember, every command here tries both. +// one for evaluators — sharing no collection. A job group therefore nests under +// the resource that produced it: a top-level `job show ` would have to guess +// the endpoint from an id prefix that is not a documented contract. const ( jobKindDataset = "dataset" jobKindEvaluator = "evaluator" ) -func newJobCommand() *cobra.Command { +// jobKind binds a group to one generation resource, so every command under it +// calls one endpoint rather than trying both and reporting whichever answered. +type jobKind struct { + name string + list func(context.Context, *evalContext) ([]eval_api.GenerationJob, error) + get func(context.Context, *evalContext, string) (*eval_api.GenerationJob, error) + cancel func(context.Context, *evalContext, string) (*eval_api.GenerationJob, error) + remove func(context.Context, *evalContext, string) error +} + +var datasetJobs = jobKind{ + name: jobKindDataset, + list: func(ctx context.Context, ec *evalContext) ([]eval_api.GenerationJob, error) { + out, err := ec.evalClient.ListDataGenerationJobs(ctx, ProjectEndpointAPIVersion) + if err != nil { + return nil, err + } + return out.Data, nil + }, + get: func(ctx context.Context, ec *evalContext, id string) (*eval_api.GenerationJob, error) { + return ec.evalClient.GetDataGenerationJob(ctx, id, ProjectEndpointAPIVersion) + }, + cancel: func(ctx context.Context, ec *evalContext, id string) (*eval_api.GenerationJob, error) { + return ec.evalClient.CancelDataGenerationJob(ctx, id, ProjectEndpointAPIVersion) + }, + remove: func(ctx context.Context, ec *evalContext, id string) error { + return ec.evalClient.DeleteDataGenerationJob(ctx, id, ProjectEndpointAPIVersion) + }, +} + +var evaluatorJobs = jobKind{ + name: jobKindEvaluator, + list: func(ctx context.Context, ec *evalContext) ([]eval_api.GenerationJob, error) { + out, err := ec.evalClient.ListEvaluatorGenerationJobs(ctx, ProjectEndpointAPIVersion) + if err != nil { + return nil, err + } + return out.Data, nil + }, + get: func(ctx context.Context, ec *evalContext, id string) (*eval_api.GenerationJob, error) { + return ec.evalClient.GetEvaluatorGenerationJob(ctx, id, ProjectEndpointAPIVersion) + }, + cancel: func(ctx context.Context, ec *evalContext, id string) (*eval_api.GenerationJob, error) { + return ec.evalClient.CancelEvaluatorGenerationJob(ctx, id, ProjectEndpointAPIVersion) + }, + remove: func(ctx context.Context, ec *evalContext, id string) error { + return ec.evalClient.DeleteEvaluatorGenerationJob(ctx, id, ProjectEndpointAPIVersion) + }, +} + +func newJobCommand(kind jobKind) *cobra.Command { cmd := &cobra.Command{ Use: "job", - Short: "Inspect and cancel generation jobs.", - Long: "Inspect and cancel generation jobs.\n\n" + - "This is the resume path for `dataset generate` and `evaluator generate`: " + - "a job started with --no-wait, or one whose client was interrupted, is " + - "reattached to here rather than restarted.", + Short: fmt.Sprintf("Inspect, cancel and delete %s generation jobs.", kind.name), + Long: fmt.Sprintf("Inspect, cancel and delete %s generation jobs.\n\n", kind.name) + + fmt.Sprintf("This is the resume path for `%s generate`: a job started with ", kind.name) + + "--no-wait, or one whose client was interrupted, is reattached to here " + + "rather than restarted.", } cmd.AddCommand( - newJobListCommand(), - newJobShowCommand(), - newJobCancelCommand(), + newJobListCommand(kind), + newJobShowCommand(kind), + newJobCancelCommand(kind), + newJobDeleteCommand(kind), ) return cmd } -func newJobListCommand() *cobra.Command { +func newJobListCommand(kind jobKind) *cobra.Command { var endpointFlg string cmd := &cobra.Command{ Use: "list", - Short: "List the project's generation jobs.", + Short: fmt.Sprintf("List the project's %s generation jobs.", kind.name), Args: cobra.NoArgs, RunE: func(cmd *cobra.Command, args []string) error { ctx := cmd.Context() @@ -53,40 +105,23 @@ func newJobListCommand() *cobra.Command { } defer ec.Close() - datasets, err := ec.evalClient.ListDataGenerationJobs(ctx, ProjectEndpointAPIVersion) - if err != nil { - return fmt.Errorf("listing dataset generation jobs: %w", err) - } - evaluators, err := ec.evalClient.ListEvaluatorGenerationJobs(ctx, ProjectEndpointAPIVersion) + jobs, err := kind.list(ctx, ec) if err != nil { - return fmt.Errorf("listing evaluator generation jobs: %w", err) - } - - type jobRow struct { - ID string `json:"id"` - Kind string `json:"kind"` - Status string `json:"status"` - } - rows := make([]jobRow, 0, len(datasets.Data)+len(evaluators.Data)) - for _, j := range datasets.Data { - rows = append(rows, jobRow{ID: j.ID, Kind: jobKindDataset, Status: j.Status}) - } - for _, j := range evaluators.Data { - rows = append(rows, jobRow{ID: j.ID, Kind: jobKindEvaluator, Status: j.Status}) + return fmt.Errorf("listing %s generation jobs: %w", kind.name, err) } if isJSON(cmd) { - return emitJSONList(cmd.OutOrStdout(), rows) + return emitJSONList(cmd.OutOrStdout(), jobs) } - if len(rows) == 0 { - fmt.Fprintln(cmd.OutOrStdout(), "No generation jobs found.") + if len(jobs) == 0 { + fmt.Fprintf(cmd.OutOrStdout(), "No %s generation jobs found.\n", kind.name) return nil } - table := make([][]string, 0, len(rows)) - for _, r := range rows { - table = append(table, []string{r.ID, r.Kind, r.Status}) + table := make([][]string, 0, len(jobs)) + for _, j := range jobs { + table = append(table, []string{j.ID, j.Status}) } - return emitTable(cmd.OutOrStdout(), []string{"JOB ID", "KIND", "STATUS"}, table) + return emitTable(cmd.OutOrStdout(), []string{"JOB ID", "STATUS"}, table) }, } @@ -94,12 +129,12 @@ func newJobListCommand() *cobra.Command { return cmd } -func newJobShowCommand() *cobra.Command { +func newJobShowCommand(kind jobKind) *cobra.Command { var endpointFlg string cmd := &cobra.Command{ Use: "show ", - Short: "Show a generation job.", + Short: fmt.Sprintf("Show a %s generation job.", kind.name), Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { jobID := args[0] @@ -111,9 +146,9 @@ func newJobShowCommand() *cobra.Command { } defer ec.Close() - job, _, err := findGenerationJob(ctx, ec, jobID) + job, err := kind.get(ctx, ec, jobID) if err != nil { - return err + return jobLookupError(kind, jobID, err) } if isJSON(cmd) { @@ -131,12 +166,12 @@ func newJobShowCommand() *cobra.Command { return cmd } -func newJobCancelCommand() *cobra.Command { +func newJobCancelCommand(kind jobKind) *cobra.Command { var endpointFlg string cmd := &cobra.Command{ Use: "cancel ", - Short: "Cancel an in-flight generation job.", + Short: fmt.Sprintf("Cancel an in-flight %s generation job.", kind.name), Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { jobID := args[0] @@ -148,28 +183,54 @@ func newJobCancelCommand() *cobra.Command { } defer ec.Close() - _, kind, err := findGenerationJob(ctx, ec, jobID) + canceled, err := kind.cancel(ctx, ec, jobID) if err != nil { - return err + return jobLookupError(kind, jobID, err) } - var canceled *eval_api.GenerationJob - if kind == jobKindDataset { - canceled, err = ec.evalClient.CancelDataGenerationJob( - ctx, jobID, ProjectEndpointAPIVersion) - } else { - canceled, err = ec.evalClient.CancelEvaluatorGenerationJob( - ctx, jobID, ProjectEndpointAPIVersion) + if isJSON(cmd) { + return emitJSON(cmd.OutOrStdout(), canceled) } + fmt.Fprintf(cmd.OutOrStdout(), "Cancelled %s generation job %s (%s)\n", + kind.name, jobID, canceled.Status) + return nil + }, + } + + cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") + return cmd +} + +func newJobDeleteCommand(kind jobKind) *cobra.Command { + var endpointFlg string + + cmd := &cobra.Command{ + Use: "delete ", + Short: fmt.Sprintf("Delete a %s generation job record.", kind.name), + Long: fmt.Sprintf("Delete a %s generation job record.\n\n", kind.name) + + "The artifact the job produced is already registered as its own version " + + "and is not affected.", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + jobID := args[0] + + ctx := cmd.Context() + ec, err := newEvalContext(ctx, endpointFlg) if err != nil { - return fmt.Errorf("cancelling job %s: %w", jobID, err) + return err + } + defer ec.Close() + + if err := kind.remove(ctx, ec, jobID); err != nil { + return jobLookupError(kind, jobID, err) } if isJSON(cmd) { - return emitJSON(cmd.OutOrStdout(), canceled) + return emitJSON(cmd.OutOrStdout(), map[string]string{ + "id": jobID, "kind": kind.name, "status": "deleted", + }) } - fmt.Fprintf(cmd.OutOrStdout(), "Cancelled %s generation job %s (%s)\n", - kind, jobID, canceled.Status) + fmt.Fprintf(cmd.OutOrStdout(), "Deleted %s generation job %s\n", kind.name, jobID) return nil }, } @@ -178,24 +239,17 @@ func newJobCancelCommand() *cobra.Command { return cmd } -// findGenerationJob resolves an id against both job types and reports which one -// answered, so that a caller never has to know which command started it. -func findGenerationJob( - ctx context.Context, - ec *evalContext, - jobID string, -) (*eval_api.GenerationJob, string, error) { - if job, err := ec.evalClient.GetDataGenerationJob( - ctx, jobID, ProjectEndpointAPIVersion, - ); err == nil { - return job, jobKindDataset, nil - } - if job, err := ec.evalClient.GetEvaluatorGenerationJob( - ctx, jobID, ProjectEndpointAPIVersion, - ); err == nil { - return job, jobKindEvaluator, nil +// jobLookupError names the sibling group, because the two job types share an id +// shape and reaching for the wrong one is the likely mistake. +func jobLookupError(kind jobKind, jobID string, err error) error { + if eval_api.IsNotFound(err) { + other := jobKindEvaluator + if kind.name == jobKindEvaluator { + other = jobKindDataset + } + return fmt.Errorf( + "no %s generation job %q in this project; if it generated a %s, "+ + "use the %s job group instead", kind.name, jobID, other, other) } - return nil, "", fmt.Errorf( - "no generation job %s in this project; "+ - "`azd ai eval job list` shows the ones there are", jobID) + return fmt.Errorf("reading %s generation job %s: %w", kind.name, jobID, err) } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/root.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/root.go index 5be5fcc9e46..4f6fcd8860e 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/root.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/root.go @@ -49,7 +49,7 @@ func NewRootCommand() *cobra.Command { newDatasetCommand(), newRunCommand(), newEvaluatorCommand(), - newJobCommand(), + newEvalCreateCommand(), newEvalListCommand(), newEvalShowCommand(), newEvalDeleteCommand(), diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/surface_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/surface_test.go index b0a70d6d038..2d266ead1e6 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/surface_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/surface_test.go @@ -54,6 +54,12 @@ func TestCommandTreeMatchesTheSpec(t *testing.T) { "dataset update", "dataset versions", "dataset versions list", + "dataset job", + "dataset job cancel", + "dataset job delete", + "dataset job list", + "dataset job show", + "create", "delete", "evaluator", "evaluator create", @@ -64,11 +70,12 @@ func TestCommandTreeMatchesTheSpec(t *testing.T) { "evaluator update", "evaluator versions", "evaluator versions list", + "evaluator job", + "evaluator job cancel", + "evaluator job delete", + "evaluator job list", + "evaluator job show", "init", - "job", - "job cancel", - "job list", - "job show", "list", "run", "run cancel", diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/operations.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/operations.go index 114a012f40d..0b18f6556a2 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/operations.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/operations.go @@ -172,6 +172,35 @@ func (c *EvalClient) cancelGenerationJob( c, ctx, http.MethodPost, path, nil, json.RawMessage(`{}`), apiVersion) } +// DeleteDataGenerationJob removes a dataset generation job record. +func (c *EvalClient) DeleteDataGenerationJob( + ctx context.Context, + operationID string, + apiVersion string, +) error { + return c.deleteGenerationJob(ctx, pathDataGenerationJobs, operationID, apiVersion) +} + +// DeleteEvaluatorGenerationJob removes an evaluator generation job record. +func (c *EvalClient) DeleteEvaluatorGenerationJob( + ctx context.Context, + operationID string, + apiVersion string, +) error { + return c.deleteGenerationJob(ctx, pathEvaluatorGenerationJobs, operationID, apiVersion) +} + +// deleteGenerationJob discards the job record. The artifact the job produced is +// already registered and is not affected. +func (c *EvalClient) deleteGenerationJob( + ctx context.Context, + basePath, operationID, apiVersion string, +) error { + path := basePath + "/" + url.PathEscape(operationID) + _, err := c.doRequest(ctx, http.MethodDelete, path, nil, nil, apiVersion) + return err +} + // GetAgent reads an agent from the project's catalog. // // Only the newest version is returned, which is the one generation is seeded From b8d40277f04e45a445598f19964287f7d04466a8 Mon Sep 17 00:00:00 2001 From: mohessie Date: Thu, 6 Aug 2026 02:59:54 +0300 Subject: [PATCH 107/320] Send the sources --from named, and pick a default that works --from was parsed, validated, and then dropped: the value reached generationPlan and nothing read it, so every dataset generate sent the same request whatever was asked for. Wiring it up turned up three more things. It was a single string, but the spec has it selecting one or more of the sources, so it is repeatable now. Asking for traces sent no traces. The builder emitted a traces source only when a day window came with it, and dataset generate has no --trace-days, so the documented --from traces produced an empty request. A window narrows the query rather than authorizing it; without one the source now goes out unbounded. Asking for something unbuildable sent a job anyway. --from prompt with no instruction, or --from file, which the service takes through dataset upload rather than as a generation source, quietly submitted whatever was left. The job is billed and what comes back looks the same either way, so the caller had no way to tell. Those now stop the command and name the flag that fixes it. The default is the spec's: traces when the project has Application Insights connected, otherwise the agent. The connection string is how a project says it collects traces at all, and the agents extension already reads that key. --from agent carries the agent's instructions as a prompt, because that is what the spec says it generates from, and because the agent source alone fails server-side for every agent -- the prompt is what the existing retry falls back to. Without it the default in a project with no Application Insights would have been a request that always fails. --- .../internal/cmd/context.go | 24 +++ .../internal/cmd/generate.go | 47 ++++- .../internal/cmd/generate_commands.go | 32 ++- .../internal/cmd/generate_sources_test.go | 59 ++++++ .../internal/cmd/surface_test.go | 32 +++ .../internal/pkg/eval_api/generation.go | 107 +++++++--- .../internal/pkg/eval_api/generation_test.go | 190 ++++++++++++++++++ 7 files changed, 447 insertions(+), 44 deletions(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_sources_test.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/generation_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/context.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/context.go index 0b987db0315..1c897a0f906 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/context.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/context.go @@ -12,6 +12,7 @@ import ( "azureaieval/internal/pkg/dataset_api" "azureaieval/internal/pkg/eval_api" + "azureaieval/internal/project" "github.com/Azure/azure-sdk-for-go/sdk/azcore" "github.com/Azure/azure-sdk-for-go/sdk/azidentity" @@ -145,6 +146,29 @@ func (ec *evalContext) getEnvValue(ctx context.Context, key string) string { return val.Value } +// appInsightsEnvKey is where a connected Application Insights resource lands in +// the azd environment. azd's own provisioning writes it, and the agents +// extension reads the same key to pass tracing configuration to a running +// agent, so its presence is the project's answer to "are traces being +// collected?". +const appInsightsEnvKey = "APPLICATIONINSIGHTS_CONNECTION_STRING" + +// defaultGenerationSource picks what `dataset generate` sends when --from was +// not given, from the Application Insights connection string the project has +// (or has not) been given. +// +// Traces are the better dataset when they exist, being real conversations +// rather than synthesized ones, so they win whenever the project is wired to +// collect them. Outside a project, or in one with no Application Insights, +// there are no traces to ask for and the agent's own definition is all that is +// left. +func defaultGenerationSource(appInsightsConnection string) []string { + if appInsightsConnection != "" { + return []string{project.GenerateFromTraces} + } + return []string{project.GenerateFromAgent} +} + func (ec *evalContext) Close() { if ec.azdClient != nil { ec.azdClient.Close() diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go index 0c27d6702da..3091cbd2f27 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go @@ -44,9 +44,9 @@ type generationPlan struct { OutputDir string // SampleSize applies to dataset generation only. SampleSize int - // From selects which source the rows are generated out of. Empty lets the - // project decide: traces when Application Insights is connected. - From string + // From is what --from named: which of the service's sources to send. Empty + // sends whatever the plan has to offer. + From []string // TraceDays seeds generation from that many days of recent traces. TraceDays int } @@ -154,9 +154,12 @@ func (ec *evalContext) generateRubric( ) (*project.ArtifactRef, error) { fmt.Fprintf(out, "Generating rubric %s...\n", plan.Name) - sources := eval_api.BuildGenerationSources( - "agent", plan.Agent, "", plan.Instruction, plan.traceOptions(), + sources, unbuildable := eval_api.BuildGenerationSources( + plan.From, plan.Agent, "", plan.Instruction, plan.traceOptions(), ) + if err := refuseUnbuildableSources(unbuildable); err != nil { + return nil, err + } req := eval_api.NewEvaluatorGenerationJobRequest(plan.Name, plan.Model, sources) job, err := ec.evalClient.CreateEvaluatorGenerationJob(ctx, req, ProjectEndpointAPIVersion) @@ -183,6 +186,33 @@ func (ec *evalContext) generateRubric( return &project.ArtifactRef{Name: plan.Name, Source: relativeSource(plan.BaseDir, path)}, nil } +// refuseUnbuildableSources reports a --from the plan could not honour. +// +// Submitting anyway would run a billed job seeded from less than was asked for +// and return a plausible-looking artifact, which is the worst outcome: the +// caller has no way to tell it apart from one built the way they intended. +func refuseUnbuildableSources(kinds []string) error { + if len(kinds) == 0 { + return nil + } + reasons := map[string]string{ + "prompt": "--from prompt needs --agent-instruction or --agent-instruction-file", + "agent": "--from agent needs a target agent; pass --target, " + + "or declare one under target: in eval.yaml", + "file": "--from file is not a generation source; " + + "register the file with `azd ai dataset create --file ` instead", + } + messages := make([]string, 0, len(kinds)) + for _, k := range kinds { + if reason, ok := reasons[k]; ok { + messages = append(messages, reason) + continue + } + messages = append(messages, fmt.Sprintf("--from %s cannot be built from this plan", k)) + } + return errors.New(strings.Join(messages, "; ")) +} + // reportSubmitted says what was started and how to get back to it. // // The job id goes into the command rather than being left as a placeholder: @@ -203,9 +233,12 @@ func (ec *evalContext) generateDataset( ) (*project.ArtifactRef, error) { fmt.Fprintf(out, "Generating dataset %s (%d samples)...\n", plan.Name, plan.SampleSize) - sources := eval_api.BuildGenerationSources( - "agent", plan.Agent, "", plan.Instruction, plan.traceOptions(), + sources, unbuildable := eval_api.BuildGenerationSources( + plan.From, plan.Agent, "", plan.Instruction, plan.traceOptions(), ) + if err := refuseUnbuildableSources(unbuildable); err != nil { + return nil, err + } req := eval_api.NewDataGenerationJobRequest(plan.Name, plan.Model, plan.SampleSize, sources) job, err := ec.evalClient.CreateDataGenerationJob(ctx, req, DataGenerationAPIVersion) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_commands.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_commands.go index b5f16b1c600..a401d9db1fb 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_commands.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_commands.go @@ -153,23 +153,25 @@ func newDatasetGenerateCommand() *cobra.Command { var ( flags generateFlags maxSamples int - from string + from []string ) cmd := &cobra.Command{ Use: "generate ", Short: "Generate a dataset and download it.", Long: "Generate a dataset and download it.\n\n" + - "--from selects one of the four sources the service accepts. " + - "Generating from the agent's own definition is a preference rather " + - "than a fallback: it covers cases no user has hit yet, and it can " + - "supply reference answers, which a transcript cannot.", + "--from selects one or more of the sources the service accepts, and " + + "is repeatable. Generating from the agent's own definition is a " + + "preference rather than a fallback: it covers cases no user has hit " + + "yet, and it can supply reference answers, which a transcript cannot.", Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { name := args[0] - if err := project.ValidateGenerateSource(from); err != nil { - return err + for _, src := range from { + if err := project.ValidateGenerateSource(src); err != nil { + return err + } } if err := project.ValidateSampleSize(maxSamples); err != nil { return err @@ -197,6 +199,12 @@ func newDatasetGenerateCommand() *cobra.Command { } defer ec.Close() + if len(plan.From) == 0 { + plan.From = defaultGenerationSource( + ec.getEnvValue(cmd.Context(), appInsightsEnvKey), + ) + } + ref, err := ec.generateDataset(cmd.Context(), plan, cmd.OutOrStdout(), flags.noWait) if err != nil { return err @@ -211,10 +219,12 @@ func newDatasetGenerateCommand() *cobra.Command { cmd.Flags().IntVar(&maxSamples, "max-samples", 0, fmt.Sprintf("Rows to synthesize (%d-%d). Defaults to %d.", project.MinSampleSize, project.MaxSampleSize, project.DefaultSampleSize)) - cmd.Flags().StringVar(&from, "from", "", - fmt.Sprintf("Where rows come from: %s. Defaults to traces when the project "+ - "has Application Insights connected, otherwise agent.", - strings.Join(project.GenerateSources, ", "))) + cmd.Flags().StringSliceVar(&from, "from", nil, + fmt.Sprintf("Where rows come from: %s. Repeatable, and the service accepts "+ + "more than one. Defaults to %s when the project has Application Insights "+ + "connected, otherwise %s.", + strings.Join(project.GenerateSources, ", "), + project.GenerateFromTraces, project.GenerateFromAgent)) addGenerateFlags(cmd, &flags) return cmd } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_sources_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_sources_test.go new file mode 100644 index 00000000000..4165df62754 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_sources_test.go @@ -0,0 +1,59 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// The spec's default: traces when the project has Application Insights +// connected, otherwise the agent. The connection string is how a project says +// it collects traces at all, so asking for traces without one would submit a +// billed job against nothing. +func TestDefaultGenerationSource(t *testing.T) { + assert.Equal(t, []string{"traces"}, + defaultGenerationSource("InstrumentationKey=00000000-0000-0000-0000-000000000000"), + "a project collecting traces should be generated from them") + + assert.Equal(t, []string{"agent"}, defaultGenerationSource(""), + "with nowhere for traces to have been collected, the agent is all there is") +} + +// --from is a request, and one the plan cannot honour has to stop the command +// rather than quietly submit a job seeded from less than was asked for. +func TestRefuseUnbuildableSources(t *testing.T) { + assert.NoError(t, refuseUnbuildableSources(nil)) + assert.NoError(t, refuseUnbuildableSources([]string{})) + + tests := []struct { + kind string + says string + }{ + {"prompt", "--agent-instruction"}, + {"agent", "--target"}, + {"file", "azd ai dataset create"}, + } + + for _, tt := range tests { + t.Run(tt.kind, func(t *testing.T) { + err := refuseUnbuildableSources([]string{tt.kind}) + require.Error(t, err) + assert.Contains(t, err.Error(), tt.says, + "the error has to name the way out, not just the problem") + }) + } +} + +// Two unhonoured sources are two things the caller has to fix, so both are +// reported at once rather than one per attempt. +func TestRefuseUnbuildableSources_ReportsAllOfThemAtOnce(t *testing.T) { + err := refuseUnbuildableSources([]string{"prompt", "agent"}) + + require.Error(t, err) + assert.Contains(t, err.Error(), "--agent-instruction") + assert.Contains(t, err.Error(), "--target") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/surface_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/surface_test.go index 2d266ead1e6..1d6b8e969c3 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/surface_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/surface_test.go @@ -11,6 +11,8 @@ import ( "strings" "testing" + "azureaieval/internal/project" + "github.com/spf13/cobra" "github.com/spf13/pflag" "github.com/stretchr/testify/assert" @@ -209,6 +211,36 @@ func TestServiceCommandsTakeProjectEndpoint(t *testing.T) { }) } +// The spec says --from "selects one or more of the four sources", so it has to +// be repeatable. Declared as a plain string it would still accept every +// documented single-source invocation and silently keep only the last of a +// repeated one, which is the kind of difference no example in the spec shows. +func TestDatasetGenerateFromTakesMoreThanOneSource(t *testing.T) { + flag := find(t, "dataset generate").Flags().Lookup("from") + require.NotNil(t, flag, "dataset generate must offer --from") + + assert.Equal(t, "stringSlice", flag.Value.Type(), + "--from selects one or more sources, so it cannot be a single string") +} + +// `--from` names sources; the set it accepts is the set the service has a path +// for, and the help has to list exactly that set. +func TestDatasetGenerateFromListsEverySource(t *testing.T) { + usage := find(t, "dataset generate").Flags().Lookup("from").Usage + + for _, source := range project.GenerateSources { + assert.Containsf(t, usage, source, + "--from accepts %q, so its help has to say so", source) + } +} + +// `--from` is the only place a source is named, so `evaluator generate`, which +// has no such flag, must not be left half-wired to one. +func TestEvaluatorGenerateHasNoFromFlag(t *testing.T) { + assert.Nil(t, find(t, "evaluator generate").Flags().Lookup("from"), + "the spec gives --from to dataset generate only") +} + // find resolves a command path, failing the test when it does not exist. func find(t *testing.T, path string) *cobra.Command { t.Helper() diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/generation.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/generation.go index 26727677270..235f58884a0 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/generation.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/generation.go @@ -20,10 +20,6 @@ type TraceOptions struct { Days int } -// BuildGenerationSources constructs the sources array for generation jobs. -// A prompt source is included when instruction is non-empty, along with the -// agent source. When traces is non-nil and Days > 0, a traces source is -// appended with start_time computed from the current time. // WithoutAgentSource returns the sources with the agent entry removed. // // Agent-seeded data generation currently fails server-side for every agent, @@ -50,35 +46,94 @@ func HasPromptSource(sources []GenerationSource) bool { return false } -func BuildGenerationSources(agentKind, agentName, version, instruction string, traces *TraceOptions) []GenerationSource { - var sources []GenerationSource +// BuildGenerationSources emits the sources the caller selected, in a stable +// order, along with the ones it asked for and nothing could be built from. +// +// kinds is what --from named. An empty kinds means "whatever this plan has to +// offer" and reports nothing missing: the caller expressed no preference, so +// there is nothing to disappoint. Naming a kind explicitly is a request, and a +// request that cannot be built is worth saying out loud rather than quietly +// submitting a job seeded from less than was asked for. +func BuildGenerationSources( + kinds []string, + agentName, version, instruction string, + traces *TraceOptions, +) (sources []GenerationSource, unbuildable []string) { + want := map[string]bool{} + for _, k := range kinds { + want[k] = true + } + // Empty kinds selects everything available; a populated one selects only + // what it names. + selected := func(kind string) bool { + return len(want) == 0 || want[kind] + } + // asked distinguishes "the default swept this up" from "the user typed it", + // which is what decides whether an empty-handed source is an error. + asked := func(kind string) bool { return want[kind] } + + // The agent is settled first because whether it was built decides whether + // its instructions have anything to be the instructions of. + var agentSource *GenerationSource + if selected("agent") { + switch { + case agentName != "": + agentSource = &GenerationSource{Type: "agent", AgentName: agentName} + if version != "" { + agentSource.AgentVersion = version + } + case asked("agent"): + unbuildable = append(unbuildable, "agent") + } + } - if instruction != "" { - sources = append(sources, GenerationSource{ - Type: "prompt", - Prompt: instruction, - }) + // Generating from an agent means generating from its instructions, so they + // travel with it as a prompt. That is also the only shape the service + // currently honours: the agent source alone fails for every agent, and the + // prompt is what the retry in generateDataset falls back to. Without this, + // `--from agent` would be a request that always fails. + promptCarriesTheAgent := agentSource != nil && asked("agent") + if selected("prompt") || promptCarriesTheAgent { + switch { + case instruction != "": + sources = append(sources, GenerationSource{ + Type: "prompt", + Prompt: instruction, + }) + case asked("prompt"): + unbuildable = append(unbuildable, "prompt") + } } - agentSource := GenerationSource{ - Type: "agent", - AgentName: agentName, + if agentSource != nil { + sources = append(sources, *agentSource) } - if version != "" { - agentSource.AgentVersion = version + + if selected("traces") { + // A window narrows the request; it does not authorize it. Asking for + // traces without one means every trace the agent has. + switch { + case traces != nil && traces.Days > 0: + sources = append(sources, GenerationSource{ + Type: "traces", + AgentName: agentName, + StartTime: time.Now().AddDate(0, 0, -traces.Days).Unix(), + }) + case asked("traces"): + sources = append(sources, GenerationSource{ + Type: "traces", + AgentName: agentName, + }) + } } - sources = append(sources, agentSource) - - if traces != nil && traces.Days > 0 { - startTime := time.Now().AddDate(0, 0, -traces.Days).Unix() - sources = append(sources, GenerationSource{ - Type: "traces", - AgentName: agentName, - StartTime: startTime, - }) + + // The service takes a file's rows through the dataset upload path, not + // through a generation source, so there is nothing here to build one from. + if asked("file") { + unbuildable = append(unbuildable, "file") } - return sources + return sources, unbuildable } // --------------------------------------------------------------------------- diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/generation_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/generation_test.go new file mode 100644 index 00000000000..85183400040 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/generation_test.go @@ -0,0 +1,190 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package eval_api + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// kindsOf reduces the built sources to what --from talks about, which is the +// only part these tests are asserting on. +func kindsOf(sources []GenerationSource) []string { + kinds := make([]string, 0, len(sources)) + for _, s := range sources { + kinds = append(kinds, s.Type) + } + return kinds +} + +// Naming a source is a request to send that one, not a hint. Everything the +// plan could otherwise have offered stays out of the request. +func TestBuildGenerationSources_SendsOnlyWhatFromNamed(t *testing.T) { + sources, unbuildable := BuildGenerationSources( + []string{"traces"}, + "support-agent", "3", "answer support questions", + &TraceOptions{Days: 7}, + ) + + assert.Equal(t, []string{"traces"}, kindsOf(sources)) + assert.Empty(t, unbuildable) +} + +// Generating from an agent means generating from its instructions, so asking +// for the agent carries them. It is also the only shape the service honours: +// the agent source on its own fails for every agent, so a `--from agent` that +// dropped the prompt would be a request that always fails. +func TestBuildGenerationSources_AgentCarriesItsInstructions(t *testing.T) { + sources, unbuildable := BuildGenerationSources( + []string{"agent"}, "support-agent", "3", "answer support questions", nil, + ) + + assert.Equal(t, []string{"prompt", "agent"}, kindsOf(sources)) + assert.Equal(t, "answer support questions", sources[0].Prompt) + assert.Empty(t, unbuildable) +} + +// The instructions ride along with the agent; they do not stand in for it. An +// agent nobody named is still nothing to generate from. +func TestBuildGenerationSources_InstructionsDoNotSubstituteForTheAgent(t *testing.T) { + sources, unbuildable := BuildGenerationSources( + []string{"agent"}, "", "", "answer support questions", nil, + ) + + assert.Empty(t, sources) + assert.Equal(t, []string{"agent"}, unbuildable) +} + +// The agent name travels with the traces source: it is what scopes the query +// to this agent's conversations rather than the whole project's. +func TestBuildGenerationSources_TracesCarryTheAgent(t *testing.T) { + sources, _ := BuildGenerationSources( + []string{"traces"}, "support-agent", "", "", &TraceOptions{Days: 7}, + ) + + require.Len(t, sources, 1) + assert.Equal(t, "support-agent", sources[0].AgentName) +} + +// A day window narrows the trace query; it is not what authorizes it. The +// documented `dataset generate --from traces` carries no window, and it +// has to mean "every trace" rather than "no traces". +func TestBuildGenerationSources_TracesWithoutAWindowAreUnbounded(t *testing.T) { + sources, unbuildable := BuildGenerationSources( + []string{"traces"}, "support-agent", "", "", nil, + ) + + require.Len(t, sources, 1) + assert.Equal(t, "traces", sources[0].Type) + assert.Zero(t, sources[0].StartTime, + "an absent window must leave start_time off the wire, not pin it to now") + assert.Empty(t, unbuildable) +} + +func TestBuildGenerationSources_TraceWindowBecomesAStartTime(t *testing.T) { + sources, _ := BuildGenerationSources( + []string{"traces"}, "support-agent", "", "", &TraceOptions{Days: 7}, + ) + + require.Len(t, sources, 1) + want := time.Now().AddDate(0, 0, -7).Unix() + assert.InDelta(t, want, sources[0].StartTime, 60) +} + +// No --from is no preference, so the plan sends everything it happens to have. +func TestBuildGenerationSources_EmptyFromSendsWhatThePlanHas(t *testing.T) { + sources, unbuildable := BuildGenerationSources( + nil, "support-agent", "3", "answer support questions", &TraceOptions{Days: 7}, + ) + + assert.Equal(t, []string{"prompt", "agent", "traces"}, kindsOf(sources)) + assert.Empty(t, unbuildable) +} + +// Expressing no preference cannot disappoint one, so an empty --from reports +// nothing missing however little the plan turns out to hold. +func TestBuildGenerationSources_EmptyFromNeverReportsMissingSources(t *testing.T) { + sources, unbuildable := BuildGenerationSources(nil, "", "", "", nil) + + assert.Empty(t, sources) + assert.Empty(t, unbuildable) +} + +// Asking for a source the plan cannot build has to surface, because the job is +// billed and what comes back looks the same either way. +func TestBuildGenerationSources_ReportsWhatItCouldNotBuild(t *testing.T) { + tests := []struct { + name string + kinds []string + agentName string + instruction string + want []string + }{ + { + name: "prompt without an instruction", + kinds: []string{"prompt"}, + want: []string{"prompt"}, + }, + { + name: "agent without a target", + kinds: []string{"agent"}, + want: []string{"agent"}, + }, + { + name: "file is not a generation source at all", + kinds: []string{"file"}, + want: []string{"file"}, + }, + { + name: "several at once", + kinds: []string{"prompt", "agent"}, + want: []string{"agent", "prompt"}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + sources, unbuildable := BuildGenerationSources( + tt.kinds, tt.agentName, "", tt.instruction, nil, + ) + + assert.Empty(t, sources) + assert.Equal(t, tt.want, unbuildable) + }) + } +} + +// A request that names two sources and can only build one still reports the +// one it could not, rather than being satisfied by the other's success. +func TestBuildGenerationSources_OneBuiltSourceDoesNotExcuseAMissingOne(t *testing.T) { + sources, unbuildable := BuildGenerationSources( + []string{"agent", "prompt"}, "support-agent", "", "", nil, + ) + + assert.Equal(t, []string{"agent"}, kindsOf(sources)) + assert.Equal(t, []string{"prompt"}, unbuildable) +} + +// `file` is only unbuildable when it was asked for. The default sweep must not +// invent a complaint about a source nobody named. +func TestBuildGenerationSources_FileIsOnlyReportedWhenAskedFor(t *testing.T) { + _, unbuildable := BuildGenerationSources( + nil, "support-agent", "", "instruction", &TraceOptions{Days: 7}, + ) + + assert.Empty(t, unbuildable) +} + +func TestBuildGenerationSources_AgentVersionIsOptional(t *testing.T) { + withVersion, _ := BuildGenerationSources([]string{"agent"}, "support-agent", "3", "", nil) + require.Len(t, withVersion, 1) + assert.Equal(t, "3", withVersion[0].AgentVersion) + + withoutVersion, _ := BuildGenerationSources([]string{"agent"}, "support-agent", "", "", nil) + require.Len(t, withoutVersion, 1) + assert.Empty(t, withoutVersion[0].AgentVersion) +} From 2fb79a7a04311996ecfea833d4c290161a5aa6d7 Mon Sep 17 00:00:00 2001 From: mohessie Date: Thu, 6 Aug 2026 03:02:24 +0300 Subject: [PATCH 108/320] Point CI at templates that exist, and verify the provider it declares The release pipeline listed eng/pipelines/release-azd-extension.yml, which is not a file -- the template lives under templates/stages/. A trigger path that matches nothing fails silently: the pipeline simply never runs for changes to the template it is built out of. It now names the same paths as the skills sibling, and every one of them resolves. go.mod was in the trigger list too, matching the repo root module rather than the extension's own. The lint workflow only ran lint. This extension declares a service-target provider, and the extensions that declare one run verify-ext-providers to check the manifest against what the host actually registers. It has the canonical providers_manifest_test.go already; nothing was calling it in CI. --- .../lint-ext-azure-ai-evaluations.yml | 28 +++++++++++++++++++ .../release-ext-azure-ai-evaluations.yml | 5 ++-- 2 files changed, 30 insertions(+), 3 deletions(-) create mode 100644 .github/workflows/lint-ext-azure-ai-evaluations.yml diff --git a/.github/workflows/lint-ext-azure-ai-evaluations.yml b/.github/workflows/lint-ext-azure-ai-evaluations.yml new file mode 100644 index 00000000000..2cb72ea1de4 --- /dev/null +++ b/.github/workflows/lint-ext-azure-ai-evaluations.yml @@ -0,0 +1,28 @@ +name: ext-azure-ai-evaluations-ci + +on: + pull_request: + paths: + - "cli/azd/extensions/azure.ai.evaluations/**" + - ".github/workflows/lint-ext-azure-ai-evaluations.yml" + - ".github/workflows/verify-ext-providers.yml" + branches: [main] + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number }} + cancel-in-progress: true + +permissions: + contents: read + pull-requests: write # required by reusable workflow lint-go.yml + +jobs: + lint: + uses: ./.github/workflows/lint-go.yml + with: + working-directory: cli/azd/extensions/azure.ai.evaluations + + verify-providers: + uses: ./.github/workflows/verify-ext-providers.yml + with: + working-directory: cli/azd/extensions/azure.ai.evaluations diff --git a/eng/pipelines/release-ext-azure-ai-evaluations.yml b/eng/pipelines/release-ext-azure-ai-evaluations.yml index d8b60935909..329c4a2181e 100644 --- a/eng/pipelines/release-ext-azure-ai-evaluations.yml +++ b/eng/pipelines/release-ext-azure-ai-evaluations.yml @@ -5,9 +5,8 @@ trigger: - main paths: include: - - go.mod - cli/azd/extensions/azure.ai.evaluations - - eng/pipelines/release-azd-extension.yml + - /eng/pipelines/templates/stages/release-azd-extension.yml - /eng/pipelines/templates/jobs/build-azd-extension.yml - /eng/pipelines/templates/jobs/cross-build-azd-extension.yml - /eng/pipelines/templates/variables/image.yml @@ -17,7 +16,7 @@ pr: include: - cli/azd/extensions/azure.ai.evaluations - eng/pipelines/release-ext-azure-ai-evaluations.yml - - eng/pipelines/release-azd-extension.yml + - /eng/pipelines/templates/stages/release-azd-extension.yml - eng/pipelines/templates/steps/publish-cli.yml exclude: - cli/azd/docs/** From a0bb3e0ad4964eb207bc5fb85213a4c458843560 Mon Sep 17 00:00:00 2001 From: mohessie Date: Thu, 6 Aug 2026 03:15:10 +0300 Subject: [PATCH 109/320] Read the agent's instructions from the project before asking the service The spec has --from agent generating from the target agent's instructions, read locally so the request needs no service lookup. Checking that against the agents extension rather than the spec text found half of it true. The optimize configuration folder is real: zd ai agent optimize writes /.agent_configs/baseline/metadata.yaml with an instruction_file pointer beside the instructions.md it names. That is now read first. The other half is not. A managed agent's definition YAML has no instructions field at all -- ContainerAgent carries kind, name, runtime, image, protocols and policies, and nothing that holds a prompt. Instructions live on skills and on prompt agents, and prompt agents are not deployable today. So there is nothing to read there, and the service lookup stays as the fallback. Local wins over the service because instructions that have been optimized but not yet deployed are the ones the author means; generating against what is still published would test the version they are replacing. A target matches either the azure.yaml service key or the agent name the service declares, since the two need not agree and a user has only seen one of them. Matching both is what makes --target mean what they typed. A tie is refused rather than broken arbitrarily, which is the spec's ambiguity rule. The file names are the agents extension's, repeated rather than imported -- azd extensions are separate modules that share no code, so reading another one's output means knowing its layout. The comment says so, because if those names move this quietly stops working and falls back to the service. --- .../internal/cmd/generate.go | 43 ++++- .../internal/project/agent_instructions.go | 151 +++++++++++++++ .../project/agent_instructions_test.go | 182 ++++++++++++++++++ 3 files changed, 374 insertions(+), 2 deletions(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/project/agent_instructions.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/project/agent_instructions_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go index 3091cbd2f27..ee07d39308e 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go @@ -16,6 +16,8 @@ import ( "azureaieval/internal/pkg/eval_api" "azureaieval/internal/project" + + "github.com/azure/azure-dev/cli/azd/pkg/azdext" ) // generatePollBudget replaces the inherited 2s x 300 (10 minute) client budget. @@ -110,8 +112,13 @@ func declaredInstructions(named, configPath string) (string, error) { // // The service accepts an agent source that is meant to pull the agent's own // instructions, but it fails for every agent, so the agent's context is read -// here instead. In precedence order: what the caller passed, then the agent's -// published instructions. +// here instead. In precedence order: what the caller passed, the instructions +// the project already holds, then the agent's published ones. +// +// The project comes before the service because a local read cannot fail +// slowly, and because instructions that have been optimized but not yet +// deployed are the ones the author means — generating against what is still +// published would test the version they are replacing. // // The last step is what makes `generate` work with no authored input at all, // which is the flow `init` sets up. @@ -128,6 +135,18 @@ func (ec *evalContext) resolveGenerationInstruction( if agentName == "" { return "", nil } + + local, path, err := ec.agentInstructionsFromProject(ctx, agentName) + if err != nil { + return "", err + } + if local != "" { + if !quiet { + fmt.Fprintf(out, " Seeding generation from %s.\n", filepath.ToSlash(path)) + } + return local, nil + } + agent, err := ec.evalClient.GetAgent(ctx, agentName, ProjectEndpointAPIVersion) if err != nil { // Generation can still proceed from the agent source alone, so a @@ -145,6 +164,26 @@ func (ec *evalContext) resolveGenerationInstruction( return instructions, nil } +// agentInstructionsFromProject reads the agent's instructions out of the azd +// project, coming back empty when there is no project to read. +// +// Running outside a project is ordinary — the atomic commands work standalone +// against the data plane — so not finding one is not an error. An ambiguous +// target inside one is, because it would otherwise pick an agent at random. +func (ec *evalContext) agentInstructionsFromProject( + ctx context.Context, + agentName string, +) (instruction string, path string, err error) { + if ec.azdClient == nil { + return "", "", nil + } + resp, err := ec.azdClient.Project().Get(ctx, &azdext.EmptyRequest{}) + if err != nil || resp.GetProject() == nil { + return "", "", nil + } + return project.AgentInstructionsFromProject(resp.GetProject(), agentName) +} + // generateRubric submits the evaluator generation job and saves the rubric. func (ec *evalContext) generateRubric( ctx context.Context, diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/agent_instructions.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/agent_instructions.go new file mode 100644 index 00000000000..d191fd184c2 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/agent_instructions.go @@ -0,0 +1,151 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package project + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "sort" + "strings" + + "github.com/azure/azure-dev/cli/azd/pkg/azdext" + "go.yaml.in/yaml/v3" +) + +// AgentHost is the service host the agents extension registers. Services +// declaring it are the ones that could be a generation target. +const AgentHost = "azure.ai.agent" + +// Where `azd ai agent optimize` leaves the configuration it settled on. +// +// These are the agents extension's file names, repeated rather than imported: +// azd extensions are separate Go modules and share no code, so the only way to +// read another one's output is to know its layout. That makes this a coupling +// worth naming — if the agents extension moves these, generation quietly stops +// finding instructions locally and falls back to the service. +const ( + agentConfigsDir = ".agent_configs" + agentBaselineDir = "baseline" + agentMetadataFile = "metadata.yaml" +) + +// agentConfigMetadata is the part of the optimize configuration's metadata.yaml +// that says where the instructions are. It points at a file rather than +// carrying the text, because the text is what a reviewer diffs. +type agentConfigMetadata struct { + InstructionFile string `yaml:"instruction_file"` +} + +// ErrAmbiguousAgentService reports that a target name matched more than one +// service, so there is no single set of instructions to read. +var ErrAmbiguousAgentService = errors.New("more than one agent service matches") + +// AgentInstructionsFromProject reads the target agent's instructions out of the +// project, returning empty when the project does not hold them. +// +// The instructions an agent was optimized with are the best description of what +// it is supposed to do, and they are already on disk, so generating from them +// needs no service call. Coming back empty is ordinary — most projects have +// never run `azd ai agent optimize` — and leaves the caller free to ask the +// service instead. +// +// The returned path is where the text came from, for a caller that wants to say +// so. +func AgentInstructionsFromProject( + proj *azdext.ProjectConfig, + agentName string, +) (instruction string, path string, err error) { + svc, err := findAgentService(proj, agentName) + if err != nil || svc == nil { + return "", "", err + } + + configDir := filepath.Join( + proj.GetPath(), serviceRelativeDir(svc), agentConfigsDir, agentBaselineDir) + + data, err := os.ReadFile(filepath.Join(configDir, agentMetadataFile)) //nolint:gosec // under the project + if err != nil { + // An agent that was never optimized has no such directory, which is + // the common case rather than a problem. + return "", "", nil + } + + var meta agentConfigMetadata + if err := yaml.Unmarshal(data, &meta); err != nil { + return "", "", fmt.Errorf( + "reading %s: %w", filepath.Join(configDir, agentMetadataFile), err) + } + if meta.InstructionFile == "" { + return "", "", nil + } + + instructionPath := meta.InstructionFile + if !filepath.IsAbs(instructionPath) { + instructionPath = filepath.Join(configDir, instructionPath) + } + text, err := os.ReadFile(instructionPath) //nolint:gosec // named by the metadata beside it + if err != nil { + // The metadata named a file that is not there. That is worth saying: + // something wrote the pointer and not the target. + return "", "", fmt.Errorf( + "%s names instruction_file %q, which could not be read: %w", + filepath.Join(configDir, agentMetadataFile), meta.InstructionFile, err) + } + + return strings.TrimSpace(string(text)), instructionPath, nil +} + +// findAgentService resolves a target name to the one service that is it. +// +// A name can match either the azure.yaml service key or the agent name the +// service declares, because the two need not agree and a user has only ever +// seen one of them. Matching both is what makes `--target` mean what they +// typed; refusing a tie is what stops it silently meaning one of two things. +func findAgentService( + proj *azdext.ProjectConfig, + agentName string, +) (*azdext.ServiceConfig, error) { + if proj == nil || agentName == "" { + return nil, nil + } + + var matched []string + services := map[string]*azdext.ServiceConfig{} + for name, svc := range proj.GetServices() { + if svc.GetHost() != AgentHost { + continue + } + if name == agentName || declaredAgentName(svc) == agentName { + matched = append(matched, name) + services[name] = svc + } + } + + switch len(matched) { + case 0: + return nil, nil + case 1: + return services[matched[0]], nil + default: + sort.Strings(matched) + return nil, fmt.Errorf( + "%w %q: %s. Name one of them with --target, or pass the text with "+ + "--agent-instruction", + ErrAmbiguousAgentService, agentName, strings.Join(matched, ", ")) + } +} + +// declaredAgentName is the name the service gives the agent, which is what the +// service publishes under and so what the eval configuration's target refers +// to. It is absent when the service key is also the agent name. +func declaredAgentName(svc *azdext.ServiceConfig) string { + props := serviceProps(svc) + if props == nil { + return "" + } + name, _ := props.AsMap()["name"].(string) + return name +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/agent_instructions_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/agent_instructions_test.go new file mode 100644 index 00000000000..dae36a18c5b --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/agent_instructions_test.go @@ -0,0 +1,182 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package project + +import ( + "os" + "path/filepath" + "testing" + + "github.com/azure/azure-dev/cli/azd/pkg/azdext" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/types/known/structpb" +) + +// writeOptimizeConfig lays out what `azd ai agent optimize` leaves behind: +// .agent_configs/baseline/metadata.yaml pointing at instructions.md beside it. +func writeOptimizeConfig(t *testing.T, serviceDir, metadata, instructions string) { + t.Helper() + dir := filepath.Join(serviceDir, ".agent_configs", "baseline") + require.NoError(t, os.MkdirAll(dir, 0o750)) + require.NoError(t, os.WriteFile(filepath.Join(dir, "metadata.yaml"), []byte(metadata), 0o600)) + if instructions != "" { + require.NoError(t, + os.WriteFile(filepath.Join(dir, "instructions.md"), []byte(instructions), 0o600)) + } +} + +// agentService builds a project holding one agent service, optionally +// declaring an agent name that differs from the service key. +func agentService(t *testing.T, root, serviceKey, declaredName string) *azdext.ProjectConfig { + t.Helper() + svc := &azdext.ServiceConfig{ + Name: serviceKey, + Host: AgentHost, + RelativePath: serviceKey, + } + if declaredName != "" { + props, err := structpb.NewStruct(map[string]any{"name": declaredName}) + require.NoError(t, err) + svc.AdditionalProperties = props + } + return &azdext.ProjectConfig{ + Path: root, + Services: map[string]*azdext.ServiceConfig{serviceKey: svc}, + } +} + +// The instructions an agent was optimized with are already on disk, so +// generating from them needs no service call. +func TestAgentInstructionsFromProject_ReadsTheOptimizeConfig(t *testing.T) { + root := t.TempDir() + writeOptimizeConfig(t, + filepath.Join(root, "support"), + "name: support\ninstruction_file: instructions.md\n", + "Answer support questions politely.\n") + + instruction, path, err := AgentInstructionsFromProject( + agentService(t, root, "support", ""), "support") + + require.NoError(t, err) + assert.Equal(t, "Answer support questions politely.", instruction) + assert.Equal(t, filepath.Join(root, "support", ".agent_configs", "baseline", "instructions.md"), + path) +} + +// A target names the agent, which need not be spelled the way the azure.yaml +// key is. A user has only ever seen one of the two. +func TestAgentInstructionsFromProject_MatchesTheDeclaredAgentName(t *testing.T) { + root := t.TempDir() + writeOptimizeConfig(t, + filepath.Join(root, "svc"), + "instruction_file: instructions.md\n", + "Be helpful.") + + instruction, _, err := AgentInstructionsFromProject( + agentService(t, root, "svc", "support-agent"), "support-agent") + + require.NoError(t, err) + assert.Equal(t, "Be helpful.", instruction) +} + +// Most projects have never run optimize, so finding nothing is the ordinary +// case and has to leave the caller free to ask the service instead. +func TestAgentInstructionsFromProject_SilentWhenThereIsNothingToRead(t *testing.T) { + root := t.TempDir() + + tests := []struct { + name string + proj *azdext.ProjectConfig + agent string + }{ + {"no project at all", nil, "support"}, + {"no agent named", agentService(t, root, "support", ""), ""}, + {"no service by that name", agentService(t, root, "support", ""), "other"}, + {"no optimize config on disk", agentService(t, root, "support", ""), "support"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + instruction, path, err := AgentInstructionsFromProject(tt.proj, tt.agent) + + assert.NoError(t, err) + assert.Empty(t, instruction) + assert.Empty(t, path) + }) + } +} + +// A service that is not an agent is not a candidate, however it is named. +func TestAgentInstructionsFromProject_IgnoresServicesThatAreNotAgents(t *testing.T) { + root := t.TempDir() + writeOptimizeConfig(t, filepath.Join(root, "support"), + "instruction_file: instructions.md\n", "Be helpful.") + + proj := agentService(t, root, "support", "") + proj.Services["support"].Host = "containerapp" + + instruction, _, err := AgentInstructionsFromProject(proj, "support") + + assert.NoError(t, err) + assert.Empty(t, instruction) +} + +// Two services answering to one name is a tie, and picking either would make +// the generated dataset describe an agent the caller did not mean. +func TestAgentInstructionsFromProject_RefusesAnAmbiguousTarget(t *testing.T) { + root := t.TempDir() + proj := agentService(t, root, "support", "") + props, err := structpb.NewStruct(map[string]any{"name": "support"}) + require.NoError(t, err) + proj.Services["helpdesk"] = &azdext.ServiceConfig{ + Name: "helpdesk", Host: AgentHost, RelativePath: "helpdesk", + AdditionalProperties: props, + } + + _, _, err = AgentInstructionsFromProject(proj, "support") + + require.ErrorIs(t, err, ErrAmbiguousAgentService) + assert.Contains(t, err.Error(), "helpdesk") + assert.Contains(t, err.Error(), "support") + assert.Contains(t, err.Error(), "--target", + "an ambiguity the caller can resolve has to say how") +} + +// A pointer with nothing behind it means something wrote half the config. +// Falling back silently would generate from the published agent while the +// author believes they are generating from what they just optimized. +func TestAgentInstructionsFromProject_ReportsADanglingInstructionFile(t *testing.T) { + root := t.TempDir() + writeOptimizeConfig(t, filepath.Join(root, "support"), + "instruction_file: instructions.md\n", "") + + _, _, err := AgentInstructionsFromProject(agentService(t, root, "support", ""), "support") + + require.Error(t, err) + assert.Contains(t, err.Error(), "instructions.md") +} + +// Metadata that names no instruction file is a config without instructions, +// not a broken one. +func TestAgentInstructionsFromProject_NoInstructionFileIsNotAnError(t *testing.T) { + root := t.TempDir() + writeOptimizeConfig(t, filepath.Join(root, "support"), "name: support\n", "") + + instruction, _, err := AgentInstructionsFromProject( + agentService(t, root, "support", ""), "support") + + assert.NoError(t, err) + assert.Empty(t, instruction) +} + +func TestAgentInstructionsFromProject_ReportsUnreadableMetadata(t *testing.T) { + root := t.TempDir() + writeOptimizeConfig(t, filepath.Join(root, "support"), "\tnot: [valid\n", "") + + _, _, err := AgentInstructionsFromProject(agentService(t, root, "support", ""), "support") + + require.Error(t, err) + assert.Contains(t, err.Error(), "metadata.yaml") +} From 2d4ce006c1091f540074f98030575b2a4eaa0047 Mon Sep 17 00:00:00 2001 From: mohessie Date: Thu, 6 Aug 2026 03:28:57 +0300 Subject: [PATCH 110/320] Test the export writers and portal URLs, and delete what duplicated them Coverage was 39.2%, and the uncovered list turned out to be two different things: logic nobody had tested, and logic nobody had called. Tested, because these are what a user reads and nothing checked them: - writeResultsCSV and writeResultsJSONL. The export default moved to csv and jsonl was added earlier in this branch with no test of either writer. The header is a contract with whatever opens the file, a run that graded nothing still has to produce a parseable artifact rather than zero bytes, and a comma in a service-supplied criterion name has to survive the round trip. - The portal URLs. They are assembled from parts rather than returned by the service, so nothing but a test says whether they land anywhere. The subscription is base64url-encoded, which is now pinned -- padding or a plain slash would break the URL segment. - IsDatasetName, which decides whether a value is looked up in the service or opened off disk. Getting it wrong sends a path to the registry, and neither failure names the real problem. Deleted, because each duplicated something live: - GenerationJob.NormalizedStatus duplicated ParseJobStatus and got it wrong: its doc says it lowercases and it does not. Nothing called it, so the bug never fired -- but the next caller would have found it. - GenerationJob.OperationID and OpenAIEval.ResolvedID were aliases. - ParseEvaluatorResult duplicated what writeRubric already does inline. - formatStat duplicated the Defined() check in run.go. - noPrompt duplicated what requireFlag already encodes. - SetFileID could not be used: readRegisteredDataset documents that a run's file_id means an uploaded file and a dataset name is rejected as one. That reason now sits on SetFileContent, where the next person will look. Testing dead code would have entrenched it, and in NormalizedStatus's case would have pinned the wrong behaviour. --- .../internal/cmd/output.go | 12 -- .../internal/cmd/run_output.go | 10 -- .../internal/cmd/run_output_write_test.go | 148 ++++++++++++++++ .../internal/pkg/eval_api/models.go | 52 +----- .../internal/pkg/eval_api/portal_urls_test.go | 166 ++++++++++++++++++ 5 files changed, 320 insertions(+), 68 deletions(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output_write_test.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/portal_urls_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/output.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/output.go index 62f9c2eade1..143b49e91ba 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/output.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/output.go @@ -79,18 +79,6 @@ func emitTable(w io.Writer, headers []string, rows [][]string) error { return tw.Flush() } -// noPrompt reports whether the command must run without any interaction. -func noPrompt(cmd *cobra.Command) bool { - if cmd == nil { - return false - } - v, err := cmd.Flags().GetBool("no-prompt") - if err != nil { - return false - } - return v -} - // requireFlag returns an error naming the missing flag, used when --no-prompt // prevents asking for a required value. func requireFlag(name string) error { diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output.go index 3c6c0e36ad7..28f2170909b 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output.go @@ -35,16 +35,6 @@ func newRunOutputCommand() *cobra.Command { return cmd } -// formatStat renders a statistic, showing an undefined one as a dash. A score -// the service left unset has no value, and printing the literal "NaN" in a -// results table reads like a failure rather than the arithmetic it is. -func formatStat(verb string, v eval_api.LenientFloat) string { - if !v.Defined() { - return "-" - } - return fmt.Sprintf(verb, float64(v)) -} - func newRunOutputListCommand() *cobra.Command { var ( failedOnly bool diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output_write_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output_write_test.go new file mode 100644 index 00000000000..ec63f399695 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output_write_test.go @@ -0,0 +1,148 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "bytes" + "encoding/csv" + "encoding/json" + "strings" + "testing" + + "azureaieval/internal/pkg/eval_api" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// twoCriteriaRun is a finished run with the shape export has to preserve: one +// row per testing criterion, all carrying the run they belong to. +func twoCriteriaRun() *eval_api.OpenAIEvalRun { + return &eval_api.OpenAIEvalRun{ + ID: "evalrun_abc", + Status: "completed", + PerTestingCriteria: []eval_api.EvalRunCriteriaResult{ + {TestingCriteria: "task_adherence", Passed: 8, Failed: 2}, + {TestingCriteria: "coherence", Passed: 10, Failed: 0}, + }, + } +} + +// An export is read by a spreadsheet or a diff, so the header is part of the +// contract: renaming a column silently breaks whatever consumes it. +func TestWriteResultsCSV(t *testing.T) { + var buf bytes.Buffer + require.NoError(t, writeResultsCSV(&buf, twoCriteriaRun())) + + rows, err := csv.NewReader(&buf).ReadAll() + require.NoError(t, err) + + assert.Equal(t, []string{"run_id", "status", "criterion", "passed", "failed"}, rows[0]) + assert.Equal(t, []string{"evalrun_abc", "completed", "task_adherence", "8", "2"}, rows[1]) + assert.Equal(t, []string{"evalrun_abc", "completed", "coherence", "10", "0"}, rows[2]) + assert.Len(t, rows, 3, "one header and one row per criterion") +} + +// A run that graded nothing still has to produce a file with a header, because +// a consumer that gets zero bytes cannot tell an empty run from a failed +// export. +func TestWriteResultsCSV_RunWithNoCriteria(t *testing.T) { + var buf bytes.Buffer + require.NoError(t, writeResultsCSV(&buf, &eval_api.OpenAIEvalRun{ + ID: "evalrun_empty", Status: "failed", + })) + + rows, err := csv.NewReader(&buf).ReadAll() + require.NoError(t, err) + + require.Len(t, rows, 2) + assert.Equal(t, []string{"run_id", "status", "criterion", "passed", "failed"}, rows[0]) + assert.Equal(t, []string{"evalrun_empty", "failed", "", "", ""}, rows[1]) +} + +// A criterion name is service-supplied, so it can hold anything. The writer +// has to quote rather than corrupt the row. +func TestWriteResultsCSV_QuotesASeparatorInTheData(t *testing.T) { + var buf bytes.Buffer + require.NoError(t, writeResultsCSV(&buf, &eval_api.OpenAIEvalRun{ + ID: "evalrun_abc", + Status: "completed", + PerTestingCriteria: []eval_api.EvalRunCriteriaResult{ + {TestingCriteria: `groundedness, strict`, Passed: 1, Failed: 0}, + }, + })) + + rows, err := csv.NewReader(&buf).ReadAll() + require.NoError(t, err) + require.Len(t, rows, 2) + assert.Equal(t, "groundedness, strict", rows[1][2], + "a comma in a criterion name must survive the round trip") +} + +// One criterion per line is what lets a downstream job stream results without +// holding the whole run. +func TestWriteResultsJSONL(t *testing.T) { + var buf bytes.Buffer + require.NoError(t, writeResultsJSONL(&buf, twoCriteriaRun())) + + lines := strings.Split(strings.TrimSpace(buf.String()), "\n") + require.Len(t, lines, 2, "one line per criterion") + + var first map[string]any + require.NoError(t, json.Unmarshal([]byte(lines[0]), &first)) + assert.Equal(t, "evalrun_abc", first["run_id"]) + assert.Equal(t, "completed", first["status"]) + assert.Equal(t, "task_adherence", first["testing_criteria"]) + assert.EqualValues(t, 8, first["passed"]) + assert.EqualValues(t, 2, first["failed"]) + + var second map[string]any + require.NoError(t, json.Unmarshal([]byte(lines[1]), &second)) + assert.Equal(t, "coherence", second["testing_criteria"]) +} + +// Every line has to parse on its own; that is the whole point of the format. +func TestWriteResultsJSONL_EachLineParsesAlone(t *testing.T) { + var buf bytes.Buffer + require.NoError(t, writeResultsJSONL(&buf, twoCriteriaRun())) + + for _, line := range strings.Split(strings.TrimSpace(buf.String()), "\n") { + var row map[string]any + assert.NoErrorf(t, json.Unmarshal([]byte(line), &row), "line is not self-contained: %s", line) + } +} + +func TestWriteResultsJSONL_RunWithNoCriteria(t *testing.T) { + var buf bytes.Buffer + require.NoError(t, writeResultsJSONL(&buf, &eval_api.OpenAIEvalRun{ + ID: "evalrun_empty", Status: "failed", + })) + + lines := strings.Split(strings.TrimSpace(buf.String()), "\n") + require.Len(t, lines, 1) + + var row map[string]any + require.NoError(t, json.Unmarshal([]byte(lines[0]), &row)) + assert.Equal(t, "evalrun_empty", row["run_id"]) + assert.Equal(t, "failed", row["status"]) + assert.NotContains(t, row, "testing_criteria", + "a run that graded nothing must not claim a criterion") +} + +// The three export formats are a documented set. A fourth spelling, or a +// missing one, is a promise broken on either side. +func TestExportFormatsAreTheDocumentedSet(t *testing.T) { + assert.Equal(t, "csv", formatCSV) + assert.Equal(t, "json", formatJSON) + assert.Equal(t, "jsonl", formatJSONL) + + usage := find(t, "run output export").Flags().Lookup("format") + require.NotNil(t, usage) + assert.Equal(t, formatCSV, usage.DefValue, + "results are a table, so the default artifact is the one a spreadsheet opens") + + for _, f := range []string{formatCSV, formatJSON, formatJSONL} { + assert.Containsf(t, usage.Usage, f, "--format accepts %q, so its help has to say so", f) + } +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/models.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/models.go index aa6c4f89ebb..ceed191847a 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/models.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/models.go @@ -91,19 +91,6 @@ type JobError struct { Message string `json:"message,omitempty"` } -// OperationID returns the job's operation identifier. -func (j *GenerationJob) OperationID() string { - return j.ID -} - -// NormalizedStatus returns the lowercase status, defaulting to "running". -func (j *GenerationJob) NormalizedStatus() string { - if j.Status == "" { - return "running" - } - return j.Status -} - // ResolvedNameVersion extracts the name and version from the generation job result. // If name is empty, both return values are empty (caller should treat as no result). // If version is empty, it defaults to "latest". @@ -209,22 +196,6 @@ type EvaluatorDimension struct { AlwaysApplicable bool `json:"always_applicable,omitempty"` } -// ParseEvaluatorResult parses a GenerationJob result into a structured EvaluatorResult. -// Returns nil if the result cannot be parsed. -func ParseEvaluatorResult(result json.RawMessage) *EvaluatorResult { - if len(result) == 0 { - return nil - } - var r EvaluatorResult - if err := json.Unmarshal(result, &r); err != nil { - return nil - } - if len(r.Definition.Dimensions) == 0 { - return nil - } - return &r -} - // --------------------------------------------------------------------------- // Datasets // --------------------------------------------------------------------------- @@ -297,14 +268,6 @@ type OpenAIEval struct { Metadata map[string]string `json:"metadata,omitempty"` } -// ResolvedID returns the eval's ID, falling back to name. -func (e *OpenAIEval) ResolvedID() string { - if e.ID != "" { - return e.ID - } - return e.Name -} - // OpenAIEvalList is the response for listing OpenAI eval definitions. type OpenAIEvalList struct { Data []OpenAIEval `json:"data"` @@ -408,7 +371,7 @@ type EvalRunDataContent struct { } // NewAgentTargetDataSource builds an EvalRunDataSource configured for agent target completions. -// The source field must be set separately via SetFileContent or SetFileID. +// The rows must be supplied separately via SetFileContent. func NewAgentTargetDataSource(agentName string, agentVersion *string) *EvalRunDataSource { return &EvalRunDataSource{ Type: EvalRunDataSourceTypeAgentTarget, @@ -432,6 +395,11 @@ func NewAgentTargetDataSource(agentName string, agentVersion *string) *EvalRunDa } // SetFileContent sets the data source to use inline file content. +// +// There is no by-reference counterpart. A run's `file_id` means an uploaded +// file, and a dataset name is not one — sending it is rejected with "invalid +// data source file ids" — so registered datasets are fetched and sent inline +// too. See readRegisteredDataset. func (ds *EvalRunDataSource) SetFileContent(items []map[string]any) { ds.Source = &EvalRunDataContent{ Type: EvalRunDataContentTypeFileContent, @@ -439,14 +407,6 @@ func (ds *EvalRunDataSource) SetFileContent(items []map[string]any) { } } -// SetFileID sets the data source to reference a remote dataset by ID. -func (ds *EvalRunDataSource) SetFileID(fileID string) { - ds.Source = &EvalRunDataContent{ - Type: EvalRunDataContentTypeFileID, - ID: fileID, - } -} - // OpenAIEvalRun is the response for an OpenAI eval run. type OpenAIEvalRun struct { ID string `json:"id"` diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/portal_urls_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/portal_urls_test.go new file mode 100644 index 00000000000..602b584ccd5 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/portal_urls_test.go @@ -0,0 +1,166 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package eval_api + +import ( + "strings" + "testing" + + "azureaieval/internal/pkg/evalcore" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const testProjectID = "/subscriptions/00000000-1111-2222-3333-444444444444/" + + "resourceGroups/rg-eval/providers/Microsoft.CognitiveServices/accounts/acct/projects/proj" + +// A portal URL is printed at the end of a run and is the one thing a user +// clicks. It is assembled from parts rather than returned by the service, so +// nothing but a test says whether it lands anywhere. +func TestPortalPrefix_BuildsEveryDocumentedURL(t *testing.T) { + p, err := NewPortalPrefix(testProjectID) + require.NoError(t, err) + + // The subscription travels base64url-encoded without padding, so the + // literal GUID must not appear anywhere in the result. + const sub = "00000000-1111-2222-3333-444444444444" + + tests := []struct { + name string + got string + want string + }{ + {"eval run", p.EvalRunURL("eval_1", "evalrun_1"), "/build/evaluations/eval_1/run/evalrun_1"}, + {"evaluator", p.EvaluatorURL("quality", "3"), "/build/evaluations/catalog/quality/3"}, + {"dataset", p.DatasetURL("regression", "2"), "/build/data/datasets/regression/2"}, + {"optimization", p.OptimizationURL("support", "op_9"), "/build/agents/support/optimization/op_9"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.True(t, strings.HasPrefix(tt.got, "https://ai.azure.com/nextgen/r/"), + "got %s", tt.got) + assert.True(t, strings.HasSuffix(tt.got, tt.want), "got %s", tt.got) + assert.Contains(t, tt.got, "rg-eval") + assert.Contains(t, tt.got, "acct") + assert.Contains(t, tt.got, "proj") + assert.NotContains(t, tt.got, sub, + "the subscription is encoded, so its plain GUID must not appear") + }) + } +} + +// The encoding is what the portal decodes on the other end, so it is pinned +// rather than merely exercised. +func TestEncodeSubscriptionForURL(t *testing.T) { + encoded, err := encodeSubscriptionForURL("00000000-1111-2222-3333-444444444444") + + require.NoError(t, err) + assert.NotContains(t, encoded, "=", "padding would need escaping inside a URL segment") + assert.NotContains(t, encoded, "+", "base64url, not standard base64") + assert.NotContains(t, encoded, "/", "a slash would split the URL segment") + assert.Equal(t, "AAAAABERIiIzM0RERERERA", encoded) +} + +func TestEncodeSubscriptionForURL_RejectsSomethingThatIsNotAGUID(t *testing.T) { + _, err := encodeSubscriptionForURL("not-a-subscription") + + require.Error(t, err) + assert.Contains(t, err.Error(), "subscription") +} + +// A resource ID that is not a project has no account to name, and guessing +// would produce a URL that resolves to someone else's project. +func TestNewPortalPrefix_RefusesWhatIsNotAProject(t *testing.T) { + tests := []struct { + name string + id string + }{ + {"not a resource id at all", "hello"}, + {"empty", ""}, + { + name: "an account rather than a project under it", + id: "/subscriptions/00000000-1111-2222-3333-444444444444/resourceGroups/rg/" + + "providers/Microsoft.CognitiveServices/accounts/acct", + }, + { + name: "a project whose subscription is not a GUID", + id: "/subscriptions/not-a-guid/resourceGroups/rg/providers/" + + "Microsoft.CognitiveServices/accounts/acct/projects/proj", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + p, err := NewPortalPrefix(tt.id) + + require.Error(t, err) + assert.Nil(t, p) + }) + } +} + +// The prefix distinguishes built-in evaluators from ones the project owns, +// which is what decides whether a version is published or referenced. +func TestIsBuiltinEvaluator(t *testing.T) { + assert.True(t, IsBuiltinEvaluator("builtin.task_adherence")) + assert.False(t, IsBuiltinEvaluator("task_adherence")) + assert.False(t, IsBuiltinEvaluator("builtin"), "the dot is part of the prefix") + assert.False(t, IsBuiltinEvaluator("my.builtin.thing"), "the prefix has to lead") + assert.False(t, IsBuiltinEvaluator("")) +} + +func TestSplitEvaluators(t *testing.T) { + generated, builtin := SplitEvaluators(evalcore.EvaluatorList{ + {Name: "builtin.coherence"}, + {Name: "support-quality"}, + {Name: "builtin.task_adherence"}, + }) + + require.Len(t, generated, 1) + assert.Equal(t, "support-quality", generated[0].Name) + require.Len(t, builtin, 2) + assert.Equal(t, "builtin.coherence", builtin[0].Name) + assert.Equal(t, "builtin.task_adherence", builtin[1].Name) +} + +// Both halves come back nil rather than empty for an empty input, so a caller +// checking len() reads the same either way. +func TestSplitEvaluators_Empty(t *testing.T) { + generated, builtin := SplitEvaluators(nil) + + assert.Empty(t, generated) + assert.Empty(t, builtin) +} + +// This decides whether a value is looked up in the service or opened off disk. +// Getting it wrong sends a path to the registry, or a registered name to the +// filesystem, and neither failure names the real problem. +func TestIsDatasetName(t *testing.T) { + names := []string{ + "support-regression", + "dataset_v2", + "name.with.dots", + "trailing.txt", + } + for _, v := range names { + assert.Truef(t, IsDatasetName(v), "%q is a registered name", v) + } + + paths := []string{ + "", + "data.jsonl", + "data.json", + "data.csv", + "DATA.JSONL", + "./data.jsonl", + "evals/datasets/x.jsonl", + `evals\datasets\x.jsonl`, + "a/b", + } + for _, v := range paths { + assert.Falsef(t, IsDatasetName(v), "%q is a path, not a name", v) + } +} From b5b56d2da99614e472619c42bb294b31fda89117 Mon Sep 17 00:00:00 2001 From: mohessie Date: Thu, 6 Aug 2026 03:44:24 +0300 Subject: [PATCH 111/320] Cover the mechanisms that fail silently when they drift Second coverage pass, picking the code where a regression produces a plausible wrong answer rather than an error. digestIDKey had no test at all, and it is what makes a rename find the eval it already deployed: the id is recorded against the eval's substance, so a declaration whose name changed still resolves. Change the key format and every deployed eval quietly loses its recorded id and gets recreated, losing its run history. The format is pinned now, along with the two properties it rests on -- same substance gives the same key, different substance does not. FingerprintKey has the same shape of failure. It maps a name into an azd environment key, and a character that reached the file unmapped would produce a key azd cannot round-trip, so the artifact would look changed on every deploy. Pinned, including that one rune maps to one underscore rather than one byte. itemColumn reads the dataset column out of an {{item.}} binding. Reading it wrong submits a run against a column the dataset does not have, and the service rejects that without saying which one. Also covered: Fingerprint itself, versionFromRaw's fallback chain, defaultEvalName (the source is in the name so two evals over one agent do not collide), jobLookupError naming the right group, WithoutAgentSource and HasPromptSource (the retry that saves the documented generate flow), and both generation request builders -- the evaluator one sends the name under two keys and setting only one returns an evaluator under the wrong name. FingerprintGroup was already covered in service_target_eval_test.go, so the new file says where rather than testing it twice. Coverage 39.2% -> 42.1%; project 66.8% -> 70.4%, eval_api 30.3% -> 42.2%. --- .../internal/cmd/helpers_test.go | 149 ++++++++++++++++++ .../internal/pkg/eval_api/generation_test.go | 57 +++++++ .../internal/project/fingerprint_test.go | 87 ++++++++++ 3 files changed, 293 insertions(+) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/helpers_test.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/project/fingerprint_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/helpers_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/helpers_test.go new file mode 100644 index 00000000000..4654145e853 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/helpers_test.go @@ -0,0 +1,149 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "strings" + "testing" + + "azureaieval/internal/project" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// digestIDKey is what makes a rename find the eval it already deployed: the id +// is recorded against the eval's substance, so a declaration whose name +// changed still resolves. That only works while the key derives from the +// digest the same way it did last deploy — change the format and every +// deployed eval silently loses its recorded id and gets recreated. +func TestDigestIDKey(t *testing.T) { + const digest = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" + + key := digestIDKey(digest) + + assert.Equal(t, "EVAL_SUBSTANCE_0123456789ABCDEF_ID", key) + for _, r := range key { + assert.Truef(t, + (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9') || r == '_', + "%q is not allowed in an environment key", r) + } +} + +// Same substance, same key — that is the whole mechanism. +func TestDigestIDKey_IsStableForTheSameSubstance(t *testing.T) { + group := project.Eval{ + Name: "support", + Dataset: "support-regression", + Target: &project.Target{Name: "support-agent"}, + } + + first, err := project.FingerprintGroup(group) + require.NoError(t, err) + + renamed := group + renamed.Name = "support-renamed" + renamed.Description = "reworded" + second, err := project.FingerprintGroup(renamed) + require.NoError(t, err) + + assert.Equal(t, digestIDKey(first), digestIDKey(second), + "a rename must land on the key the first deploy wrote") +} + +// Different substance, different key, so a genuinely new eval does not adopt +// an unrelated one's id. +func TestDigestIDKey_DiffersWhenTheSubstanceDoes(t *testing.T) { + a, err := project.FingerprintGroup(project.Eval{Name: "x", Dataset: "one"}) + require.NoError(t, err) + b, err := project.FingerprintGroup(project.Eval{Name: "x", Dataset: "two"}) + require.NoError(t, err) + + assert.NotEqual(t, digestIDKey(a), digestIDKey(b)) +} + +// The version recorded for an artifact comes out of what the service returned, +// falling back to what the caller already knew. +func TestVersionFromRaw(t *testing.T) { + tests := []struct { + name string + raw string + fallback string + want string + }{ + {"version in the body wins", `{"version":"7"}`, "3", "7"}, + {"empty version falls back", `{"version":""}`, "3", "3"}, + {"absent version falls back", `{"name":"x"}`, "3", "3"}, + {"unparseable body falls back", `not json`, "3", "3"}, + {"empty body falls back", ``, "3", "3"}, + {"no fallback either", `{}`, "", ""}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.want, versionFromRaw([]byte(tt.raw), tt.fallback)) + }) + } +} + +// A criterion binds to a dataset column through `{{item.}}`. Reading the +// name wrong is how a run is submitted against a column the dataset does not +// have, which the service rejects without saying which one. +func TestItemColumn(t *testing.T) { + bound := map[string]string{ + "{{item.query}}": "query", + "{{item.ground_truth}}": "ground_truth", + "{{item.a.b}}": "a.b", + } + for binding, want := range bound { + got, ok := itemColumn(binding) + assert.Truef(t, ok, "%q is a binding", binding) + assert.Equal(t, want, got) + } + + notBound := []string{ + "", + "query", + "{{item.}}", + "{{ item.query }}", + "{{item.query", + "item.query}}", + "{{response.output}}", + } + for _, binding := range notBound { + got, ok := itemColumn(binding) + assert.Falsef(t, ok, "%q is not an item binding", binding) + assert.Empty(t, got) + } +} + +// An eval is named after what it evaluates and what it reads, so two evals over +// the same agent from different sources do not collide. +func TestDefaultEvalName(t *testing.T) { + assert.Equal(t, "support-agent-trace-eval", + defaultEvalName("support-agent", initSourceTraces)) + assert.Equal(t, "support-agent-eval", + defaultEvalName("support-agent", "dataset")) + assert.Equal(t, "support-agent-eval", + defaultEvalName("support-agent", "")) + + assert.NotEqual(t, + defaultEvalName("support-agent", initSourceTraces), + defaultEvalName("support-agent", "dataset"), + "the source is in the name so the two do not collide") +} + +// The reattach line printed by --no-wait has to name the group the job +// actually belongs to; the two job types share no collection, so the wrong +// group is a command that returns "not found". +func TestJobLookupErrorNamesTheGroup(t *testing.T) { + for _, kind := range []jobKind{datasetJobs, evaluatorJobs} { + err := jobLookupError(kind, "job_1", assert.AnError) + + require.Error(t, err) + assert.Contains(t, err.Error(), "job_1") + assert.Truef(t, strings.Contains(err.Error(), kind.name), + "the error must name the %q group so the retry goes to the right one", kind.name) + } +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/generation_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/generation_test.go index 85183400040..6a5c993f820 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/generation_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/generation_test.go @@ -188,3 +188,60 @@ func TestBuildGenerationSources_AgentVersionIsOptional(t *testing.T) { require.Len(t, withoutVersion, 1) assert.Empty(t, withoutVersion[0].AgentVersion) } + +// The retry that saves the documented flow: agent-seeded generation fails +// server-side for every agent, and the same request without the agent source +// succeeds. +func TestWithoutAgentSource(t *testing.T) { + sources := []GenerationSource{ + {Type: "prompt", Prompt: "be helpful"}, + {Type: "agent", AgentName: "support"}, + {Type: "traces", AgentName: "support"}, + } + + kept := WithoutAgentSource(sources) + + assert.Equal(t, []string{"prompt", "traces"}, kindsOf(kept)) + assert.Len(t, sources, 3, "the original must not be modified; it is retried from") +} + +// The retry only happens when something is left to generate from, so this is +// what stops a second billed job that would fail the same way. +func TestHasPromptSource(t *testing.T) { + assert.True(t, HasPromptSource([]GenerationSource{{Type: "prompt", Prompt: "x"}})) + assert.False(t, HasPromptSource([]GenerationSource{{Type: "prompt"}}), + "an empty prompt is nothing to generate from") + assert.False(t, HasPromptSource([]GenerationSource{{Type: "agent", AgentName: "s"}})) + assert.False(t, HasPromptSource(nil)) +} + +// The request body is what the service validates, so the fields it keys on are +// pinned rather than left to whatever the builder happens to set. +func TestNewDataGenerationJobRequest(t *testing.T) { + sources := []GenerationSource{{Type: "prompt", Prompt: "be helpful"}} + + req := NewDataGenerationJobRequest("support-regression", "gpt-4o", 15, sources) + + require.NotNil(t, req) + assert.Equal(t, "support-regression", req.Inputs.Name) + assert.Equal(t, "evaluation", req.Inputs.Scenario) + assert.Equal(t, "simple_qna", req.Inputs.Options.Type) + assert.Equal(t, 15, req.Inputs.Options.MaxSamples) + assert.Equal(t, "gpt-4o", req.Inputs.Options.ModelOptions.Model) + assert.Equal(t, sources, req.Inputs.Sources) +} + +// The evaluator request sends the name twice, under two keys the service reads +// separately. Setting only one produces a job that runs and returns an +// evaluator under the wrong name. +func TestNewEvaluatorGenerationJobRequest(t *testing.T) { + sources := []GenerationSource{{Type: "prompt", Prompt: "grade politeness"}} + + req := NewEvaluatorGenerationJobRequest("support-quality", "gpt-4o", sources) + + require.NotNil(t, req) + assert.Equal(t, "support-quality", req.Inputs.Name) + assert.Equal(t, "support-quality", req.Inputs.EvaluatorName) + assert.Equal(t, "gpt-4o", req.Inputs.Model) + assert.Equal(t, sources, req.Inputs.Sources) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/fingerprint_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/fingerprint_test.go new file mode 100644 index 00000000000..2cf1fe8e889 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/fingerprint_test.go @@ -0,0 +1,87 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package project + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// FingerprintGroup is covered in service_target_eval_test.go. These cover the +// file hash and the environment key it is stored under, which nothing did. + +// A fingerprint is compared against the one recorded at the last deploy, so +// identical content must hash identically and a single changed byte must not. +func TestFingerprint(t *testing.T) { + dir := t.TempDir() + a := filepath.Join(dir, "a.jsonl") + b := filepath.Join(dir, "b.jsonl") + require.NoError(t, os.WriteFile(a, []byte(`{"query":"hi"}`), 0o600)) + require.NoError(t, os.WriteFile(b, []byte(`{"query":"hi"}`), 0o600)) + + sumA, err := Fingerprint(a) + require.NoError(t, err) + sumB, err := Fingerprint(b) + require.NoError(t, err) + + assert.Equal(t, sumA, sumB, "same content, same fingerprint") + assert.Len(t, sumA, 64, "sha-256 as hex") + + require.NoError(t, os.WriteFile(b, []byte(`{"query":"hI"}`), 0o600)) + sumB, err = Fingerprint(b) + require.NoError(t, err) + assert.NotEqual(t, sumA, sumB, "one changed byte has to show") +} + +// A missing file names itself, because the usual cause is a catalog entry +// pointing at something that was moved or never generated. +func TestFingerprint_MissingFileNamesIt(t *testing.T) { + _, err := Fingerprint(filepath.Join(t.TempDir(), "gone.jsonl")) + + require.Error(t, err) + assert.Contains(t, err.Error(), "gone.jsonl") +} + +// The key goes into an azd environment file, which accepts only uppercase +// letters, digits and underscores. A name that reached it unmapped would +// produce a key azd cannot round-trip, and the artifact would look changed on +// every deploy. +func TestFingerprintKey_IsAValidEnvironmentKey(t *testing.T) { + tests := []struct { + kind, name, want string + }{ + {"dataset", "support-regression", "DATASET_SUPPORT_REGRESSION"}, + {"evaluator", "quality.v2", "EVALUATOR_QUALITY_V2"}, + {"dataset", "Mixed Case Name", "DATASET_MIXED_CASE_NAME"}, + // One rune maps to one underscore, so a multi-byte character does not + // widen the key. + {"eval", "unicode-caf\u00e9", "EVAL_UNICODE_CAF_"}, + } + + for _, tt := range tests { + t.Run(tt.want, func(t *testing.T) { + key := FingerprintKey(tt.kind, tt.name) + + assert.Equal(t, EnvKeyFingerprintPrefix+tt.want, key) + for _, r := range strings.TrimPrefix(key, EnvKeyFingerprintPrefix) { + assert.Truef(t, + (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9') || r == '_', + "%q is not allowed in an environment key", r) + } + }) + } +} + +// Two artifacts of different kinds can share a name, and they must not share a +// key — one would overwrite the other's recorded fingerprint. +func TestFingerprintKey_KindSeparatesTheNamespaces(t *testing.T) { + assert.NotEqual(t, + FingerprintKey("dataset", "quality"), + FingerprintKey("evaluator", "quality")) +} From 0fae3f96eef3f124832d690a9005b156e4867414 Mon Sep 17 00:00:00 2001 From: mohessie Date: Thu, 6 Aug 2026 03:47:44 +0300 Subject: [PATCH 112/320] Check the run flags against the spec's own table The command tree and init's flags were pinned to the spec; the run table was not, and it is the one with placement rules a reader can check. --eval is given to every run command, --dataset to run start alone, --fail-on and --wait to run start and run show, --format to run output export. A third command growing one of those is a promise the spec does not make, and a missing one is a promise it does. Both directions are asserted, because the failure that actually happens is the quiet one: a flag added to the command next to the right one still compiles, still parses, and only stops making sense when someone reads the spec beside the help text. Placement only, not the whole flag list. Unlike init's table, this one is headed "Flag | Commands | Default" and documents defaults rather than enumerating flags, so pinning it whole would fail on flags the spec never meant to exclude. Everything passes as written, so this records the current state rather than fixing a drift -- which is the point of having it before the drift. --- .../internal/cmd/surface_test.go | 62 +++++++++++++++++++ 1 file changed, 62 insertions(+) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/surface_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/surface_test.go index 1d6b8e969c3..cf415a846ed 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/surface_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/surface_test.go @@ -8,6 +8,7 @@ import ( "os" "path/filepath" "regexp" + "slices" "strings" "testing" @@ -241,6 +242,67 @@ func TestEvaluatorGenerateHasNoFromFlag(t *testing.T) { "the spec gives --from to dataset generate only") } +// The spec's run table says which commands carry which flag. Where it says +// "every", that is checkable; where it names two commands, a third carrying the +// flag is a promise the spec does not make and a missing one is a promise it +// does. +// +// This pins placement, not the whole flag list: unlike init's, the run table is +// headed "Flag | Commands | Default" and documents defaults rather than +// enumerating every flag. +func TestRunFlagsSitWhereTheSpecSaysTheyDo(t *testing.T) { + // Flag → exactly the run commands that may declare it. nil means every + // run command that does something. + placement := map[string][]string{ + "eval": nil, + "dataset": {"run start"}, + "fail-on": {"run start", "run show"}, + "wait": {"run start", "run show"}, + "format": {"run output export"}, + } + + // Every run command that actually runs, which is what "every run command" + // means — the bare groups take no flags. + var runCommands []string + walk(t, NewRootCommand(), nil, func(path string, cmd *cobra.Command) { + if strings.HasPrefix(path, "run") && cmd.RunE != nil { + runCommands = append(runCommands, path) + } + }) + require.NotEmpty(t, runCommands) + + for flag, allowed := range placement { + if allowed == nil { + allowed = runCommands + } + for _, path := range runCommands { + has := find(t, path).Flags().Lookup(flag) != nil + want := slices.Contains(allowed, path) + + switch { + case want && !has: + t.Errorf("%s must accept --%s; the spec's run table says so", path, flag) + case !want && has: + t.Errorf("%s declares --%s, which the spec gives only to %s", + path, flag, strings.Join(allowed, ", ")) + } + } + } +} + +// `--eval` is how a run command finds the eval, and the spec gives it to every +// one of them. Losing it from a single command makes that command unusable in a +// project with more than one eval. +func TestEveryRunCommandTakesEval(t *testing.T) { + walk(t, NewRootCommand(), nil, func(path string, cmd *cobra.Command) { + if !strings.HasPrefix(path, "run") || cmd.RunE == nil { + return + } + assert.NotNilf(t, cmd.Flags().Lookup("eval"), + "%s must accept --eval, which the spec gives to every run command", path) + }) +} + // find resolves a command path, failing the test when it does not exist. func find(t *testing.T, path string) *cobra.Command { t.Helper() From f7e4d6a10617f8b84fb28be0d5e8b3c0a8d85c84 Mon Sep 17 00:00:00 2001 From: mohessie Date: Thu, 6 Aug 2026 03:57:32 +0300 Subject: [PATCH 113/320] Repair the tagged suites, and stop them rotting unnoticed The live and hero suites drive the binary by writing flags as strings, so a removed flag still compiles there. go test ./... builds neither tag, so nothing said a word when this branch removed --eval-id and the generation spec file: 28 uses of --eval-id and two --config tests stayed behind, and every one of them would have failed at the first live run, in front of whoever happened to have the credentials. The hero suite was worse, because it asserts terminal output whole. It still expected the pre-split shape -- evals/support-agent-smoke.yaml plus evals/generate.yaml, a 'Planned evaluators' line, a 'support-agent-smoke' service, and --generation-model on init, which init no longer takes. It has been the only thing asserting what init prints, and it was asserting a version of init that stopped existing several commits ago. So: Scenario 1 now matches the spec's block against the single evals/eval.yaml and the support-agent-evals service, and a second test asserts the config file the spec prints beside it -- the traces source, the agent_name that scopes it, the max_traces cap, and the judge written per evaluator reference. It also asserts what must be absent: no datasets: catalog, because there is no file to register, and no target:, because a trace run invokes nothing. The two --config tests are deleted rather than repaired. They covered a generation spec file that no longer exists; generateFlags says so in as many words. The guard is the part that matters. A test in the default suite reads every flag literal in tests/ and checks it against what the commands actually declare, so a rename is caught by the person doing the renaming rather than by whoever next runs the live suite. Reintroducing --eval-id makes it fail naming the file and line. --- .../internal/cmd/surface_test.go | 48 +++++++++ .../tests/cli/generate_test.go | 33 ------ .../tests/cli/handoff_test.go | 10 +- .../tests/cli/run_ops_test.go | 30 +++--- .../tests/cli/run_output_test.go | 16 +-- .../tests/hero/init_test.go | 100 ++++++++++++------ 6 files changed, 146 insertions(+), 91 deletions(-) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/surface_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/surface_test.go index cf415a846ed..6623442e1c0 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/surface_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/surface_test.go @@ -303,6 +303,54 @@ func TestEveryRunCommandTakesEval(t *testing.T) { }) } +// The tagged suites drive the binary by writing flags as strings, so a flag +// that is renamed or removed still compiles there and only fails when someone +// has the credentials to run them. +// +// That is not hypothetical. Removing `--eval-id` and the generation spec file +// left 28 uses of `--eval-id` and two `--config` tests behind in tests/cli, +// every one of which would have failed at the first live run — under `live` and +// `hero` tags that `go test ./...` never builds. This checks them from the +// default suite, where a rename is caught by the person doing the renaming. +func TestTaggedSuitesNameFlagsThatExist(t *testing.T) { + // Every flag any command declares, plus the globals azd contributes. + known := map[string]bool{ + "output": true, "no-prompt": true, "environment": true, + "cwd": true, "debug": true, "help": true, + } + walk(t, NewRootCommand(), nil, func(_ string, cmd *cobra.Command) { + cmd.Flags().VisitAll(func(f *pflag.Flag) { known[f.Name] = true }) + }) + + // Only string literals, which is how a test spells a flag it passes to the + // binary. Prose in a comment is not a flag. + literal := regexp.MustCompile(`"--([a-z][a-z0-9-]*)"`) + + err := filepath.WalkDir("../../tests", func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() || !strings.HasSuffix(path, ".go") { + return nil + } + body, err := os.ReadFile(path) + if err != nil { + return err + } + for i, line := range strings.Split(string(body), "\n") { + if strings.HasPrefix(strings.TrimSpace(line), "//") { + continue + } + for _, m := range literal.FindAllStringSubmatch(line, -1) { + assert.Truef(t, known[m[1]], + "%s:%d passes --%s, which no command declares", path, i+1, m[1]) + } + } + return nil + }) + require.NoError(t, err) +} + // find resolves a command path, failing the test when it does not exist. func find(t *testing.T, path string) *cobra.Command { t.Helper() diff --git a/cli/azd/extensions/azure.ai.evaluations/tests/cli/generate_test.go b/cli/azd/extensions/azure.ai.evaluations/tests/cli/generate_test.go index 71466a9b203..3b7c80b7cd4 100644 --- a/cli/azd/extensions/azure.ai.evaluations/tests/cli/generate_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/tests/cli/generate_test.go @@ -88,39 +88,6 @@ func TestCLIGenerateNamesTheArtifact(t *testing.T) { func TestCLIGenerateNoPromptNamesWhatIsMissing(t *testing.T) { r := requireFailure(t, runIn(t, t.TempDir(), "dataset", "generate", "d", "--no-prompt")) require.Contains(t, r.Combined(), "--generation-model") - require.Contains(t, r.Combined(), "generationModel", - "the message must name both ways of supplying it") -} - -// TestCLIGenerateReadsTheSpec proves the config file is loaded and that its -// entries are looked up by artifact name, rather than only the flags being read. -// -// The sample size is the clearest evidence: a value the service would reject is -// refused before any job is submitted, and it can only have come from the file. -func TestCLIGenerateReadsTheSpec(t *testing.T) { - dir := t.TempDir() - spec := filepath.Join(dir, "generate.yaml") - require.NoError(t, os.WriteFile(spec, []byte(` -generationModel: gpt-4o-mini -dataset: - spec-dataset: - sampleSize: 5 -`), 0o600)) - - r := requireFailure(t, runIn(t, dir, "dataset", "generate", "spec-dataset", "--config", spec)) - require.Contains(t, r.Combined(), "between 15 and 1000", - "the spec's sampleSize must be validated before a job is billed") -} - -// A spec that cannot be parsed has to name itself, or the caller is left -// guessing which of several YAML files the command choked on. -func TestCLIGenerateReportsAnUnparseableSpec(t *testing.T) { - dir := t.TempDir() - spec := filepath.Join(dir, "generate.yaml") - require.NoError(t, os.WriteFile(spec, []byte("dataset: [not-a-mapping\n"), 0o600)) - - r := requireFailure(t, runIn(t, dir, "dataset", "generate", "d", "--config", spec)) - require.Contains(t, r.Combined(), "generate.yaml") } // TestCLIGenerateFlagsAreScopedToTheirArtifact asserts the two commands do not diff --git a/cli/azd/extensions/azure.ai.evaluations/tests/cli/handoff_test.go b/cli/azd/extensions/azure.ai.evaluations/tests/cli/handoff_test.go index 56b5465beba..6751ddd8bde 100644 --- a/cli/azd/extensions/azure.ai.evaluations/tests/cli/handoff_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/tests/cli/handoff_test.go @@ -26,7 +26,7 @@ func TestCLIStartNoWaitEmitsTheHandoff(t *testing.T) { f := sharedEval(t) r := requireSuccess(t, run(t, - "run", "start", "--eval-id", f.EvalID, "--no-wait", "-o", "json")) + "run", "start", "--eval", f.EvalID, "--no-wait", "-o", "json")) var handoff struct { RunID string `json:"run_id"` @@ -45,12 +45,12 @@ func TestCLIStartNoWaitEmitsTheHandoff(t *testing.T) { assert.NotEqual(t, "completed", handoff.Status) deferTeardown(func() { - runQuietly("run", "cancel", handoff.RunID, "--eval-id", f.EvalID) + runQuietly("run", "cancel", handoff.RunID, "--eval", f.EvalID) }) // The id it handed back has to be one the next step can use. shown := requireSuccess(t, run(t, - "run", "show", handoff.RunID, "--eval-id", f.EvalID, "-o", "json")) + "run", "show", handoff.RunID, "--eval", f.EvalID, "-o", "json")) var reattached struct { ID string `json:"id"` } @@ -65,7 +65,7 @@ func TestCLIStartNoWaitEmitsTheHandoff(t *testing.T) { func TestCLIStartNoWaitTellsAPersonHowToReattach(t *testing.T) { f := sharedEval(t) - r := requireSuccess(t, run(t, "run", "start", "--eval-id", f.EvalID, "--no-wait")) + r := requireSuccess(t, run(t, "run", "start", "--eval", f.EvalID, "--no-wait")) assert.Contains(t, r.Stdout, "Reattach with: azd ai eval run show") assert.Contains(t, r.Stdout, f.EvalID, @@ -81,5 +81,5 @@ func TestCLIStartNoWaitTellsAPersonHowToReattach(t *testing.T) { } } require.NotEmpty(t, runID, "the run id must be printed:\n%s", r.Stdout) - deferTeardown(func() { runQuietly("run", "cancel", runID, "--eval-id", f.EvalID) }) + deferTeardown(func() { runQuietly("run", "cancel", runID, "--eval", f.EvalID) }) } diff --git a/cli/azd/extensions/azure.ai.evaluations/tests/cli/run_ops_test.go b/cli/azd/extensions/azure.ai.evaluations/tests/cli/run_ops_test.go index 626ab9dbccd..3d68a583083 100644 --- a/cli/azd/extensions/azure.ai.evaluations/tests/cli/run_ops_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/tests/cli/run_ops_test.go @@ -28,7 +28,7 @@ func TestCLIRunList(t *testing.T) { f := sharedEval(t) t.Run("table", func(t *testing.T) { - r := requireSuccess(t, run(t, "run", "list", "--eval-id", f.EvalID)) + r := requireSuccess(t, run(t, "run", "list", "--eval", f.EvalID)) for _, header := range []string{"RUN ID", "NAME", "STATUS", "RESULTS"} { require.Containsf(t, r.Stdout, header, "the listing lost its %s column", header) } @@ -39,7 +39,7 @@ func TestCLIRunList(t *testing.T) { }) t.Run("json", func(t *testing.T) { - r := requireSuccess(t, run(t, "run", "list", "--eval-id", f.EvalID, "-o", "json")) + r := requireSuccess(t, run(t, "run", "list", "--eval", f.EvalID, "-o", "json")) require.True(t, strings.HasPrefix(strings.TrimSpace(r.Stdout), "["), "a list must be a bare array, not the service's envelope") @@ -63,14 +63,14 @@ func TestCLIRunList(t *testing.T) { // The client has always taken a limit; until recently the command did not // expose one, so a service-side truncation would have passed unnoticed. t.Run("limit", func(t *testing.T) { - r := requireSuccess(t, run(t, "run", "list", "--eval-id", f.EvalID, "--limit", "1", "-o", "json")) + r := requireSuccess(t, run(t, "run", "list", "--eval", f.EvalID, "--limit", "1", "-o", "json")) var runs []runSummary r.JSON(t, &runs) require.Len(t, runs, 1, "--limit must reach the service") }) t.Run("unknown eval is brief", func(t *testing.T) { - r := requireFailure(t, run(t, "run", "list", "--eval-id", "eval_azdcli_no_such_eval")) + r := requireFailure(t, run(t, "run", "list", "--eval", "eval_azdcli_no_such_eval")) require.Less(t, len(r.Combined()), 600, "a not-found must stay short, not dump the service body:\n%s", r.Combined()) require.Contains(t, r.Combined(), "eval_azdcli_no_such_eval") @@ -81,7 +81,7 @@ func TestCLIRunShow(t *testing.T) { f := sharedEval(t) t.Run("by run id", func(t *testing.T) { - r := requireSuccess(t, run(t, "run", "show", f.FirstRunID, "--eval-id", f.EvalID)) + r := requireSuccess(t, run(t, "run", "show", f.FirstRunID, "--eval", f.EvalID)) require.Contains(t, r.Stdout, f.FirstRunID) require.Contains(t, r.Stdout, "status") require.Contains(t, r.Stdout, "completed") @@ -93,12 +93,12 @@ func TestCLIRunShow(t *testing.T) { // environment there is no remembered id to fall back on, so what is // exercised is the listing path. t.Run("defaults to the most recent run", func(t *testing.T) { - listed := requireSuccess(t, run(t, "run", "list", "--eval-id", f.EvalID, "--limit", "1", "-o", "json")) + listed := requireSuccess(t, run(t, "run", "list", "--eval", f.EvalID, "--limit", "1", "-o", "json")) var newest []runSummary listed.JSON(t, &newest) require.Len(t, newest, 1) - r := requireSuccess(t, run(t, "run", "show", "--eval-id", f.EvalID, "-o", "json")) + r := requireSuccess(t, run(t, "run", "show", "--eval", f.EvalID, "-o", "json")) var shown runSummary r.JSON(t, &shown) require.Equal(t, newest[0].ID, shown.ID, @@ -114,7 +114,7 @@ func TestCLIRunShow(t *testing.T) { // about 1700 characters of raw JSON — recorded in the report rather than // pinned here, since pinning it would make the length a requirement. t.Run("an unknown run id is reported, not silently replaced", func(t *testing.T) { - r := requireFailure(t, run(t, "run", "show", "evalrun_azdcli_nope", "--eval-id", f.EvalID)) + r := requireFailure(t, run(t, "run", "show", "evalrun_azdcli_nope", "--eval", f.EvalID)) require.Contains(t, r.Combined(), "evalrun_azdcli_nope", "the failure must name the run that was asked for") require.NotContains(t, r.Combined(), f.FirstRunID, @@ -132,7 +132,7 @@ func TestCLIRunCancelAndDelete(t *testing.T) { f := sharedEval(t) t.Run("a finished run is refused", func(t *testing.T) { - r := requireFailure(t, run(t, "run", "cancel", f.FirstRunID, "--eval-id", f.EvalID)) + r := requireFailure(t, run(t, "run", "cancel", f.FirstRunID, "--eval", f.EvalID)) require.Contains(t, r.Combined(), "already finished") require.Contains(t, r.Combined(), "completed") }) @@ -148,21 +148,21 @@ func TestCLIRunCancelAndDelete(t *testing.T) { t.Run("an in-flight run is cancelled, and the delete is accepted", func(t *testing.T) { runID := startCancellableRun(t, f) - cancelled := requireSuccess(t, run(t, "run", "cancel", runID, "--eval-id", f.EvalID)) + cancelled := requireSuccess(t, run(t, "run", "cancel", runID, "--eval", f.EvalID)) require.Contains(t, cancelled.Stdout, runID) require.Contains(t, cancelled.Stdout, "is now") - shown := requireSuccess(t, run(t, "run", "show", runID, "--eval-id", f.EvalID, "-o", "json")) + shown := requireSuccess(t, run(t, "run", "show", runID, "--eval", f.EvalID, "-o", "json")) var after runSummary shown.JSON(t, &after) require.NotEqual(t, "completed", after.Status, "a cancelled run must not go on to complete") - deleted := requireSuccess(t, run(t, "run", "delete", runID, "--eval-id", f.EvalID)) + deleted := requireSuccess(t, run(t, "run", "delete", runID, "--eval", f.EvalID)) require.Contains(t, deleted.Stdout, "Deleted run") require.Contains(t, deleted.Stdout, runID) - still := requireSuccess(t, run(t, "run", "show", runID, "--eval-id", f.EvalID, "-o", "json")) + still := requireSuccess(t, run(t, "run", "show", runID, "--eval", f.EvalID, "-o", "json")) var survivor runSummary still.JSON(t, &survivor) t.Logf("the run is still readable after a successful delete (status %q); "+ @@ -172,12 +172,12 @@ func TestCLIRunCancelAndDelete(t *testing.T) { // Deleting is not undoable, so the id is required rather than defaulted to // whichever run happens to be newest. t.Run("delete requires the run id", func(t *testing.T) { - r := requireFailure(t, run(t, "run", "delete", "--eval-id", f.EvalID)) + r := requireFailure(t, run(t, "run", "delete", "--eval", f.EvalID)) require.Contains(t, r.Combined(), "accepts 1 arg") }) t.Run("deleting an unknown run is reported briefly", func(t *testing.T) { - r := requireFailure(t, run(t, "run", "delete", "evalrun_azdcli_nope", "--eval-id", f.EvalID)) + r := requireFailure(t, run(t, "run", "delete", "evalrun_azdcli_nope", "--eval", f.EvalID)) require.Contains(t, r.Combined(), "evalrun_azdcli_nope") require.Less(t, len(r.Combined()), 600, "a not-found must stay short, not dump the service body:\n%s", r.Combined()) diff --git a/cli/azd/extensions/azure.ai.evaluations/tests/cli/run_output_test.go b/cli/azd/extensions/azure.ai.evaluations/tests/cli/run_output_test.go index c81258dd9b2..181cc5df69b 100644 --- a/cli/azd/extensions/azure.ai.evaluations/tests/cli/run_output_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/tests/cli/run_output_test.go @@ -56,7 +56,7 @@ type resultsPayload struct { func TestCLIResultsShowRendersTheRows(t *testing.T) { f := sharedEval(t) - r := requireSuccess(t, run(t, "run", "output", "list", f.FirstRunID, "--eval-id", f.EvalID)) + r := requireSuccess(t, run(t, "run", "output", "list", f.FirstRunID, "--eval", f.EvalID)) require.Contains(t, r.Stdout, f.FirstRunID) require.Contains(t, r.Stdout, "Totals:") @@ -133,7 +133,7 @@ func TestCLIResultsShowFailedOnly(t *testing.T) { } r := requireSuccess(t, run(t, "run", "output", "list", f.FirstRunID, - "--eval-id", f.EvalID, "--failed-only")) + "--eval", f.EvalID, "--failed-only")) if failing == 0 { // Saying so is not the same as printing an empty table. @@ -155,7 +155,7 @@ func TestCLIResultsShowFailedOnly(t *testing.T) { // they can decide what the rendered output should say. func resultsFor(t *testing.T, evalID, runID string) resultsPayload { t.Helper() - r := requireSuccess(t, run(t, "run", "output", "list", runID, "--eval-id", evalID, "-o", "json")) + r := requireSuccess(t, run(t, "run", "output", "list", runID, "--eval", evalID, "-o", "json")) var payload resultsPayload r.JSON(t, &payload) return payload @@ -166,7 +166,7 @@ func TestCLIResultsExport(t *testing.T) { t.Run("json to stdout", func(t *testing.T) { r := requireSuccess(t, run(t, "run", "output", "export", f.FirstRunID, - "--eval-id", f.EvalID, "--format", "json")) + "--eval", f.EvalID, "--format", "json")) var exported struct { ID string `json:"id"` @@ -185,7 +185,7 @@ func TestCLIResultsExport(t *testing.T) { t.Run("csv to stdout", func(t *testing.T) { r := requireSuccess(t, run(t, "run", "output", "export", f.FirstRunID, - "--eval-id", f.EvalID, "--format", "csv")) + "--eval", f.EvalID, "--format", "csv")) rows, err := csv.NewReader(strings.NewReader(r.Stdout)).ReadAll() require.NoError(t, err, "--format csv must emit parseable CSV:\n%s", r.Stdout) @@ -202,7 +202,7 @@ func TestCLIResultsExport(t *testing.T) { path := filepath.Join(dir, "results.csv") r := requireSuccess(t, runIn(t, dir, "run", "output", "export", f.FirstRunID, - "--eval-id", f.EvalID, "--format", "csv", "--output-file", path)) + "--eval", f.EvalID, "--format", "csv", "--output-file", path)) require.Empty(t, strings.TrimSpace(r.Stdout), "--output-file redirects the payload; leaving it on stdout too would double it") @@ -214,13 +214,13 @@ func TestCLIResultsExport(t *testing.T) { t.Run("an unknown format is refused", func(t *testing.T) { r := requireFailure(t, run(t, "run", "output", "export", f.FirstRunID, - "--eval-id", f.EvalID, "--format", "xml")) + "--eval", f.EvalID, "--format", "xml")) require.Contains(t, r.Combined(), "json or csv") }) } func TestCLIResultsUnknownEvalIsBrief(t *testing.T) { - r := requireFailure(t, run(t, "run", "output", "list", "--eval-id", "eval_does_not_exist")) + r := requireFailure(t, run(t, "run", "output", "list", "--eval", "eval_does_not_exist")) require.Contains(t, r.Combined(), "eval_does_not_exist") require.NotContains(t, r.Combined(), "RESPONSE 404") } diff --git a/cli/azd/extensions/azure.ai.evaluations/tests/hero/init_test.go b/cli/azd/extensions/azure.ai.evaluations/tests/hero/init_test.go index 8fe79718d53..6c02b3b8ecd 100644 --- a/cli/azd/extensions/azure.ai.evaluations/tests/hero/init_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/tests/hero/init_test.go @@ -184,28 +184,63 @@ func normalize(s string) string { func TestHeroScenario1ColdStart(t *testing.T) { const ( agent = "support-agent" - model = "gpt-5.6-luna" + judge = "gpt-5.6-luna" ) dir := project(t, agent) - out, code := azdEval(t, dir, "init", "--target", agent, "--generation-model", model) + // The evaluator and judge are passed rather than prompted for, because the + // spec's two `?` lines are answers to prompts and a test has no terminal to + // answer them at. + out, code := azdEval(t, dir, "init", + "--target", agent, "--source", "traces", + "--evaluator", "builtin.task_adherence", "--judge-model", judge) require.Zero(t, code, "init makes no service calls, so nothing can fail it here") want := `(✓) Done: Detected agent target: support-agent -(✓) Done: Detected model deployment: gpt-5.6-luna -(✓) Done: Planned evaluators: builtin.task_adherence, support-agent-quality (rubric) +(✓) Done: Using data source: traces (Application Insights) +(✓) Done: Judge model deployment: gpt-5.6-luna Created - evals/support-agent-smoke.yaml eval definition - evals/generate.yaml generation settings (15 samples, 1 rubric) - azure.yaml added service 'support-agent-smoke' + evals/eval.yaml evaluation configuration + azure.yaml added service 'support-agent-evals' -Next: azd ai eval dataset generate support-agent-smoke - azd ai eval evaluator generate support-agent-quality` +Next: azd up + azd ai eval run start` require.Equal(t, want, normalize(out)) } +// Scenario 1's second half: the eval.yaml the terminal block promised. The spec +// prints this file, so its shape is as much a promise as the output above — +// and it is the file a reader reviews before running `azd up`. +func TestHeroScenario1WritesTheDocumentedConfig(t *testing.T) { + dir := project(t, "support-agent") + + _, code := azdEval(t, dir, "init", + "--target", "support-agent", "--source", "traces", + "--evaluator", "builtin.task_adherence", "--judge-model", "gpt-5.6-luna") + require.Zero(t, code) + + body, err := os.ReadFile(filepath.Join(dir, "evals", "eval.yaml")) + require.NoError(t, err) + text := string(body) + + require.Contains(t, text, "name: support-agent-trace-eval") + require.Contains(t, text, "type: traces") + require.Contains(t, text, "agent_name: support-agent", + "a trace run has no target, so agent_name is what scopes it") + require.Contains(t, text, "max_traces: 20", + "a first run is bounded rather than taking the service default of 1000") + require.Contains(t, text, "evaluator: builtin.task_adherence") + require.Contains(t, text, "model: gpt-5.6-luna", + "the judge is written per evaluator reference as initialization_parameters.model") + + require.NotContains(t, text, "datasets:", + "there is no file to register, so the catalog is absent rather than empty") + require.NotContains(t, text, "target:", + "a trace run invokes nothing") +} + // `init` is offline, and being offline is the property that makes its output a // reviewable local diff. A service call here would also make the command fail // for a user who has not authenticated yet, which is exactly when they run it. @@ -213,7 +248,8 @@ func TestHeroInitMakesNoServiceCalls(t *testing.T) { dir := project(t, "support-agent") cmd := exec.Command("azd", "ai", "eval", "init", - "--target", "support-agent", "--generation-model", "m") + "--target", "support-agent", "--evaluator", "builtin.task_adherence", + "--judge-model", "m") cmd.Dir = dir // A proxy pointing nowhere fails any outbound request, so a command that // stays offline is unaffected and one that does not cannot be mistaken for @@ -234,17 +270,18 @@ func TestHeroInitMakesNoServiceCalls(t *testing.T) { func TestHeroInitWiresTheServiceIntoTheProject(t *testing.T) { dir := project(t, "support-agent") - _, code := azdEval(t, dir, "init", "--target", "support-agent", "--generation-model", "m") + _, code := azdEval(t, dir, "init", "--target", "support-agent", + "--evaluator", "builtin.task_adherence", "--judge-model", "m") require.Zero(t, code) root, err := os.ReadFile(filepath.Join(dir, "azure.yaml")) require.NoError(t, err) text := string(root) - require.Contains(t, text, "support-agent-smoke:", - "the service key is the eval's name") + require.Contains(t, text, "support-agent-evals:", + "the service is named for the agent it evaluates") require.Contains(t, text, "host: azure.ai.eval") - require.Contains(t, text, "$ref: ./evals/support-agent-smoke.yaml") + require.Contains(t, text, "$ref: ./evals/eval.yaml") // azd owns the edit, so everything the project already declared survives it. require.Contains(t, text, "name: support-app") @@ -252,14 +289,15 @@ func TestHeroInitWiresTheServiceIntoTheProject(t *testing.T) { require.Contains(t, text, "host: azure.ai.agent") // The eval reads both, so azd has to deploy both first. - require.Regexp(t, `(?s)support-agent-smoke:.*uses:.*ai-project.*support-agent`, text) + require.Regexp(t, `(?s)support-agent-evals:.*uses:.*ai-project.*support-agent`, text) } // Running `init` twice must not deploy the same eval twice. The service key is // the eval's name, so the second run recognizes its own work. func TestHeroInitIsIdempotent(t *testing.T) { dir := project(t, "support-agent") - args := []string{"init", "--target", "support-agent", "--generation-model", "m"} + args := []string{"init", "--target", "support-agent", + "--evaluator", "builtin.task_adherence", "--judge-model", "m"} _, code := azdEval(t, dir, args...) require.Zero(t, code) @@ -284,7 +322,7 @@ func TestHeroInitIsIdempotent(t *testing.T) { require.NoError(t, err) require.Equal(t, 1, strings.Count(string(third), "host: azure.ai.eval"), "a second eval service would deploy the same eval twice") - require.Contains(t, normalize(out), "already declares service 'support-agent-smoke'") + require.Contains(t, normalize(out), "already declares service 'support-agent-evals'") } // Evals attach to a project; they do not create one. Naming the command that @@ -310,16 +348,19 @@ func TestHeroInitExplicitEvaluatorsOptOutOfGeneration(t *testing.T) { dir := project(t, "support-agent") out, code := azdEval(t, dir, "init", - "--target", "support-agent", "--generation-model", "m", + "--target", "support-agent", "--judge-model", "m", "--evaluator", "builtin.task_adherence") require.Zero(t, code, out) text := normalize(out) - require.Contains(t, text, "Planned evaluators: builtin.task_adherence") - require.NotContains(t, text, "(rubric)") require.NotContains(t, text, "evaluator generate", "nothing was scheduled to be generated, so nothing should be suggested") - require.Contains(t, text, "generation settings (15 samples, 0 rubric)") + + body, err := os.ReadFile(filepath.Join(dir, "evals", "eval.yaml")) + require.NoError(t, err) + require.Contains(t, string(body), "evaluator: builtin.task_adherence") + require.NotContains(t, string(body), "support-agent-quality", + "the default rubric was replaced, not added to") } // A supplied dataset is not generated either, so `init` has nothing left to @@ -329,20 +370,19 @@ func TestHeroInitSuppliedDatasetIsNotGenerated(t *testing.T) { dir := project(t, "support-agent") out, code := azdEval(t, dir, "init", - "--target", "support-agent", "--generation-model", "m", + "--target", "support-agent", "--judge-model", "m", "--dataset", "prod-golden", "--evaluator", "builtin.task_adherence") require.Zero(t, code, out) - require.NotContains(t, normalize(out), "dataset generate") - body, err := os.ReadFile(filepath.Join(dir, "evals", "generate.yaml")) - require.NoError(t, err) - require.NotContains(t, string(body), "dataset:", - "nothing is left to generate, so the spec declares nothing") + text := normalize(out) + require.NotContains(t, text, "dataset generate") + require.Contains(t, text, "Next: azd up", + "with nothing left to generate, the next step is the deploy") - eval, err := os.ReadFile(filepath.Join(dir, "evals", "support-agent-smoke.yaml")) + body, err := os.ReadFile(filepath.Join(dir, "evals", "eval.yaml")) require.NoError(t, err) - require.Contains(t, string(eval), "name: prod-golden") - require.NotContains(t, string(eval), "source:", + require.Contains(t, string(body), "dataset: prod-golden") + require.NotContains(t, string(body), "source:", "a registered dataset has nothing to upload") } From 21a8b5f40cb1565e3559dd04501240fdd3d95494 Mon Sep 17 00:00:00 2001 From: mohessie Date: Thu, 6 Aug 2026 04:06:27 +0300 Subject: [PATCH 114/320] Suggest commands this extension actually serves Three printed instructions pointed at `azd ai dataset ...`: init's next step after scaffolding, the --no-wait reattach line, and the --from file error. No installed extension serves that namespace. This one is `ai.eval`, so its commands are `azd ai eval dataset ...`, and every user who followed one of those lines would have got "unknown command". Two of the three are mine, from earlier in this branch. They were written for the world after the dataset commands move to their own extension -- which has no registry entry, no milestone and no dependency entry, so it is not the world anyone is typing into. TestSuggestedCommandsExist is the test that exists to stop exactly this, and it missed all three because it matched on `azd ai eval` and these did not say `eval`. It searches `azd ai` now, and a namespace this extension does not serve fails by name. Reintroducing the init string makes it fail. The --from file error also lost its `` and `` placeholders, which the sibling placeholder check caught once the string came into scope. It names the command and stops; the flags are in that command's help. --- .../internal/cmd/generate.go | 4 ++-- .../internal/cmd/generate_sources_test.go | 2 +- .../azure.ai.evaluations/internal/cmd/init.go | 2 +- .../internal/cmd/init_test.go | 2 +- .../internal/cmd/surface_test.go | 22 +++++++++++++++++-- 5 files changed, 25 insertions(+), 7 deletions(-) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go index ee07d39308e..d88cc539cc1 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go @@ -239,7 +239,7 @@ func refuseUnbuildableSources(kinds []string) error { "agent": "--from agent needs a target agent; pass --target, " + "or declare one under target: in eval.yaml", "file": "--from file is not a generation source; " + - "register the file with `azd ai dataset create --file ` instead", + "register the file with `azd ai eval dataset create` instead", } messages := make([]string, 0, len(kinds)) for _, k := range kinds { @@ -285,7 +285,7 @@ func (ec *evalContext) generateDataset( return nil, fmt.Errorf("submitting the data generation job: %w", err) } if noWait { - reportSubmitted(out, "azd ai dataset", job.ID) + reportSubmitted(out, "azd ai eval dataset", job.ID) return nil, nil } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_sources_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_sources_test.go index 4165df62754..ba8d5861714 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_sources_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_sources_test.go @@ -35,7 +35,7 @@ func TestRefuseUnbuildableSources(t *testing.T) { }{ {"prompt", "--agent-instruction"}, {"agent", "--target"}, - {"file", "azd ai dataset create"}, + {"file", "azd ai eval dataset create"}, } for _, tt := range tests { diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go index 46ec0c0047c..306232e24e3 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go @@ -389,7 +389,7 @@ func (s scaffold) evaluatorNames() []string { func (s scaffold) nextSteps() []string { var steps []string if s.generateDataset { - steps = append(steps, "azd ai dataset generate "+s.datasetName) + steps = append(steps, "azd ai eval dataset generate "+s.datasetName) } if s.generateRubric { steps = append(steps, "azd ai eval evaluator generate "+s.rubricName) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_test.go index 4f714bfcf03..3b505df7f4d 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_test.go @@ -144,7 +144,7 @@ func TestScaffold_NextStepsOfferOnlyWhatIsScheduled(t *testing.T) { evalName: "support-agent-smoke", target: "support-agent", judgeModel: "m", }) require.Equal(t, []string{ - "azd ai dataset generate support-agent-smoke", + "azd ai eval dataset generate support-agent-smoke", "azd ai eval evaluator generate support-agent-quality", }, plan.nextSteps()) }) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/surface_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/surface_test.go index 6623442e1c0..bf6be972f2a 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/surface_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/surface_test.go @@ -370,9 +370,15 @@ func find(t *testing.T, path string) *cobra.Command { // it was the one thing guaranteed to fail. Nothing catches that: the string // compiles, the command that prints it succeeds, and only someone following // the advice finds out. +// This extension's namespace is `ai.eval`, so every command it can suggest +// begins `azd ai eval`. Anchoring on that prefix is what caught the renamed +// command above — and anchoring only on it is what let three suggestions +// through pointing at `azd ai dataset`, a namespace no installed extension +// serves. So the prefix checked is `azd ai`, and anything under it that is not +// this extension's own is a command nobody can run. func TestSuggestedCommandsExist(t *testing.T) { root := "../.." - pattern := regexp.MustCompile("azd ai eval ([a-z][a-z0-9-]*(?: [a-z][a-z0-9-]*)*)") + pattern := regexp.MustCompile("azd ai ([a-z][a-z0-9-]*(?: [a-z][a-z0-9-]*)*)") err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error { if err != nil { @@ -395,6 +401,18 @@ func TestSuggestedCommandsExist(t *testing.T) { } for _, m := range pattern.FindAllStringSubmatch(line, -1) { words := strings.Fields(m[1]) + + // `ai.eval` is this extension's namespace, so it is the only + // thing under `azd ai` that resolves here. Another namespace is + // a command this extension cannot suggest, whether or not some + // future extension serves it. + if len(words) == 0 || words[0] != "eval" { + t.Errorf("%s suggests `azd ai %s`, which is not this extension's "+ + "namespace; commands here are `azd ai eval ...`", path, m[1]) + continue + } + words = words[1:] + // Trim trailing prose: "run start" is a command, "run start // and summarize" is a sentence that begins with one. for len(words) > 0 { @@ -407,7 +425,7 @@ func TestSuggestedCommandsExist(t *testing.T) { words = words[:len(words)-1] } assert.NotEmpty(t, words, - "%s suggests `azd ai eval %s`, which is not a command", path, m[1]) + "%s suggests `azd ai %s`, which is not a command", path, m[1]) } } return nil From 79d2da4137a769fcfa49cc12ce6be7639b9d0e0f Mon Sep 17 00:00:00 2001 From: mohessie Date: Thu, 6 Aug 2026 04:31:15 +0300 Subject: [PATCH 115/320] Add the azure.ai.dataset extension The spec puts the dataset command groups in their own extension, since a dataset is a general Foundry asset that fine-tuning and other scenarios need as much as evaluation does. This is that extension. It is not a file move. dataset generate and dataset job call the evaluation service's generation operations, and azd extensions share no code, so the module carries its own client for them -- gen_api, the data-generation half of eval_api. The evaluator half stays behind, because only azure.ai.evaluations generates evaluators. Three things are deliberately smaller here than in the extension they came from. There is one job kind rather than two, so the jobKind indirection collapses and a not-found names the evaluator group in prose instead. There is no eval configuration, so validation and paths are local constants rather than an imported project package. And generate writes the artifact and stops. That last one is where the spec contradicts itself. Scenario 2 shows zd ai dataset generate printing "Added dataset ... to evals/eval.yaml", while the command surface section says the datasets: catalog "stays with azure.ai.eval". Both cannot hold: eval.yaml is the other extension's file and its schema would have to be duplicated here to append to it. Rather than pick silently, generate prints the eval-side command that registers the file. Raised as an open question on the spec side. Nothing is removed from azure.ai.evaluations yet. It keeps zd ai eval dataset ... working until this extension has a registry entry, because deleting the commands first would break the documented flow for anyone who has not got a second extension to install. --- .../workflows/lint-ext-azure-ai-dataset.yml | 22 + .../extensions/azure.ai.dataset/.gitignore | 5 + .../azure.ai.dataset/.golangci.yaml | 21 + .../extensions/azure.ai.dataset/CHANGELOG.md | 11 + cli/azd/extensions/azure.ai.dataset/README.md | 55 ++ cli/azd/extensions/azure.ai.dataset/build.ps1 | 78 +++ cli/azd/extensions/azure.ai.dataset/build.sh | 66 ++ .../extensions/azure.ai.dataset/ci-build.ps1 | 114 ++++ .../extensions/azure.ai.dataset/ci-test.ps1 | 56 ++ .../extensions/azure.ai.dataset/cspell.yaml | 27 + .../azure.ai.dataset/extension.yaml | 24 + cli/azd/extensions/azure.ai.dataset/go.mod | 106 +++ cli/azd/extensions/azure.ai.dataset/go.sum | 318 +++++++++ .../internal/cmd/apiversions.go | 13 + .../internal/cmd/artifacts.go | 96 +++ .../azure.ai.dataset/internal/cmd/context.go | 155 +++++ .../azure.ai.dataset/internal/cmd/dataset.go | 338 ++++++++++ .../azure.ai.dataset/internal/cmd/debug.go | 65 ++ .../azure.ai.dataset/internal/cmd/generate.go | 366 +++++++++++ .../azure.ai.dataset/internal/cmd/job.go | 197 ++++++ .../azure.ai.dataset/internal/cmd/output.go | 86 +++ .../azure.ai.dataset/internal/cmd/root.go | 62 ++ .../internal/cmd/surface_test.go | 130 ++++ .../internal/pkg/dataset_api/download_test.go | 47 ++ .../internal/pkg/dataset_api/list.go | 103 +++ .../internal/pkg/dataset_api/models.go | 210 ++++++ .../internal/pkg/dataset_api/operations.go | 622 ++++++++++++++++++ .../pkg/dataset_api/upload_version_test.go | 157 +++++ .../internal/pkg/dataset_api/uri_test.go | 75 +++ .../internal/pkg/dataset_api/version_test.go | 68 ++ .../internal/pkg/gen_api/errors.go | 49 ++ .../internal/pkg/gen_api/generation.go | 180 +++++ .../internal/pkg/gen_api/generation_test.go | 232 +++++++ .../internal/pkg/gen_api/models.go | 144 ++++ .../internal/pkg/gen_api/operations.go | 224 +++++++ .../internal/pkg/gen_api/poller.go | 193 ++++++ .../internal/version/version.go | 11 + cli/azd/extensions/azure.ai.dataset/main.go | 14 + .../extensions/azure.ai.dataset/version.txt | 1 + .../release-ext-azure-ai-dataset.yml | 45 ++ 40 files changed, 4786 insertions(+) create mode 100644 .github/workflows/lint-ext-azure-ai-dataset.yml create mode 100644 cli/azd/extensions/azure.ai.dataset/.gitignore create mode 100644 cli/azd/extensions/azure.ai.dataset/.golangci.yaml create mode 100644 cli/azd/extensions/azure.ai.dataset/CHANGELOG.md create mode 100644 cli/azd/extensions/azure.ai.dataset/README.md create mode 100644 cli/azd/extensions/azure.ai.dataset/build.ps1 create mode 100644 cli/azd/extensions/azure.ai.dataset/build.sh create mode 100644 cli/azd/extensions/azure.ai.dataset/ci-build.ps1 create mode 100644 cli/azd/extensions/azure.ai.dataset/ci-test.ps1 create mode 100644 cli/azd/extensions/azure.ai.dataset/cspell.yaml create mode 100644 cli/azd/extensions/azure.ai.dataset/extension.yaml create mode 100644 cli/azd/extensions/azure.ai.dataset/go.mod create mode 100644 cli/azd/extensions/azure.ai.dataset/go.sum create mode 100644 cli/azd/extensions/azure.ai.dataset/internal/cmd/apiversions.go create mode 100644 cli/azd/extensions/azure.ai.dataset/internal/cmd/artifacts.go create mode 100644 cli/azd/extensions/azure.ai.dataset/internal/cmd/context.go create mode 100644 cli/azd/extensions/azure.ai.dataset/internal/cmd/dataset.go create mode 100644 cli/azd/extensions/azure.ai.dataset/internal/cmd/debug.go create mode 100644 cli/azd/extensions/azure.ai.dataset/internal/cmd/generate.go create mode 100644 cli/azd/extensions/azure.ai.dataset/internal/cmd/job.go create mode 100644 cli/azd/extensions/azure.ai.dataset/internal/cmd/output.go create mode 100644 cli/azd/extensions/azure.ai.dataset/internal/cmd/root.go create mode 100644 cli/azd/extensions/azure.ai.dataset/internal/cmd/surface_test.go create mode 100644 cli/azd/extensions/azure.ai.dataset/internal/pkg/dataset_api/download_test.go create mode 100644 cli/azd/extensions/azure.ai.dataset/internal/pkg/dataset_api/list.go create mode 100644 cli/azd/extensions/azure.ai.dataset/internal/pkg/dataset_api/models.go create mode 100644 cli/azd/extensions/azure.ai.dataset/internal/pkg/dataset_api/operations.go create mode 100644 cli/azd/extensions/azure.ai.dataset/internal/pkg/dataset_api/upload_version_test.go create mode 100644 cli/azd/extensions/azure.ai.dataset/internal/pkg/dataset_api/uri_test.go create mode 100644 cli/azd/extensions/azure.ai.dataset/internal/pkg/dataset_api/version_test.go create mode 100644 cli/azd/extensions/azure.ai.dataset/internal/pkg/gen_api/errors.go create mode 100644 cli/azd/extensions/azure.ai.dataset/internal/pkg/gen_api/generation.go create mode 100644 cli/azd/extensions/azure.ai.dataset/internal/pkg/gen_api/generation_test.go create mode 100644 cli/azd/extensions/azure.ai.dataset/internal/pkg/gen_api/models.go create mode 100644 cli/azd/extensions/azure.ai.dataset/internal/pkg/gen_api/operations.go create mode 100644 cli/azd/extensions/azure.ai.dataset/internal/pkg/gen_api/poller.go create mode 100644 cli/azd/extensions/azure.ai.dataset/internal/version/version.go create mode 100644 cli/azd/extensions/azure.ai.dataset/main.go create mode 100644 cli/azd/extensions/azure.ai.dataset/version.txt create mode 100644 eng/pipelines/release-ext-azure-ai-dataset.yml diff --git a/.github/workflows/lint-ext-azure-ai-dataset.yml b/.github/workflows/lint-ext-azure-ai-dataset.yml new file mode 100644 index 00000000000..b0b4cfd13bd --- /dev/null +++ b/.github/workflows/lint-ext-azure-ai-dataset.yml @@ -0,0 +1,22 @@ +name: ext-azure-ai-dataset-ci + +on: + pull_request: + paths: + - "cli/azd/extensions/azure.ai.dataset/**" + - ".github/workflows/lint-ext-azure-ai-dataset.yml" + branches: [main] + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number }} + cancel-in-progress: true + +permissions: + contents: read + pull-requests: write # required by reusable workflow lint-go.yml + +jobs: + lint: + uses: ./.github/workflows/lint-go.yml + with: + working-directory: cli/azd/extensions/azure.ai.dataset diff --git a/cli/azd/extensions/azure.ai.dataset/.gitignore b/cli/azd/extensions/azure.ai.dataset/.gitignore new file mode 100644 index 00000000000..0d5b6d76489 --- /dev/null +++ b/cli/azd/extensions/azure.ai.dataset/.gitignore @@ -0,0 +1,5 @@ +# Test report written by ci-test.ps1 for the pipeline to publish. +junitTestReport.xml + +# Debug log written when --debug or AZD_EXT_DEBUG is set. +azd-ai-eval-*.log diff --git a/cli/azd/extensions/azure.ai.dataset/.golangci.yaml b/cli/azd/extensions/azure.ai.dataset/.golangci.yaml new file mode 100644 index 00000000000..9777522d023 --- /dev/null +++ b/cli/azd/extensions/azure.ai.dataset/.golangci.yaml @@ -0,0 +1,21 @@ +version: "2" + +linters: + default: none + enable: + - gosec + - lll + - unused + - errorlint + settings: + lll: + line-length: 220 + tab-width: 4 + gosec: + excludes: + - G204 # Subprocess launched with variable (bicep build invoked in tests) + - G304 # Potential file inclusion via variable + +formatters: + enable: + - gofmt diff --git a/cli/azd/extensions/azure.ai.dataset/CHANGELOG.md b/cli/azd/extensions/azure.ai.dataset/CHANGELOG.md new file mode 100644 index 00000000000..d4808aec590 --- /dev/null +++ b/cli/azd/extensions/azure.ai.dataset/CHANGELOG.md @@ -0,0 +1,11 @@ +# Release History + +## 1.0.0-beta.1 (Unreleased) + +### Features Added + +- Initial release. `create`, `update`, `generate`, `list`, `show`, `delete`, + `versions list`, and the `job` group for reattaching to generation jobs. +- The command groups moved here from `azure.ai.evaluations`, where they shipped + first because M1's hero scenarios needed `dataset generate` to exist before a + second extension could be published. diff --git a/cli/azd/extensions/azure.ai.dataset/README.md b/cli/azd/extensions/azure.ai.dataset/README.md new file mode 100644 index 00000000000..2059b58fcc7 --- /dev/null +++ b/cli/azd/extensions/azure.ai.dataset/README.md @@ -0,0 +1,55 @@ +# Foundry datasets (Beta) + +Create, generate and version Foundry datasets from your terminal. + +```console +$ azd extension install azure.ai.dataset +$ azd ai dataset --help +``` + +A dataset is a general Foundry asset: evaluation needs one, and so do +fine-tuning and other scenarios. That is why these commands live here rather +than inside `azure.ai.evaluations`. + +## Commands + +| Command | What it does | +|---|---| +| `azd ai dataset create --from-file ` | Register a dataset, publishing its first version | +| `azd ai dataset update --from-file ` | Publish a further version | +| `azd ai dataset generate ` | Generate a dataset and download it | +| `azd ai dataset list` | List the project's datasets | +| `azd ai dataset show ` | Show one dataset | +| `azd ai dataset delete ` | Delete a dataset version | +| `azd ai dataset versions list ` | List a dataset's versions | +| `azd ai dataset job {list,show,cancel,delete}` | Reattach to a generation job | + +## Generation + +`generate` submits a job to the evaluation service and downloads what it +produced, so this extension speaks to two APIs: datasets are registered through +the dataset API but generated by the evaluation service. + +`--from` selects one or more of the sources the service accepts, and is +repeatable. It defaults to `traces` when the project has Application Insights +connected — real conversations make a better dataset than synthesized ones — +and to `agent` otherwise. + +Registering a generated file in an evaluation configuration is a separate step: +`evals/eval.yaml` belongs to `azure.ai.evaluations`, so `generate` prints the +command rather than writing that file itself. + +## Project endpoint + +Every command resolves the Foundry project endpoint in this order: + +1. `--project-endpoint` +2. `FOUNDRY_PROJECT_ENDPOINT` in the active azd environment +3. the host environment variable of the same name + +## Building + +```console +$ go build ./... +$ go test ./... +``` diff --git a/cli/azd/extensions/azure.ai.dataset/build.ps1 b/cli/azd/extensions/azure.ai.dataset/build.ps1 new file mode 100644 index 00000000000..f37f80cabf0 --- /dev/null +++ b/cli/azd/extensions/azure.ai.dataset/build.ps1 @@ -0,0 +1,78 @@ +# Ensure script fails on any error +$ErrorActionPreference = 'Stop' + +# Get the directory of the script +$EXTENSION_DIR = Split-Path -Parent $MyInvocation.MyCommand.Path + +# Change to the script directory +Set-Location -Path $EXTENSION_DIR + +# Create a safe version of EXTENSION_ID replacing dots with dashes +$EXTENSION_ID_SAFE = $env:EXTENSION_ID -replace '\.', '-' + +# Define output directory +$OUTPUT_DIR = if ($env:OUTPUT_DIR) { $env:OUTPUT_DIR } else { Join-Path $EXTENSION_DIR "bin" } + +# Create output directory if it doesn't exist +if (-not (Test-Path -Path $OUTPUT_DIR)) { + New-Item -ItemType Directory -Path $OUTPUT_DIR | Out-Null +} + +# Get Git commit hash and build date +$COMMIT = git rev-parse HEAD +if ($LASTEXITCODE -ne 0) { + Write-Host "Error: Failed to get git commit hash" + exit 1 +} +$BUILD_DATE = (Get-Date -Format "yyyy-MM-ddTHH:mm:ssZ") + +# List of OS and architecture combinations +if ($env:EXTENSION_PLATFORM) { + $PLATFORMS = @($env:EXTENSION_PLATFORM) +} +else { + $PLATFORMS = @( + "windows/amd64", + "windows/arm64", + "darwin/amd64", + "darwin/arm64", + "linux/amd64", + "linux/arm64" + ) +} + +$VERSION_PATH = "azureaieval/internal/version" + +# Loop through platforms and build +foreach ($PLATFORM in $PLATFORMS) { + $OS, $ARCH = $PLATFORM -split '/' + + $OUTPUT_NAME = Join-Path $OUTPUT_DIR "$EXTENSION_ID_SAFE-$OS-$ARCH" + + if ($OS -eq "windows") { + $OUTPUT_NAME += ".exe" + } + + Write-Host "Building for $OS/$ARCH..." + + # Delete the output file if it already exists + if (Test-Path -Path $OUTPUT_NAME) { + Remove-Item -Path $OUTPUT_NAME -Force + } + + # Set environment variables for Go build + $env:GOOS = $OS + $env:GOARCH = $ARCH + + go build ` + -ldflags="-X '$VERSION_PATH.Version=$env:EXTENSION_VERSION' -X '$VERSION_PATH.Commit=$COMMIT' -X '$VERSION_PATH.BuildDate=$BUILD_DATE'" ` + -o $OUTPUT_NAME + + if ($LASTEXITCODE -ne 0) { + Write-Host "An error occurred while building for $OS/$ARCH" + exit 1 + } +} + +Write-Host "Build completed successfully!" +Write-Host "Binaries are located in the $OUTPUT_DIR directory." diff --git a/cli/azd/extensions/azure.ai.dataset/build.sh b/cli/azd/extensions/azure.ai.dataset/build.sh new file mode 100644 index 00000000000..4165a516ac4 --- /dev/null +++ b/cli/azd/extensions/azure.ai.dataset/build.sh @@ -0,0 +1,66 @@ +#!/bin/bash + +# Get the directory of the script +EXTENSION_DIR="$(cd "$(dirname "$0")" && pwd)" + +# Change to the script directory +cd "$EXTENSION_DIR" || exit + +# Create a safe version of EXTENSION_ID replacing dots with dashes +EXTENSION_ID_SAFE="${EXTENSION_ID//./-}" + +# Define output directory +OUTPUT_DIR="${OUTPUT_DIR:-$EXTENSION_DIR/bin}" + +# Create output and target directories if they don't exist +mkdir -p "$OUTPUT_DIR" + +# Get Git commit hash and build date +COMMIT=$(git rev-parse HEAD) +BUILD_DATE=$(date -u +%Y-%m-%dT%H:%M:%SZ) + +# List of OS and architecture combinations +if [ -n "$EXTENSION_PLATFORM" ]; then + PLATFORMS=("$EXTENSION_PLATFORM") +else + PLATFORMS=( + "windows/amd64" + "windows/arm64" + "darwin/amd64" + "darwin/arm64" + "linux/amd64" + "linux/arm64" + ) +fi + +VERSION_PATH="azureaieval/internal/version" + +# Loop through platforms and build +for PLATFORM in "${PLATFORMS[@]}"; do + OS=$(echo "$PLATFORM" | cut -d'/' -f1) + ARCH=$(echo "$PLATFORM" | cut -d'/' -f2) + + OUTPUT_NAME="$OUTPUT_DIR/$EXTENSION_ID_SAFE-$OS-$ARCH" + + if [ "$OS" = "windows" ]; then + OUTPUT_NAME+='.exe' + fi + + echo "Building for $OS/$ARCH..." + + # Delete the output file if it already exists + [ -f "$OUTPUT_NAME" ] && rm -f "$OUTPUT_NAME" + + # Set environment variables for Go build + GOOS=$OS GOARCH=$ARCH go build \ + -ldflags="-X '$VERSION_PATH.Version=$EXTENSION_VERSION' -X '$VERSION_PATH.Commit=$COMMIT' -X '$VERSION_PATH.BuildDate=$BUILD_DATE'" \ + -o "$OUTPUT_NAME" + + if [ $? -ne 0 ]; then + echo "An error occurred while building for $OS/$ARCH" + exit 1 + fi +done + +echo "Build completed successfully!" +echo "Binaries are located in the $OUTPUT_DIR directory." diff --git a/cli/azd/extensions/azure.ai.dataset/ci-build.ps1 b/cli/azd/extensions/azure.ai.dataset/ci-build.ps1 new file mode 100644 index 00000000000..403bc23b08d --- /dev/null +++ b/cli/azd/extensions/azure.ai.dataset/ci-build.ps1 @@ -0,0 +1,114 @@ +param( + [string] $Version = (Get-Content "$PSScriptRoot/version.txt"), + [string] $SourceVersion = (git rev-parse HEAD), + [switch] $CodeCoverageEnabled, + # Accepted because the shared CI template always passes it. This extension + # has no record/playback mode, so there is no second binary to produce. + [switch] $BuildRecordMode, + [string] $MSYS2Shell, # path to msys2_shell.cmd + [string] $OutputFileName +) +$PSNativeCommandArgumentPassing = 'Legacy' + +# Remove any previously built binaries. +go clean + +if ($LASTEXITCODE) { + Write-Host "Error running go clean" + exit $LASTEXITCODE +} + +# Run `go help build` for detail on these flags. +$buildFlags = @( + # Remove file system paths from the binary. Recorded file names become a + # module path@version, or a plain import path for the standard library. + "-trimpath", + + # Position Independent Executable, for memory-corruption hardening across + # platforms. On Windows this enables ASLR and sets DYNAMICBASE and + # HIGH-ENTROPY-VA in the PE header. + "-buildmode=pie" +) + +if ($CodeCoverageEnabled) { + $buildFlags += "-cover" +} + +# cfi: Control Flow Integrity, cfg: Control Flow Guard, +# osusergo: use the pure Go user lookup. +$tagsFlag = "-tags=cfi,cfg,osusergo" + +# -s: omit the symbol table, -w: omit DWARF, -X: set a variable at link time. +$ldFlag = "-ldflags=-s -w " + + "-X 'azureaieval/internal/version.Version=$Version' " + + "-X 'azureaieval/internal/version.Commit=$SourceVersion' " + + "-X 'azureaieval/internal/version.BuildDate=$(Get-Date -Format o)' " + +if ($IsWindows) { + Write-Host "Building for Windows" +} +elseif ($IsLinux) { + Write-Host "Building for linux" + + # Disable cgo for the x64 Linux build. This also links statically, which + # widens compatibility with older Linux distributions. + if ($env:GOARCH -ne "arm64") { + $env:CGO_ENABLED = "0" + } +} +elseif ($IsMacOS) { + Write-Host "Building for macOS" +} + +$outputFlag = "-o=$OutputFileName" + +$buildFlags += @( + $tagsFlag, + $ldFlag, + $outputFlag +) + +function PrintFlags() { + param( + [string] $flags + ) + + # Format the flags so they can be pasted straight into pwsh. + $i = 0 + foreach ($buildFlag in $buildFlags) { + # Quote values so characters such as ',' survive a repaste. Not needed + # for the direct invocation below. + $argWithValue = $buildFlag.Split('=', 2) + if ($argWithValue.Length -eq 2 -and !$argWithValue[1].StartsWith("`"")) { + $buildFlag = "$($argWithValue[0])=`"$($argWithValue[1])`"" + } + + if ($i -eq $buildFlags.Length - 1) { + Write-Host " $buildFlag" + } + else { + Write-Host " $buildFlag ``" + } + $i++ + } +} + +$oldGOEXPERIMENT = $env:GOEXPERIMENT +# Opt into per-iteration loop variables, which is what most readers expect and +# what the Go team intends to make the default. +$env:GOEXPERIMENT = "loopvar" + +try { + Write-Host "Running: go build ``" + PrintFlags -flags $buildFlags + go build @buildFlags + if ($LASTEXITCODE) { + Write-Host "Error running go build" + exit $LASTEXITCODE + } + + Write-Host "go build succeeded" +} +finally { + $env:GOEXPERIMENT = $oldGOEXPERIMENT +} diff --git a/cli/azd/extensions/azure.ai.dataset/ci-test.ps1 b/cli/azd/extensions/azure.ai.dataset/ci-test.ps1 new file mode 100644 index 00000000000..6175585d318 --- /dev/null +++ b/cli/azd/extensions/azure.ai.dataset/ci-test.ps1 @@ -0,0 +1,56 @@ +# Runs the unit tests and writes a JUnit report. +# +# The pipeline publishes **/junitTestReport.xml from the extension directory, +# so the report has to be written under that name for results to show up in the +# build. gotestsum produces it; the go test fallback does not, so the fallback +# only runs when gotestsum is unavailable. +# +# The live integration tests are excluded: they carry the `live` build tag, so +# an untagged run does not compile them, and they additionally require +# AZURE_AI_EVAL_E2E_LIVE and a project endpoint. They are still type-checked +# below, so a change that breaks them cannot reach main unnoticed. +# +# TODO before the first release: PR CI runs this script on windows, linux and +# darwin amd64, so the untagged tests are covered on all three. The live and +# hero suites are only type-checked, never executed, and both have only ever +# run on Windows by hand. Run them once on linux, where they assume a path +# separator and shell out to `azd` and to a proxy address. + +$gopath = go env GOPATH +$gotestsumBinary = "gotestsum" +if ($IsWindows) { + $gotestsumBinary += ".exe" +} +$gotestsum = Join-Path $gopath "bin" $gotestsumBinary + +Write-Host "Running unit tests..." + +if (Test-Path $gotestsum) { + & $gotestsum --format testname --junitfile junitTestReport.xml -- ./... -count=1 +} else { + Write-Host "gotestsum not found; falling back to go test (no JUnit report)." -ForegroundColor Yellow + go test ./... -v -count=1 +} + +if ($LASTEXITCODE -ne 0) { + Write-Host "" + Write-Host "Tests failed with exit code: $LASTEXITCODE" -ForegroundColor Red + exit $LASTEXITCODE +} + +# The tagged suites are never run here, so without this nothing compiles them +# and a change that breaks one reaches main silently. Type-checking needs no +# credentials, so it costs a few seconds and runs everywhere the tests do. +Write-Host "" +Write-Host "Type-checking the live and hero suites..." +go vet -tags live,hero ./... + +if ($LASTEXITCODE -ne 0) { + Write-Host "" + Write-Host "The tagged test suites do not compile: $LASTEXITCODE" -ForegroundColor Red + exit $LASTEXITCODE +} + +Write-Host "" +Write-Host "All tests passed!" -ForegroundColor Green +exit 0 diff --git a/cli/azd/extensions/azure.ai.dataset/cspell.yaml b/cli/azd/extensions/azure.ai.dataset/cspell.yaml new file mode 100644 index 00000000000..4d6c8ad651b --- /dev/null +++ b/cli/azd/extensions/azure.ai.dataset/cspell.yaml @@ -0,0 +1,27 @@ +import: ../../.vscode/cspell.yaml +words: + # Go module and package names + - azureaieval + - evalcore + - httptest + - creack + # Service identifiers and API fields + - evalrun + - lookback + - AOAI + # Built-in evaluator names + - ifeval + - groundedness + # Repository names + - foundrysdk + # Terms + - inlines + - negotiables + - parseable + - retargeted + - subsetting + - undeployed + - undoable + - unpassed + - unscored + - Unparseable diff --git a/cli/azd/extensions/azure.ai.dataset/extension.yaml b/cli/azd/extensions/azure.ai.dataset/extension.yaml new file mode 100644 index 00000000000..7616c3ee65d --- /dev/null +++ b/cli/azd/extensions/azure.ai.dataset/extension.yaml @@ -0,0 +1,24 @@ +# yaml-language-server: $schema=../extension.schema.json +id: azure.ai.dataset +namespace: ai.dataset +displayName: Foundry datasets (Beta) +description: Create, generate and version Foundry datasets from your terminal. (Beta) +usage: azd ai dataset [options] +# NOTE: Make sure version.txt is in sync with this version. +version: 1.0.0-beta.1 +requiredAzdVersion: ">=1.27.1" +language: go +capabilities: + - custom-commands + - metadata +examples: + - name: create + description: Register a dataset from a local file. + usage: azd ai dataset create support-regression --from-file ./data/golden.jsonl + - name: generate + description: Generate a dataset from an agent's traces. + usage: azd ai dataset generate support-regression --from traces +tags: + - ai + - foundry + - dataset diff --git a/cli/azd/extensions/azure.ai.dataset/go.mod b/cli/azd/extensions/azure.ai.dataset/go.mod new file mode 100644 index 00000000000..d995ace863a --- /dev/null +++ b/cli/azd/extensions/azure.ai.dataset/go.mod @@ -0,0 +1,106 @@ +module azureaidataset + +go 1.26.4 + +require ( + github.com/Azure/azure-sdk-for-go/sdk/azcore v1.21.0 + github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.14.0-beta.3 + github.com/azure/azure-dev/cli/azd v1.28.0 + github.com/fatih/color v1.18.0 + github.com/google/uuid v1.6.0 + github.com/spf13/cobra v1.10.1 + github.com/stretchr/testify v1.11.1 + go.yaml.in/yaml/v3 v3.0.4 + google.golang.org/protobuf v1.36.11 +) + +require ( + dario.cat/mergo v1.0.2 // indirect + github.com/AlecAivazis/survey/v2 v2.3.7 // indirect + github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2 // indirect + github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/appservice/armappservice/v2 v2.3.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/keyvault/armkeyvault v1.5.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armsubscriptions v1.3.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azsecrets v1.4.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/internal v1.2.0 // indirect + github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0 // indirect + github.com/Masterminds/semver/v3 v3.4.0 // indirect + github.com/adam-lavrik/go-imath v0.0.0-20210910152346-265a42a96f0b // indirect + github.com/alecthomas/chroma/v2 v2.20.0 // indirect + github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect + github.com/aymerick/douceur v0.2.0 // indirect + github.com/bahlo/generic-list-go v0.2.0 // indirect + github.com/blang/semver/v4 v4.0.0 // indirect + github.com/braydonk/yaml v0.9.0 // indirect + github.com/buger/goterm v1.0.4 // indirect + github.com/buger/jsonparser v1.1.2 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/charmbracelet/colorprofile v0.3.2 // indirect + github.com/charmbracelet/glamour v0.10.0 // indirect + github.com/charmbracelet/lipgloss v1.1.1-0.20250404203927-76690c660834 // indirect + github.com/charmbracelet/x/ansi v0.10.2 // indirect + github.com/charmbracelet/x/cellbuf v0.0.13 // indirect + github.com/charmbracelet/x/exp/slice v0.0.0-20251008171431-5d3777519489 // indirect + github.com/charmbracelet/x/term v0.2.1 // indirect + github.com/cli/browser v1.3.0 // indirect + github.com/clipperhouse/uax29/v2 v2.2.0 // indirect + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + github.com/dlclark/regexp2 v1.11.5 // indirect + github.com/drone/envsubst v1.0.3 // indirect + github.com/go-logr/logr v1.4.3 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/gofrs/flock v0.12.1 // indirect + github.com/golang-jwt/jwt/v5 v5.3.0 // indirect + github.com/golobby/container/v3 v3.3.2 // indirect + github.com/gorilla/css v1.0.1 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/invopop/jsonschema v0.13.0 // indirect + github.com/jmespath-community/go-jmespath v1.1.1 // indirect + github.com/joho/godotenv v1.5.1 // indirect + github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect + github.com/kylelemons/godebug v1.1.0 // indirect + github.com/lucasb-eyer/go-colorful v1.3.0 // indirect + github.com/mailru/easyjson v0.9.1 // indirect + github.com/mark3labs/mcp-go v0.41.1 // indirect + github.com/mattn/go-colorable v0.1.14 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect + github.com/mattn/go-runewidth v0.0.19 // indirect + github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d // indirect + github.com/microcosm-cc/bluemonday v1.0.27 // indirect + github.com/microsoft/ApplicationInsights-Go v0.4.4 // indirect + github.com/microsoft/go-deviceid v1.0.0 // indirect + github.com/muesli/reflow v0.3.0 // indirect + github.com/muesli/termenv v0.16.0 // indirect + github.com/nathan-fiscaletti/consolesize-go v0.0.0-20220204101620-317176b6684d // indirect + github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect + github.com/rivo/uniseg v0.4.7 // indirect + github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 // indirect + github.com/sethvargo/go-retry v0.3.0 // indirect + github.com/spf13/cast v1.10.0 // indirect + github.com/spf13/pflag v1.0.10 // indirect + github.com/theckman/yacspin v0.13.12 // indirect + github.com/wk8/go-ordered-map/v2 v2.1.8 // indirect + github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect + github.com/yosida95/uritemplate/v3 v3.0.2 // indirect + github.com/yuin/goldmark v1.7.13 // indirect + github.com/yuin/goldmark-emoji v1.0.6 // indirect + go.opentelemetry.io/auto/sdk v1.2.1 // indirect + go.opentelemetry.io/otel v1.43.0 // indirect + go.opentelemetry.io/otel/metric v1.43.0 // indirect + go.opentelemetry.io/otel/sdk v1.43.0 // indirect + go.opentelemetry.io/otel/trace v1.43.0 // indirect + go.uber.org/atomic v1.11.0 // indirect + go.uber.org/multierr v1.11.0 // indirect + golang.org/x/crypto v0.53.0 // indirect + golang.org/x/exp v0.0.0-20250911091902-df9299821621 // indirect + golang.org/x/net v0.56.0 // indirect + golang.org/x/sync v0.21.0 // indirect + golang.org/x/sys v0.46.0 // indirect + golang.org/x/term v0.44.0 // indirect + golang.org/x/text v0.38.0 // indirect + golang.org/x/time v0.9.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260401024825-9d38bb4040a9 // indirect + google.golang.org/grpc v1.80.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/cli/azd/extensions/azure.ai.dataset/go.sum b/cli/azd/extensions/azure.ai.dataset/go.sum new file mode 100644 index 00000000000..81c10e45793 --- /dev/null +++ b/cli/azd/extensions/azure.ai.dataset/go.sum @@ -0,0 +1,318 @@ +code.cloudfoundry.org/clock v0.0.0-20180518195852-02e53af36e6c/go.mod h1:QD9Lzhd/ux6eNQVUDVRJX/RKTigpewimNYBi7ivZKY8= +dario.cat/mergo v1.0.2 h1:85+piFYR1tMbRrLcDwR18y4UKJ3aH1Tbzi24VRW1TK8= +dario.cat/mergo v1.0.2/go.mod h1:E/hbnu0NxMFBjpMIE34DRGLWqDy0g5FuKDhCb31ngxA= +github.com/AlecAivazis/survey/v2 v2.3.7 h1:6I/u8FvytdGsgonrYsVn2t8t4QiRnh6QSTqkkhIiSjQ= +github.com/AlecAivazis/survey/v2 v2.3.7/go.mod h1:xUTIdE4KCOIjsBAE1JYsUPoCqYdZ1reCfTwbto0Fduo= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.21.0 h1:fou+2+WFTib47nS+nz/ozhEBnvU96bKHy6LjRsY4E28= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.21.0/go.mod h1:t76Ruy8AHvUAC8GfMWJMa0ElSbuIcO03NLpynfbgsPA= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.14.0-beta.3 h1:0g4UTtvRA9goC37cmD9ZHdW6CCNJR4cOXBnHz0r4ubM= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.14.0-beta.3/go.mod h1:fEiHi0sbYqbo3shUkIF1SNxm8GyeEJl+Poc/djOvbdE= +github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.3.2 h1:yz1bePFlP5Vws5+8ez6T3HWXPmwOK7Yvq8QxDBD3SKY= +github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.3.2/go.mod h1:Pa9ZNPuoNu/GztvBSKk9J1cDJW6vk/n0zLtV4mgd8N8= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2 h1:9iefClla7iYpfYWdzPCRDozdmndjTm8DXdpCzPajMgA= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2/go.mod h1:XtLgD3ZD34DAaVIIAyG3objl5DynM3CQ/vMcbBNJZGI= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/appservice/armappservice/v2 v2.3.0 h1:JI8PcWOImyvIUEZ0Bbmfe05FOlWkMi2KhjG+cAKaUms= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/appservice/armappservice/v2 v2.3.0/go.mod h1:nJLFPGJkyKfDDyJiPuHIXsCi/gpJkm07EvRgiX7SGlI= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/internal/v2 v2.0.0 h1:PTFGRSlMKCQelWwxUyYVEUqseBJVemLyqWJjvMyt0do= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/internal/v2 v2.0.0/go.mod h1:LRr2FzBTQlONPPa5HREE5+RjSCTXl7BwOvYOaWTqCaI= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/internal/v3 v3.1.0 h1:2qsIIvxVT+uE6yrNldntJKlLRgxGbZ85kgtz5SNBhMw= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/internal/v3 v3.1.0/go.mod h1:AW8VEadnhw9xox+VaVd9sP7NjzOAnaZBLRH6Tq3cJ38= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/keyvault/armkeyvault v1.5.0 h1:nnQ9vXH039UrEFxi08pPuZBE7VfqSJt343uJLw0rhWI= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/keyvault/armkeyvault v1.5.0/go.mod h1:4YIVtzMFVsPwBvitCDX7J9sqthSj43QD1sP6fYc1egc= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.2.0 h1:Dd+RhdJn0OTtVGaeDLZpcumkIVCtA/3/Fo42+eoYvVM= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.2.0/go.mod h1:5kakwfW5CjC9KK+Q4wjXAg+ShuIm2mBMua0ZFj2C8PE= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armsubscriptions v1.3.0 h1:wxQx2Bt4xzPIKvW59WQf1tJNx/ZZKPfN+EhPX3Z6CYY= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armsubscriptions v1.3.0/go.mod h1:TpiwjwnW/khS0LKs4vW5UmmT9OWcxaveS8U7+tlknzo= +github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azsecrets v1.4.0 h1:/g8S6wk65vfC6m3FIxJ+i5QDyN9JWwXI8Hb0Img10hU= +github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azsecrets v1.4.0/go.mod h1:gpl+q95AzZlKVI3xSoseF9QPrypk0hQqBiJYeB/cR/I= +github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/internal v1.2.0 h1:nCYfgcSyHZXJI8J0IWE5MsCGlb2xp9fJiXyxWgmOFg4= +github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/internal v1.2.0/go.mod h1:ucUjca2JtSZboY8IoUqyQyuuXvwbMBVwFOm0vdQPNhA= +github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1 h1:WJTmL004Abzc5wDB5VtZG2PJk5ndYDgVacGqfirKxjM= +github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1/go.mod h1:tCcJZ0uHAmvjsVYzEFivsRTN00oz5BEsRgQHu5JZ9WE= +github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0 h1:XRzhVemXdgvJqCH0sFfrBUTnUJSBrBf7++ypk+twtRs= +github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0/go.mod h1:HKpQxkWaGLJ+D/5H8QRpyQXA1eKjxkFlOMwck5+33Jk= +github.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1Xbatp0= +github.com/Masterminds/semver/v3 v3.4.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= +github.com/Netflix/go-expect v0.0.0-20220104043353-73e0943537d2 h1:+vx7roKuyA63nhn5WAunQHLTznkw5W8b1Xc0dNjp83s= +github.com/Netflix/go-expect v0.0.0-20220104043353-73e0943537d2/go.mod h1:HBCaDeC1lPdgDeDbhX8XFpy1jqjK0IBG8W5K+xYqA0w= +github.com/adam-lavrik/go-imath v0.0.0-20210910152346-265a42a96f0b h1:g9SuFmxM/WucQFKTMSP+irxyf5m0RiUJreBDhGI6jSA= +github.com/adam-lavrik/go-imath v0.0.0-20210910152346-265a42a96f0b/go.mod h1:XjvqMUpGd3Xn9Jtzk/4GEBCSoBX0eB2RyriXgne0IdM= +github.com/alecthomas/assert/v2 v2.11.0 h1:2Q9r3ki8+JYXvGsDyBXwH3LcJ+WK5D0gc5E8vS6K3D0= +github.com/alecthomas/assert/v2 v2.11.0/go.mod h1:Bze95FyfUr7x34QZrjL+XP+0qgp/zg8yS+TtBj1WA3k= +github.com/alecthomas/chroma/v2 v2.20.0 h1:sfIHpxPyR07/Oylvmcai3X/exDlE8+FA820NTz+9sGw= +github.com/alecthomas/chroma/v2 v2.20.0/go.mod h1:e7tViK0xh/Nf4BYHl00ycY6rV7b8iXBksI9E359yNmA= +github.com/alecthomas/repr v0.5.1 h1:E3G4t2QbHTSNpPKBgMTln5KLkZHLOcU7r37J4pXBuIg= +github.com/alecthomas/repr v0.5.1/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4= +github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k= +github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8= +github.com/aymanbagabas/go-udiff v0.2.0 h1:TK0fH4MteXUDspT88n8CKzvK0X9O2xu9yQjWpi6yML8= +github.com/aymanbagabas/go-udiff v0.2.0/go.mod h1:RE4Ex0qsGkTAJoQdQQCA0uG+nAzJO/pI/QwceO5fgrA= +github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= +github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4= +github.com/azure/azure-dev/cli/azd v1.28.0 h1:mqqyV85m7A1XfWJFjV/Ut0QoIEImFeF++1Ruq/cRp0s= +github.com/azure/azure-dev/cli/azd v1.28.0/go.mod h1:Ge7QaU9PoJM7i6J0xArDoQCf2tUn6O7OIKkoItxFTA8= +github.com/bahlo/generic-list-go v0.2.0 h1:5sz/EEAK+ls5wF+NeqDpk5+iNdMDXrh3z3nPnH1Wvgk= +github.com/bahlo/generic-list-go v0.2.0/go.mod h1:2KvAjgMlE5NNynlg/5iLrrCCZ2+5xWbdbCW3pNTGyYg= +github.com/benbjohnson/clock v1.3.5 h1:VvXlSJBzZpA/zum6Sj74hxwYI2DIxRWuNIoXAzHZz5o= +github.com/benbjohnson/clock v1.3.5/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= +github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= +github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= +github.com/bradleyjkemp/cupaloy/v2 v2.8.0 h1:any4BmKE+jGIaMpnU8YgH/I2LPiLBufr6oMMlVBbn9M= +github.com/bradleyjkemp/cupaloy/v2 v2.8.0/go.mod h1:bm7JXdkRd4BHJk9HpwqAI8BoAY1lps46Enkdqw6aRX0= +github.com/braydonk/yaml v0.9.0 h1:ewGMrVmEVpsm3VwXQDR388sLg5+aQ8Yihp6/hc4m+h4= +github.com/braydonk/yaml v0.9.0/go.mod h1:hcm3h581tudlirk8XEUPDBAimBPbmnL0Y45hCRl47N4= +github.com/buger/goterm v1.0.4 h1:Z9YvGmOih81P0FbVtEYTFF6YsSgxSUKEhf/f9bTMXbY= +github.com/buger/goterm v1.0.4/go.mod h1:HiFWV3xnkolgrBV3mY8m0X0Pumt4zg4QhbdOzQtB8tE= +github.com/buger/jsonparser v1.1.2 h1:frqHqw7otoVbk5M8LlE/L7HTnIq2v9RX6EJ48i9AxJk= +github.com/buger/jsonparser v1.1.2/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/charmbracelet/colorprofile v0.3.2 h1:9J27WdztfJQVAQKX2WOlSSRB+5gaKqqITmrvb1uTIiI= +github.com/charmbracelet/colorprofile v0.3.2/go.mod h1:mTD5XzNeWHj8oqHb+S1bssQb7vIHbepiebQ2kPKVKbI= +github.com/charmbracelet/glamour v0.10.0 h1:MtZvfwsYCx8jEPFJm3rIBFIMZUfUJ765oX8V6kXldcY= +github.com/charmbracelet/glamour v0.10.0/go.mod h1:f+uf+I/ChNmqo087elLnVdCiVgjSKWuXa/l6NU2ndYk= +github.com/charmbracelet/lipgloss v1.1.1-0.20250404203927-76690c660834 h1:ZR7e0ro+SZZiIZD7msJyA+NjkCNNavuiPBLgerbOziE= +github.com/charmbracelet/lipgloss v1.1.1-0.20250404203927-76690c660834/go.mod h1:aKC/t2arECF6rNOnaKaVU6y4t4ZeHQzqfxedE/VkVhA= +github.com/charmbracelet/x/ansi v0.10.2 h1:ith2ArZS0CJG30cIUfID1LXN7ZFXRCww6RUvAPA+Pzw= +github.com/charmbracelet/x/ansi v0.10.2/go.mod h1:HbLdJjQH4UH4AqA2HpRWuWNluRE6zxJH/yteYEYCFa8= +github.com/charmbracelet/x/cellbuf v0.0.13 h1:/KBBKHuVRbq1lYx5BzEHBAFBP8VcQzJejZ/IA3iR28k= +github.com/charmbracelet/x/cellbuf v0.0.13/go.mod h1:xe0nKWGd3eJgtqZRaN9RjMtK7xUYchjzPr7q6kcvCCs= +github.com/charmbracelet/x/exp/golden v0.0.0-20240806155701-69247e0abc2a h1:G99klV19u0QnhiizODirwVksQB91TJKV/UaTnACcG30= +github.com/charmbracelet/x/exp/golden v0.0.0-20240806155701-69247e0abc2a/go.mod h1:wDlXFlCrmJ8J+swcL/MnGUuYnqgQdW9rhSD61oNMb6U= +github.com/charmbracelet/x/exp/slice v0.0.0-20251008171431-5d3777519489 h1:a5q2sWiet6kgqucSGjYN1jhT2cn4bMKUwprtm2IGRto= +github.com/charmbracelet/x/exp/slice v0.0.0-20251008171431-5d3777519489/go.mod h1:vqEfX6xzqW1pKKZUUiFOKg0OQ7bCh54Q2vR/tserrRA= +github.com/charmbracelet/x/term v0.2.1 h1:AQeHeLZ1OqSXhrAWpYUtZyX1T3zVxfpZuEQMIQaGIAQ= +github.com/charmbracelet/x/term v0.2.1/go.mod h1:oQ4enTYFV7QN4m0i9mzHrViD7TQKvNEEkHUMCmsxdUg= +github.com/cli/browser v1.3.0 h1:LejqCrpWr+1pRqmEPDGnTZOjsMe7sehifLynZJuqJpo= +github.com/cli/browser v1.3.0/go.mod h1:HH8s+fOAxjhQoBUAsKuPCbqUuxZDhQ2/aD+SzsEfBTk= +github.com/clipperhouse/uax29/v2 v2.2.0 h1:ChwIKnQN3kcZteTXMgb1wztSgaU+ZemkgWdohwgs8tY= +github.com/clipperhouse/uax29/v2 v2.2.0/go.mod h1:EFJ2TJMRUaplDxHKj1qAEhCtQPW2tJSwu5BF98AuoVM= +github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= +github.com/creack/pty v1.1.17 h1:QeVUsEDNrLBW4tMgZHvxy18sKtr6VI492kBhUfhDJNI= +github.com/creack/pty v1.1.17/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dlclark/regexp2 v1.11.5 h1:Q/sSnsKerHeCkc/jSTNq1oCm7KiVgUMZRDUoRu0JQZQ= +github.com/dlclark/regexp2 v1.11.5/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= +github.com/drone/envsubst v1.0.3 h1:PCIBwNDYjs50AsLZPYdfhSATKaRg/FJmDc2D6+C2x8g= +github.com/drone/envsubst v1.0.3/go.mod h1:N2jZmlMufstn1KEqvbHjw40h1KyTmnVzHcSc9bFiJ2g= +github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM= +github.com/fatih/color v1.18.0/go.mod h1:4FelSpRwEGDpQ12mAdzqdOukCy4u8WUtOY6lkT/6HfU= +github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= +github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/gofrs/flock v0.12.1 h1:MTLVXXHf8ekldpJk3AKicLij9MdwOWkZ+a/jHHZby9E= +github.com/gofrs/flock v0.12.1/go.mod h1:9zxTsyu5xtJ9DK+1tFZyibEV7y3uwDxPPfbxeeHCoD0= +github.com/gofrs/uuid v3.3.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= +github.com/golang-jwt/jwt/v5 v5.3.0 h1:pv4AsKCKKZuqlgs5sUmn4x8UlGa0kEVt/puTpKx9vvo= +github.com/golang-jwt/jwt/v5 v5.3.0/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/golobby/container/v3 v3.3.2 h1:7u+RgNnsdVlhGoS8gY4EXAG601vpMMzLZlYqSp77Quw= +github.com/golobby/container/v3 v3.3.2/go.mod h1:RDdKpnKpV1Of11PFBe7Dxc2C1k2KaLE4FD47FflAmj0= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/gorilla/css v1.0.1 h1:ntNaBIghp6JmvWnxbZKANoLyuXTPZ4cAMlo6RyhlbO8= +github.com/gorilla/css v1.0.1/go.mod h1:BvnYkspnSzMmwRK+b8/xgNPLiIuNZr6vbZBTPQ2A3b0= +github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM= +github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg= +github.com/hinshun/vt10x v0.0.0-20220119200601-820417d04eec h1:qv2VnGeEQHchGaZ/u7lxST/RaJw+cv273q79D81Xbog= +github.com/hinshun/vt10x v0.0.0-20220119200601-820417d04eec/go.mod h1:Q48J4R4DvxnHolD5P8pOtXigYlRuPLGl6moFx3ulM68= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/invopop/jsonschema v0.13.0 h1:KvpoAJWEjR3uD9Kbm2HWJmqsEaHt8lBUpd0qHcIi21E= +github.com/invopop/jsonschema v0.13.0/go.mod h1:ffZ5Km5SWWRAIN6wbDXItl95euhFz2uON45H2qjYt+0= +github.com/jmespath-community/go-jmespath v1.1.1 h1:bFikPhsi/FdmlZhVgSCd2jj1e7G/rw+zyQfyg5UF+L4= +github.com/jmespath-community/go-jmespath v1.1.1/go.mod h1:4gOyFJsR/Gk+05RgTKYrifT7tBPWD8Lubtb5jRrfy9I= +github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= +github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= +github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs= +github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8= +github.com/keybase/go-keychain v0.0.1 h1:way+bWYa6lDppZoZcgMbYsvC7GxljxrskdNInRtuthU= +github.com/keybase/go-keychain v0.0.1/go.mod h1:PdEILRW3i9D8JcdM+FmY6RwkHGnhHxXwkPPMeUgOK1k= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/lucasb-eyer/go-colorful v1.3.0 h1:2/yBRLdWBZKrf7gB40FoiKfAWYQ0lqNcbuQwVHXptag= +github.com/lucasb-eyer/go-colorful v1.3.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= +github.com/mailru/easyjson v0.9.1 h1:LbtsOm5WAswyWbvTEOqhypdPeZzHavpZx96/n553mR8= +github.com/mailru/easyjson v0.9.1/go.mod h1:1+xMtQp2MRNVL/V1bOzuP3aP8VNwRW55fQUto+XFtTU= +github.com/mark3labs/mcp-go v0.41.1 h1:w78eWfiQam2i8ICL7AL0WFiq7KHNJQ6UB53ZVtH4KGA= +github.com/mark3labs/mcp-go v0.41.1/go.mod h1:T7tUa2jO6MavG+3P25Oy/jR7iCeJPHImCZHRymCn39g= +github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE= +github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= +github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-runewidth v0.0.12/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk= +github.com/mattn/go-runewidth v0.0.19 h1:v++JhqYnZuu5jSKrk9RbgF5v4CGUjqRfBm05byFGLdw= +github.com/mattn/go-runewidth v0.0.19/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhgLpndooCuJAs= +github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= +github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d h1:5PJl274Y63IEHC+7izoQE9x6ikvDFZS2mDVS3drnohI= +github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= +github.com/microcosm-cc/bluemonday v1.0.27 h1:MpEUotklkwCSLeH+Qdx1VJgNqLlpY2KXwXFM08ygZfk= +github.com/microcosm-cc/bluemonday v1.0.27/go.mod h1:jFi9vgW+H7c3V0lb6nR74Ib/DIB5OBs92Dimizgw2cA= +github.com/microsoft/ApplicationInsights-Go v0.4.4 h1:G4+H9WNs6ygSCe6sUyxRc2U81TI5Es90b2t/MwX5KqY= +github.com/microsoft/ApplicationInsights-Go v0.4.4/go.mod h1:fKRUseBqkw6bDiXTs3ESTiU/4YTIHsQS4W3fP2ieF4U= +github.com/microsoft/go-deviceid v1.0.0 h1:i5AQ654Xk9kfvwJeKQm3w2+eT1+ImBDVEpAR0AjpP40= +github.com/microsoft/go-deviceid v1.0.0/go.mod h1:KY13FeVdHkzD8gy+6T8+kVmD/7RMpTaWW75K+T4uZWg= +github.com/muesli/reflow v0.3.0 h1:IFsN6K9NfGtjeggFP+68I4chLZV2yIKsXJFNZ+eWh6s= +github.com/muesli/reflow v0.3.0/go.mod h1:pbwTDkVPibjO2kyvBQRBxTWEEGDGq0FlB1BIKtnHY/8= +github.com/muesli/termenv v0.16.0 h1:S5AlUN9dENB57rsbnkPyfdGuWIlkmzJjbFf0Tf5FWUc= +github.com/muesli/termenv v0.16.0/go.mod h1:ZRfOIKPFDYQoDFF4Olj7/QJbW60Ol/kL1pU3VfY/Cnk= +github.com/nathan-fiscaletti/consolesize-go v0.0.0-20220204101620-317176b6684d h1:NqRhLdNVlozULwM1B3VaHhcXYSgrOAv8V5BE65om+1Q= +github.com/nathan-fiscaletti/consolesize-go v0.0.0-20220204101620-317176b6684d/go.mod h1:cxIIfNMTwff8f/ZvRouvWYF6wOoO7nj99neWSx2q/Es= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ= +github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= +github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= +github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= +github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= +github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= +github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 h1:KRzFb2m7YtdldCEkzs6KqmJw4nqEVZGK7IN2kJkjTuQ= +github.com/santhosh-tekuri/jsonschema/v6 v6.0.2/go.mod h1:JXeL+ps8p7/KNMjDQk3TCwPpBy0wYklyWTfbkIzdIFU= +github.com/sethvargo/go-retry v0.3.0 h1:EEt31A35QhrcRZtrYFDTBg91cqZVnFL2navjDrah2SE= +github.com/sethvargo/go-retry v0.3.0/go.mod h1:mNX17F0C/HguQMyMyJxcnU471gOZGxCLyYaFyAZraas= +github.com/spf13/cast v1.10.0 h1:h2x0u2shc1QuLHfxi+cTJvs30+ZAHOGRic8uyGTDWxY= +github.com/spf13/cast v1.10.0/go.mod h1:jNfB8QC9IA6ZuY2ZjDp0KtFO2LZZlg4S/7bzP6qqeHo= +github.com/spf13/cobra v1.10.1 h1:lJeBwCfmrnXthfAupyUTzJ/J4Nc1RsHC/mSRU2dll/s= +github.com/spf13/cobra v1.10.1/go.mod h1:7SmJGaTHFVBY0jW4NXGluQoLvhqFQM+6XSKD+P4XaB0= +github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk= +github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/tedsuo/ifrit v0.0.0-20180802180643-bea94bb476cc/go.mod h1:eyZnKCc955uh98WQvzOm0dgAeLnf2O0Rz0LPoC5ze+0= +github.com/theckman/yacspin v0.13.12 h1:CdZ57+n0U6JMuh2xqjnjRq5Haj6v1ner2djtLQRzJr4= +github.com/theckman/yacspin v0.13.12/go.mod h1:Rd2+oG2LmQi5f3zC3yeZAOl245z8QOvrH4OPOJNZxLg= +github.com/wk8/go-ordered-map/v2 v2.1.8 h1:5h/BUHu93oj4gIdvHHHGsScSTMijfx5PeYkE/fJgbpc= +github.com/wk8/go-ordered-map/v2 v2.1.8/go.mod h1:5nJHM5DyteebpVlHnWMV0rPz6Zp7+xBAnxjb1X5vnTw= +github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavMF/ppJZNG9ZpyihvCd0w101no= +github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJuqunuUZ/Dhy/avygyECGrLceyNeo4LiM= +github.com/yosida95/uritemplate/v3 v3.0.2 h1:Ed3Oyj9yrmi9087+NczuL5BwkIc4wvTb5zIM+UJPGz4= +github.com/yosida95/uritemplate/v3 v3.0.2/go.mod h1:ILOh0sOhIJR3+L/8afwt/kE++YT040gmv5BQTMR2HP4= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +github.com/yuin/goldmark v1.7.13 h1:GPddIs617DnBLFFVJFgpo1aBfe/4xcvMc3SB5t/D0pA= +github.com/yuin/goldmark v1.7.13/go.mod h1:ip/1k0VRfGynBgxOz0yCqHrbZXhcjxyuS66Brc7iBKg= +github.com/yuin/goldmark-emoji v1.0.6 h1:QWfF2FYaXwL74tfGOW5izeiZepUDroDJfWubQI9HTHs= +github.com/yuin/goldmark-emoji v1.0.6/go.mod h1:ukxJDKFpdFb5x0a5HqbdlcKtebh086iJpI31LTKmWuA= +go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= +go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= +go.opentelemetry.io/otel v1.43.0 h1:mYIM03dnh5zfN7HautFE4ieIig9amkNANT+xcVxAj9I= +go.opentelemetry.io/otel v1.43.0/go.mod h1:JuG+u74mvjvcm8vj8pI5XiHy1zDeoCS2LB1spIq7Ay0= +go.opentelemetry.io/otel/metric v1.43.0 h1:d7638QeInOnuwOONPp4JAOGfbCEpYb+K6DVWvdxGzgM= +go.opentelemetry.io/otel/metric v1.43.0/go.mod h1:RDnPtIxvqlgO8GRW18W6Z/4P462ldprJtfxHxyKd2PY= +go.opentelemetry.io/otel/sdk v1.43.0 h1:pi5mE86i5rTeLXqoF/hhiBtUNcrAGHLKQdhg4h4V9Dg= +go.opentelemetry.io/otel/sdk v1.43.0/go.mod h1:P+IkVU3iWukmiit/Yf9AWvpyRDlUeBaRg6Y+C58QHzg= +go.opentelemetry.io/otel/sdk/metric v1.43.0 h1:S88dyqXjJkuBNLeMcVPRFXpRw2fuwdvfCGLEo89fDkw= +go.opentelemetry.io/otel/sdk/metric v1.43.0/go.mod h1:C/RJtwSEJ5hzTiUz5pXF1kILHStzb9zFlIEe85bhj6A= +go.opentelemetry.io/otel/trace v1.43.0 h1:BkNrHpup+4k4w+ZZ86CZoHHEkohws8AY+WTX09nk+3A= +go.opentelemetry.io/otel/trace v1.43.0/go.mod h1:/QJhyVBUUswCphDVxq+8mld+AvhXZLhe+8WVFxiFff0= +go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= +go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= +go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.53.0 h1:QZ4Muo8THX6CizN2vPPd5fBGHyogrdK9fG4wLPFUsto= +golang.org/x/crypto v0.53.0/go.mod h1:DNLU434OwVakk9PzuwV8w62mAJpRJL3vsgcfp4Qnsio= +golang.org/x/exp v0.0.0-20250911091902-df9299821621 h1:2id6c1/gto0kaHYyrixvknJ8tUK/Qs5IsmBtrc+FtgU= +golang.org/x/exp v0.0.0-20250911091902-df9299821621/go.mod h1:TwQYMMnGpvZyc+JpB/UAuTNIsVJifOlSkrZkhcvpVUk= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.56.0 h1:Rw8j/hFzGvJUZwNBXnAtf5sVDVt+65SK2C7IxCxZt5o= +golang.org/x/net v0.56.0/go.mod h1:D3Ku6r+V6JROoZK144D2XfMHFcMq/0zSfLelVTCFKec= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM= +golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210331175145-43e1dd70ce54/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw= +golang.org/x/sys v0.46.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.44.0 h1:0rLvDRCtNj0gZkyIXhCyOb2OAzEhLVqc4B+hrsBhrmc= +golang.org/x/term v0.44.0/go.mod h1:7ze4MdzUzLXpSAoFP1H0bOI9aXDqveSvatT5vKcFh2Y= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.38.0 h1:sXmwo9DwP3OK9EZ7PqAdaooSGozfl/3a6/xJcbzPRhE= +golang.org/x/text v0.38.0/go.mod h1:YXZt3QhHUKYT53r2lLKFIVi6Ao1jdzrTR/KQ09qyxF4= +golang.org/x/time v0.9.0 h1:EsRrnYcQiGH+5FfbgvV4AP7qEZstoyrHB0DzarOQ4ZY= +golang.org/x/time v0.9.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= +gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260401024825-9d38bb4040a9 h1:m8qni9SQFH0tJc1X0vmnpw/0t+AImlSvp30sEupozUg= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260401024825-9d38bb4040a9/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= +google.golang.org/grpc v1.80.0 h1:Xr6m2WmWZLETvUNvIUmeD5OAagMw3FiKmMlTdViWsHM= +google.golang.org/grpc v1.80.0/go.mod h1:ho/dLnxwi3EDJA4Zghp7k2Ec1+c2jqup0bFkw07bwF4= +google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= +google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/cli/azd/extensions/azure.ai.dataset/internal/cmd/apiversions.go b/cli/azd/extensions/azure.ai.dataset/internal/cmd/apiversions.go new file mode 100644 index 00000000000..16ad4a772c4 --- /dev/null +++ b/cli/azd/extensions/azure.ai.dataset/internal/cmd/apiversions.go @@ -0,0 +1,13 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +// API versions used by the Foundry data plane. +const ( + // ProjectEndpointAPIVersion covers datasets on the project endpoint. + ProjectEndpointAPIVersion = "2025-11-15-preview" + + // DataGenerationAPIVersion covers dataset generation jobs. + DataGenerationAPIVersion = "v1" +) diff --git a/cli/azd/extensions/azure.ai.dataset/internal/cmd/artifacts.go b/cli/azd/extensions/azure.ai.dataset/internal/cmd/artifacts.go new file mode 100644 index 00000000000..7d05564e271 --- /dev/null +++ b/cli/azd/extensions/azure.ai.dataset/internal/cmd/artifacts.go @@ -0,0 +1,96 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "fmt" + "path/filepath" + "strings" +) + +// The sample count the service accepts, and the default the spec documents. +const ( + minSampleSize = 15 + maxSampleSize = 1000 + defaultSampleSize = 15 +) + +// defaultOutputDir is where a generated dataset lands, matching the layout +// `azd ai eval init` scaffolds so a generated file is already where the eval +// configuration expects to find it. +const defaultOutputDir = "evals/datasets" + +// Sources a dataset can be generated from. +const ( + generateFromTraces = "traces" + generateFromAgent = "agent" + generateFromPrompt = "prompt" + generateFromFile = "file" +) + +// generateSources is what --from accepts, in help order. +var generateSources = []string{ + generateFromTraces, generateFromAgent, generateFromPrompt, generateFromFile, +} + +// validateGenerateSource rejects a --from value the service has no path for. +func validateGenerateSource(from string) error { + switch from { + case "", generateFromTraces, generateFromAgent, generateFromPrompt, generateFromFile: + return nil + default: + return fmt.Errorf( + "--from %q is not a source; use one of %s", + from, strings.Join(generateSources, ", ")) + } +} + +// validateSampleSize rejects a row count the service would reject, before a +// generation job is submitted and billed. +func validateSampleSize(n int) error { + if n != 0 && (n < minSampleSize || n > maxSampleSize) { + return fmt.Errorf( + "--max-samples must be between %d and %d, got %d", + minSampleSize, maxSampleSize, n) + } + return nil +} + +// artifactPath is where a generated dataset is written. +func artifactPath(outputDir, name string) string { + return filepath.Join(outputDir, name+".jsonl") +} + +// envKeyDatasetVersion caches the version resolved at the last publish, so a +// later read does not have to list every version to find the newest. +const envKeyDatasetVersion = "EVAL_DATASET_VERSION" + +// checkAssetExistence enforces the one difference between create and update. +func checkAssetExistence(verb, kind, name string, exists bool) error { + switch { + case verb == "create" && exists: + return fmt.Errorf( + "%s %q already exists: use `update` to publish a new version", kind, name) + case verb == "update" && !exists: + return fmt.Errorf( + "%s %q does not exist: use `create` to register it", kind, name) + } + return nil +} + +// defaultGenerationSource picks what `dataset generate` sends when --from was +// not given, from the Application Insights connection string the project has +// (or has not) been given. +// +// Traces are the better dataset when they exist, being real conversations +// rather than synthesized ones, so they win whenever the project is wired to +// collect them. Outside a project, or in one with no Application Insights, +// there are no traces to ask for and the agent's own definition is all that is +// left. +func defaultGenerationSource(appInsightsConnection string) []string { + if appInsightsConnection != "" { + return []string{generateFromTraces} + } + return []string{generateFromAgent} +} diff --git a/cli/azd/extensions/azure.ai.dataset/internal/cmd/context.go b/cli/azd/extensions/azure.ai.dataset/internal/cmd/context.go new file mode 100644 index 00000000000..f8637c2b073 --- /dev/null +++ b/cli/azd/extensions/azure.ai.dataset/internal/cmd/context.go @@ -0,0 +1,155 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "context" + "errors" + "fmt" + "os" + "strings" + + "azureaidataset/internal/pkg/dataset_api" + "azureaidataset/internal/pkg/gen_api" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore" + "github.com/Azure/azure-sdk-for-go/sdk/azidentity" + "github.com/azure/azure-dev/cli/azd/pkg/azdext" +) + +// projectEndpointEnvKey is the azd environment key holding the Foundry project +// endpoint the data-plane clients target. +const projectEndpointEnvKey = "FOUNDRY_PROJECT_ENDPOINT" + +// appInsightsEnvKey is where a connected Application Insights resource lands in +// the azd environment, and so how a project says it collects traces at all. +const appInsightsEnvKey = "APPLICATIONINSIGHTS_CONNECTION_STRING" + +// datasetContext carries everything the commands need to reach the data plane. +// +// Two clients, because a dataset is registered through the dataset API but +// generated by the evaluation service. +type datasetContext struct { + azdClient *azdext.AzdClient + endpoint string + envName string + cred azcore.TokenCredential + + datasetClient *dataset_api.DatasetClient + genClient *gen_api.Client +} + +// newDatasetContext resolves the project endpoint and builds the data-plane +// clients. Endpoint resolution order: +// +// 1. --project-endpoint +// 2. the active azd environment's FOUNDRY_PROJECT_ENDPOINT +// 3. the host environment variable of the same name +func newDatasetContext(ctx context.Context, endpointFlag string) (*datasetContext, error) { + azdClient, err := azdext.NewAzdClient() + if err != nil { + return nil, fmt.Errorf("connecting to azd: %w", err) + } + + dc := &datasetContext{azdClient: azdClient} + + // The environment name is resolved regardless of where the endpoint came + // from: it is what cached version numbers are read from and written to. + azdEndpoint, envName := lookupEndpointFromAzd(ctx, azdClient) + dc.envName = envName + + if endpointFlag != "" { + dc.endpoint = endpointFlag + } else { + dc.endpoint = azdEndpoint + } + if dc.endpoint == "" { + dc.endpoint = os.Getenv(projectEndpointEnvKey) + } + if dc.endpoint == "" { + return nil, fmt.Errorf( + "no Foundry project endpoint found; pass --project-endpoint or set %s "+ + "in the azd environment (azd env set %s )", + projectEndpointEnvKey, projectEndpointEnvKey) + } + dc.endpoint = strings.TrimSuffix(dc.endpoint, "/") + + cred, err := azidentity.NewAzureDeveloperCLICredential( + &azidentity.AzureDeveloperCLICredentialOptions{}, + ) + if err != nil { + return nil, fmt.Errorf("creating Azure credential: %w", err) + } + dc.cred = cred + + dc.datasetClient = dataset_api.NewDatasetClient(dc.endpoint, cred) + dc.genClient = gen_api.NewClient(dc.endpoint, cred) + + return dc, nil +} + +// lookupEndpointFromAzd reads the endpoint from the active azd environment, +// returning empty strings when azd has no current environment. +func lookupEndpointFromAzd(ctx context.Context, azdClient *azdext.AzdClient) (endpoint, envName string) { + envResp, err := azdClient.Environment().GetCurrent(ctx, &azdext.EmptyRequest{}) + if err != nil || envResp == nil || envResp.Environment == nil { + return "", "" + } + val, err := azdClient.Environment().GetValue(ctx, &azdext.GetEnvRequest{ + EnvName: envResp.Environment.Name, + Key: projectEndpointEnvKey, + }) + if err != nil || val == nil || val.Value == "" { + return "", envResp.Environment.Name + } + return val.Value, envResp.Environment.Name +} + +// errNoAzdEnvironment reports that there is no azd environment to persist into. +// +// These commands work standalone against the data plane, so running outside a +// project is ordinary rather than a problem worth reporting. +var errNoAzdEnvironment = errors.New("no active azd environment") + +// setEnvValue persists a value into the active azd environment. +func (dc *datasetContext) setEnvValue(ctx context.Context, key, value string) error { + if dc.envName == "" { + envResp, err := dc.azdClient.Environment().GetCurrent(ctx, &azdext.EmptyRequest{}) + if err != nil || envResp == nil || envResp.Environment == nil { + return fmt.Errorf("%w to write %s into", errNoAzdEnvironment, key) + } + dc.envName = envResp.Environment.Name + } + _, err := dc.azdClient.Environment().SetValue(ctx, &azdext.SetEnvRequest{ + EnvName: dc.envName, + Key: key, + Value: value, + }) + if err != nil { + return fmt.Errorf("writing %s to the azd environment: %w", key, err) + } + return nil +} + +// getEnvValue reads a value from the active azd environment, returning empty +// when it is unset. +func (dc *datasetContext) getEnvValue(ctx context.Context, key string) string { + if dc.envName == "" { + return "" + } + val, err := dc.azdClient.Environment().GetValue(ctx, &azdext.GetEnvRequest{ + EnvName: dc.envName, + Key: key, + }) + if err != nil || val == nil { + return "" + } + return val.Value +} + +func (dc *datasetContext) Close() { + if dc.azdClient != nil { + dc.azdClient.Close() + } +} diff --git a/cli/azd/extensions/azure.ai.dataset/internal/cmd/dataset.go b/cli/azd/extensions/azure.ai.dataset/internal/cmd/dataset.go new file mode 100644 index 00000000000..97d2cb8ff9f --- /dev/null +++ b/cli/azd/extensions/azure.ai.dataset/internal/cmd/dataset.go @@ -0,0 +1,338 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "strings" + + "azureaidataset/internal/pkg/dataset_api" + "azureaidataset/internal/pkg/gen_api" + + "github.com/spf13/cobra" +) + +// firstDatasetVersion is the version the service assigns to a dataset's first +// publish, and so the one that exists for every dataset that exists at all. +const firstDatasetVersion = "1" + +func newDatasetCommand() *cobra.Command { + cmd := &cobra.Command{ + Use: "dataset", + Short: "Manage Foundry datasets.", + } + cmd.AddCommand( + newDatasetCreateCommand(), + newDatasetUpdateCommand(), + newDatasetGenerateCommand(), + newDatasetListCommand(), + newDatasetShowCommand(), + newDatasetDeleteCommand(), + newDatasetVersionsCommand(), + newJobCommand(), + ) + return cmd +} + +// newDatasetCreateCommand builds `dataset create `, which registers a +// dataset that does not exist yet. +func newDatasetCreateCommand() *cobra.Command { + return newDatasetWriteCommand("create", "Register a dataset, publishing its first version.") +} + +// newDatasetUpdateCommand builds `dataset update `, which publishes a +// further version of one that does. +func newDatasetUpdateCommand() *cobra.Command { + return newDatasetWriteCommand("update", "Publish a new version of a dataset.") +} + +// newDatasetWriteCommand builds create and update. Both run the same upload, +// and the existence check is the only thing that separates them: a version is +// brought into being by startPendingUpload, which neither knows nor cares +// whether the name was already in use. +func newDatasetWriteCommand(verb, short string) *cobra.Command { + var ( + fromFile string + version string + endpointFlg string + ) + + cmd := &cobra.Command{ + Use: verb + " ", + Short: short, + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + name := args[0] + if fromFile == "" { + return requireFlag("from-file") + } + + localDir, err := datasetUploadDir(fromFile) + if err != nil { + return err + } + + ctx := cmd.Context() + ec, err := newDatasetContext(ctx, endpointFlg) + if err != nil { + return err + } + defer ec.Close() + + existing, err := ec.datasetClient.ListDatasetVersions( + ctx, name, ProjectEndpointAPIVersion, + ) + exists := err == nil && existing != nil && len(existing.Value) > 0 + if !exists { + // The version listing lags a publish, so a `create` followed by + // an `update` was told the dataset it had just made does not + // exist. A direct read of the first version settles it: point + // reads go consistent immediately. + if _, err := ec.datasetClient.GetDataset( + ctx, name, firstDatasetVersion, ProjectEndpointAPIVersion, + ); err == nil { + exists = true + } + } + if err := checkAssetExistence(verb, "dataset", name, exists); err != nil { + return err + } + + ds, err := ec.datasetClient.UploadNextVersion( + ctx, name, version, localDir, ProjectEndpointAPIVersion, + ) + if err != nil { + return fmt.Errorf("registering dataset %q: %w", name, err) + } + + if err := ec.setEnvValue(ctx, envKeyDatasetVersion, ds.Version); err != nil { + // Persisting is a convenience, so this never fails the command. + // It goes to stdout because azd does not surface an extension's + // stderr, and is skipped outside a project, where having nowhere + // to persist is expected rather than notable. + if !errors.Is(err, errNoAzdEnvironment) && !isJSON(cmd) { + fmt.Fprintf(cmd.OutOrStdout(), "warning: %v\n", err) + } + } + + if isJSON(cmd) { + return emitJSON(cmd.OutOrStdout(), ds) + } + fmt.Fprintf(cmd.OutOrStdout(), "Registered dataset %s version %s\n", ds.Name, ds.Version) + return nil + }, + } + + cmd.Flags().StringVar(&fromFile, "from-file", "", + "Path to a .jsonl file, or a directory containing one.") + cmd.Flags().StringVar(&version, "version", "", + "Current version to increment from. Omit to increment from the latest registered version.") + cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") + return cmd +} + +// datasetUploadDir resolves what was named into the directory the upload scans. +func datasetUploadDir(path string) (string, error) { + info, err := os.Stat(path) + if err != nil { + return "", fmt.Errorf("reading --from-file %q: %w", path, err) + } + if info.IsDir() { + return path, nil + } + if !strings.EqualFold(filepath.Ext(path), ".jsonl") { + return "", fmt.Errorf( + "--from-file must be a .jsonl file or a directory containing one, got %q", path) + } + return filepath.Dir(path), nil +} + +func newDatasetListCommand() *cobra.Command { + var endpointFlg string + + cmd := &cobra.Command{ + Use: "list", + Short: "List the project's datasets.", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + ctx := cmd.Context() + ec, err := newDatasetContext(ctx, endpointFlg) + if err != nil { + return err + } + defer ec.Close() + + list, err := ec.datasetClient.ListDatasets(ctx, ProjectEndpointAPIVersion) + if err != nil { + return fmt.Errorf("listing datasets: %w", err) + } + return renderDatasets(cmd, list) + }, + } + + cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") + return cmd +} + +// newDatasetVersionsCommand groups the version listing, so that `list` means +// the assets rather than the history of one of them. +func newDatasetVersionsCommand() *cobra.Command { + cmd := &cobra.Command{ + Use: "versions", + Short: "Inspect the versions of one dataset.", + } + cmd.AddCommand(newDatasetVersionsListCommand()) + return cmd +} + +func newDatasetVersionsListCommand() *cobra.Command { + var endpointFlg string + + cmd := &cobra.Command{ + Use: "list ", + Short: "List the versions of a dataset.", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + name := args[0] + + ctx := cmd.Context() + ec, err := newDatasetContext(ctx, endpointFlg) + if err != nil { + return err + } + defer ec.Close() + + list, err := ec.datasetClient.ListDatasetVersions(ctx, name, ProjectEndpointAPIVersion) + if err != nil { + return fmt.Errorf("listing versions of dataset %q: %w", name, err) + } + return renderDatasets(cmd, list) + }, + } + + cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") + return cmd +} + +func renderDatasets(cmd *cobra.Command, list *dataset_api.DatasetList) error { + if isJSON(cmd) { + return emitJSONList(cmd.OutOrStdout(), list.Value) + } + rows := make([][]string, 0, len(list.Value)) + for _, d := range list.Value { + rows = append(rows, []string{d.Name, d.Version, d.Format}) + } + if len(rows) == 0 { + fmt.Fprintln(cmd.OutOrStdout(), "No datasets found.") + return nil + } + return emitTable(cmd.OutOrStdout(), []string{"NAME", "VERSION", "FORMAT"}, rows) +} + +func newDatasetShowCommand() *cobra.Command { + var ( + version string + endpointFlg string + ) + + cmd := &cobra.Command{ + Use: "show ", + Short: "Show a dataset version.", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + name := args[0] + + ctx := cmd.Context() + ec, err := newDatasetContext(ctx, endpointFlg) + if err != nil { + return err + } + defer ec.Close() + + if version == "" { + list, err := ec.datasetClient.ListDatasetVersions(ctx, name, ProjectEndpointAPIVersion) + if err != nil { + return fmt.Errorf("resolving the latest version of %q: %w", name, err) + } + if len(list.Value) == 0 { + return fmt.Errorf("dataset %q has no versions", name) + } + version = dataset_api.LatestVersion(list.Value) + } + + ds, err := ec.datasetClient.GetDataset(ctx, name, version, ProjectEndpointAPIVersion) + if err != nil { + if gen_api.IsNotFound(err) { + return fmt.Errorf( + "no dataset %q at version %q in this project; "+ + "`azd ai eval dataset list` shows the ones there are", name, version) + } + return fmt.Errorf("reading dataset %q version %q: %w", name, version, err) + } + + if isJSON(cmd) { + return emitJSON(cmd.OutOrStdout(), ds) + } + return emitTable(cmd.OutOrStdout(), + []string{"NAME", "VERSION", "FORMAT", "URI"}, + [][]string{{ds.Name, ds.Version, ds.Format, ds.ResolvedBlobURI()}}, + ) + }, + } + + cmd.Flags().StringVar(&version, "version", "", "Version to show. Omit for the latest.") + cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") + return cmd +} + +func newDatasetDeleteCommand() *cobra.Command { + var ( + version string + endpointFlg string + ) + + cmd := &cobra.Command{ + Use: "delete ", + Short: "Delete a dataset version.", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + name := args[0] + if version == "" { + return requireFlag("version") + } + + ctx := cmd.Context() + ec, err := newDatasetContext(ctx, endpointFlg) + if err != nil { + return err + } + defer ec.Close() + + if err := ec.datasetClient.DeleteDatasetVersion( + ctx, name, version, ProjectEndpointAPIVersion, + ); err != nil { + if gen_api.IsNotFound(err) { + return fmt.Errorf( + "no dataset %q at version %q in this project", name, version) + } + return fmt.Errorf("deleting dataset %q version %q: %w", name, version, err) + } + + if isJSON(cmd) { + return emitJSON(cmd.OutOrStdout(), map[string]string{ + "name": name, "version": version, "status": "deleted", + }) + } + fmt.Fprintf(cmd.OutOrStdout(), "Deleted dataset %s version %s\n", name, version) + return nil + }, + } + + cmd.Flags().StringVar(&version, "version", "", "Version to delete.") + cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") + return cmd +} diff --git a/cli/azd/extensions/azure.ai.dataset/internal/cmd/debug.go b/cli/azd/extensions/azure.ai.dataset/internal/cmd/debug.go new file mode 100644 index 00000000000..48fc0802aaf --- /dev/null +++ b/cli/azd/extensions/azure.ai.dataset/internal/cmd/debug.go @@ -0,0 +1,65 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "fmt" + "io" + "log" + "os" + "strconv" + "time" + + azcorelog "github.com/Azure/azure-sdk-for-go/sdk/azcore/log" + "github.com/spf13/pflag" +) + +// setupDebugLogging silences the standard logger unless debug mode is on. +// +// The data-plane clients trace every request through log.Printf, which Go +// writes to stderr by default. Without this the CLI interleaves raw HTTP traces +// with its own output on every command. Returns a cleanup function the caller +// should defer. +func setupDebugLogging(flags *pflag.FlagSet) func() { + if !isDebug(flags) { + log.SetOutput(io.Discard) + azcorelog.SetListener(nil) + return func() {} + } + + logFileName := fmt.Sprintf("azd-ai-eval-%s.log", time.Now().Format("2006-01-02")) + + //nolint:gosec // the name is generated locally from the date, not user input + logFile, err := os.OpenFile(logFileName, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0600) + + var w io.Writer + var closeFile func() + if err != nil { + w = os.Stderr + closeFile = func() {} + } else { + w = logFile + closeFile = func() { logFile.Close() } //nolint:gosec // best-effort cleanup + } + + log.SetOutput(w) + azcorelog.SetListener(func(event azcorelog.Event, msg string) { + fmt.Fprintf(w, "[%s] %s: %s\n", time.Now().Format(time.RFC3339), event, msg) + }) + + return func() { + log.SetOutput(io.Discard) + azcorelog.SetListener(nil) + closeFile() + } +} + +// isDebug reports whether --debug or AZD_EXT_DEBUG is set. +func isDebug(flags *pflag.FlagSet) bool { + if debugFlag, err := flags.GetBool("debug"); err == nil && debugFlag { + return true + } + debug, _ := strconv.ParseBool(os.Getenv("AZD_EXT_DEBUG")) + return debug +} diff --git a/cli/azd/extensions/azure.ai.dataset/internal/cmd/generate.go b/cli/azd/extensions/azure.ai.dataset/internal/cmd/generate.go new file mode 100644 index 00000000000..75478e9a546 --- /dev/null +++ b/cli/azd/extensions/azure.ai.dataset/internal/cmd/generate.go @@ -0,0 +1,366 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "context" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "strings" + "time" + + "azureaidataset/internal/pkg/gen_api" + + "github.com/spf13/cobra" +) + +// generatePollBudget replaces the inherited 2s x 300 (10 minute) client budget. +// The generation job is not gateway-capped; the old limit simply gave up while +// the service was still working, forcing a second command. +var generatePollBudget = gen_api.PollerOptions{ + Interval: 5 * time.Second, + MaxAttempts: 720, // one hour +} + +// generationPlan is everything one generation job needs, after the flags have +// been reconciled. +type generationPlan struct { + Name string + Agent string + Instruction string + Model string + OutputDir string + SampleSize int + // From is what --from named: which of the service's sources to send. Empty + // sends whatever the plan has to offer. + From []string + // TraceDays seeds generation from that many days of recent traces. + TraceDays int +} + +func (p generationPlan) traceOptions() *gen_api.TraceOptions { + if p.TraceDays <= 0 { + return nil + } + return &gen_api.TraceOptions{Days: p.TraceDays} +} + +func newDatasetGenerateCommand() *cobra.Command { + var ( + name string + target string + instruction string + instructionFile string + model string + outputDir string + endpoint string + from []string + maxSamples int + traceDays int + noWait bool + force bool + ) + + cmd := &cobra.Command{ + Use: "generate ", + Short: "Generate a dataset and download it.", + Long: "Generate a dataset and download it.\n\n" + + "--from selects one or more of the sources the service accepts, and " + + "is repeatable. Generating from the agent's own definition is a " + + "preference rather than a fallback: it covers cases no user has hit " + + "yet, and it can supply reference answers, which a transcript cannot.", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + name = args[0] + + for _, src := range from { + if err := validateGenerateSource(src); err != nil { + return err + } + } + if err := validateSampleSize(maxSamples); err != nil { + return err + } + if instruction == "" && instructionFile != "" { + body, err := os.ReadFile(instructionFile) + if err != nil { + return fmt.Errorf("reading --agent-instruction-file %q: %w", instructionFile, err) + } + instruction = strings.TrimSpace(string(body)) + } + if model == "" { + return errors.New("--generation-model is required") + } + + plan := generationPlan{ + Name: name, + Agent: target, + Instruction: instruction, + Model: model, + OutputDir: outputDir, + SampleSize: maxSamples, + From: from, + TraceDays: traceDays, + } + if plan.OutputDir == "" { + plan.OutputDir = defaultOutputDir + } + if plan.SampleSize == 0 { + plan.SampleSize = defaultSampleSize + } + if err := refuseExistingArtifact(artifactPath(plan.OutputDir, plan.Name), force); err != nil { + return err + } + + ctx := cmd.Context() + dc, err := newDatasetContext(ctx, endpoint) + if err != nil { + return err + } + defer dc.Close() + + if len(plan.From) == 0 { + plan.From = defaultGenerationSource(dc.getEnvValue(ctx, appInsightsEnvKey)) + } + plan.Instruction, err = dc.resolveGenerationInstruction( + ctx, plan.Instruction, plan.Agent, cmd.OutOrStdout(), isJSON(cmd)) + if err != nil { + return err + } + + return dc.generateDataset(ctx, plan, cmd.OutOrStdout(), noWait, isJSON(cmd)) + }, + } + + cmd.Flags().StringVar(&target, "target", "", "Agent whose context seeds generation.") + cmd.Flags().StringVar(&instruction, "agent-instruction", "", + "What the agent does and what to test.") + cmd.Flags().StringVar(&instructionFile, "agent-instruction-file", "", + "Read the agent instruction from this file. Mutually exclusive with --agent-instruction.") + cmd.MarkFlagsMutuallyExclusive("agent-instruction", "agent-instruction-file") + cmd.Flags().StringVar(&model, "generation-model", "", + "Model deployment that generates the dataset.") + cmd.Flags().StringVar(&outputDir, "output-dir", "", + fmt.Sprintf("Directory the generated dataset is written to. Defaults to %s.", defaultOutputDir)) + cmd.Flags().IntVar(&maxSamples, "max-samples", 0, + fmt.Sprintf("Rows to synthesize (%d-%d). Defaults to %d.", + minSampleSize, maxSampleSize, defaultSampleSize)) + cmd.Flags().StringSliceVar(&from, "from", nil, + fmt.Sprintf("Where rows come from: %s. Repeatable, and the service accepts "+ + "more than one. Defaults to %s when the project has Application Insights "+ + "connected, otherwise %s.", + strings.Join(generateSources, ", "), generateFromTraces, generateFromAgent)) + cmd.Flags().IntVar(&traceDays, "trace-days", 0, + "Narrow a traces source to this many days of recent activity.") + cmd.Flags().BoolVar(&noWait, "no-wait", false, + "Submit the job and return its id without polling.") + cmd.Flags().BoolVar(&force, "force", false, + "Overwrite a dataset file that already exists.") + cmd.Flags().StringVar(&endpoint, "project-endpoint", "", "Foundry project endpoint.") + return cmd +} + +// refuseExistingArtifact stops a generation that would overwrite a checked-in +// file, because the job is billed and the diff is what the author reviews. +func refuseExistingArtifact(path string, force bool) error { + if force { + return nil + } + if _, err := os.Stat(path); err == nil { + return fmt.Errorf( + "%s already exists; pass --force to overwrite it, or --output-dir to write elsewhere", + filepath.ToSlash(path)) + } + return nil +} + +// resolveGenerationInstruction decides what generation is seeded from: what the +// caller passed, then the agent's published instructions. +// +// The service accepts an agent source that is meant to pull those instructions, +// but it fails for every agent, so they are read here instead. +func (dc *datasetContext) resolveGenerationInstruction( + ctx context.Context, + explicit, agentName string, + out io.Writer, + quiet bool, +) (string, error) { + if explicit != "" || agentName == "" { + return explicit, nil + } + + agent, err := dc.genClient.GetAgent(ctx, agentName, ProjectEndpointAPIVersion) + if err != nil { + // Generation can still proceed from the agent source alone, so a + // failure to read the agent is reported without stopping. + if !quiet { + fmt.Fprintf(out, " warning: could not read agent %q for generation context: %v\n", + agentName, err) + } + return "", nil + } + instructions := agent.Instructions() + if instructions != "" && !quiet { + fmt.Fprintf(out, " Seeding generation from the instructions of agent %q.\n", agentName) + } + return instructions, nil +} + +// generateDataset submits the generation job and downloads what it produced. +func (dc *datasetContext) generateDataset( + ctx context.Context, + plan generationPlan, + out io.Writer, + noWait, quiet bool, +) error { + if !quiet { + fmt.Fprintf(out, "Generating dataset %s (%d samples)...\n", plan.Name, plan.SampleSize) + } + + sources, unbuildable := gen_api.BuildGenerationSources( + plan.From, plan.Agent, "", plan.Instruction, plan.traceOptions()) + if err := refuseUnbuildableSources(unbuildable); err != nil { + return err + } + req := gen_api.NewDataGenerationJobRequest(plan.Name, plan.Model, plan.SampleSize, sources) + + job, err := dc.genClient.CreateDataGenerationJob(ctx, req, DataGenerationAPIVersion) + if err != nil { + return fmt.Errorf("submitting the data generation job: %w", err) + } + if noWait { + fmt.Fprintf(out, " submitted job %s\n", job.ID) + fmt.Fprintf(out, "\nReattach with: azd ai dataset job show %s\n", job.ID) + return nil + } + + completed, err := dc.pollGeneration(ctx, job.ID, DataGenerationAPIVersion, + dc.genClient.GetDataGenerationJob) + if err != nil && isAgentSeededGenerationFailure(err) { + // Agent-seeded generation fails server-side for every agent, while the + // same request carrying only the prompt succeeds. Failing the whole + // command would block the documented flow on a defect the user cannot + // do anything about, so retry without the agent and say so. + promptOnly := gen_api.WithoutAgentSource(sources) + if gen_api.HasPromptSource(promptOnly) { + fmt.Fprintf(out, + " warning: generating from agent %q failed in the service; "+ + "retrying from the instruction alone.\n", plan.Agent) + + req = gen_api.NewDataGenerationJobRequest( + plan.Name, plan.Model, plan.SampleSize, promptOnly) + job, err = dc.genClient.CreateDataGenerationJob(ctx, req, DataGenerationAPIVersion) + if err != nil { + return fmt.Errorf("submitting the data generation job: %w", err) + } + completed, err = dc.pollGeneration(ctx, job.ID, DataGenerationAPIVersion, + dc.genClient.GetDataGenerationJob) + } + } + if err != nil { + return fmt.Errorf("data generation: %w", explainDataGenerationFailure(err, plan.Agent)) + } + + name, version := completed.ResolvedNameVersion() + if name == "" { + return fmt.Errorf("the data generation job returned no dataset reference") + } + + // Confirm the version exists before reading it, so a missing dataset is + // reported as such rather than as a download failure. + if _, err := dc.datasetClient.GetDataset(ctx, name, version, ProjectEndpointAPIVersion); err != nil { + return fmt.Errorf("reading the generated dataset %q: %w", name, err) + } + content, err := dc.datasetClient.DownloadDatasetContent( + ctx, name, version, ProjectEndpointAPIVersion) + if err != nil { + return fmt.Errorf("downloading the generated dataset %q: %w", name, err) + } + + path := artifactPath(plan.OutputDir, plan.Name) + if err := os.MkdirAll(filepath.Dir(path), 0o750); err != nil { + return fmt.Errorf("creating %q: %w", filepath.Dir(path), err) + } + if err := os.WriteFile(path, content, 0o600); err != nil { + return fmt.Errorf("writing %q: %w", path, err) + } + + if quiet { + return emitJSON(out, map[string]any{ + "dataset": name, "version": version, "path": filepath.ToSlash(path), + }) + } + fmt.Fprintf(out, " wrote %s\n", filepath.ToSlash(path)) + + // The datasets: catalog belongs to `azure.ai.eval`, which owns eval.yaml, + // so registering the file there is a separate step rather than a silent + // cross-extension write. + fmt.Fprintf(out, "\nNext: azd ai eval dataset create %s --from-file %s\n", + plan.Name, filepath.ToSlash(path)) + return nil +} + +// pollGeneration waits for a generation job using the raised budget. +func (dc *datasetContext) pollGeneration( + ctx context.Context, + operationID, apiVersion string, + get gen_api.GetJobFunc, +) (*gen_api.GenerationJob, error) { + poller := gen_api.NewPoller(operationID, apiVersion, get) + poller.Options = generatePollBudget + return poller.Poll(ctx) +} + +// refuseUnbuildableSources reports a --from the plan could not honour. +// +// Submitting anyway would run a billed job seeded from less than was asked for +// and return a plausible-looking artifact, which is the worst outcome: the +// caller has no way to tell it apart from one built the way they intended. +func refuseUnbuildableSources(kinds []string) error { + if len(kinds) == 0 { + return nil + } + reasons := map[string]string{ + generateFromPrompt: "--from prompt needs --agent-instruction or --agent-instruction-file", + generateFromAgent: "--from agent needs a target agent; pass --target", + generateFromFile: "--from file is not a generation source; " + + "register the file with `azd ai dataset create` instead", + } + messages := make([]string, 0, len(kinds)) + for _, k := range kinds { + if reason, ok := reasons[k]; ok { + messages = append(messages, reason) + continue + } + messages = append(messages, fmt.Sprintf("--from %s cannot be built from this plan", k)) + } + return errors.New(strings.Join(messages, "; ")) +} + +// isAgentSeededGenerationFailure recognizes the service-side failure that hits +// every agent, so it can be retried without the agent rather than surfaced. +func isAgentSeededGenerationFailure(err error) bool { + if err == nil { + return false + } + text := err.Error() + return strings.Contains(text, "DataGenerationJobSystemError") || + strings.Contains(text, "Something went wrong during data generation") +} + +// explainDataGenerationFailure adds context to the service's opaque system +// error, which says only that something went wrong and to try again — sending +// users into a retry loop against a deterministic failure. +func explainDataGenerationFailure(err error, agentName string) error { + if err == nil || agentName == "" || !isAgentSeededGenerationFailure(err) { + return err + } + return fmt.Errorf( + "%w\n\nSeeding generation from agent %q fails server-side for every agent. "+ + "Pass --agent-instruction to generate from the instruction alone.", + err, agentName) +} diff --git a/cli/azd/extensions/azure.ai.dataset/internal/cmd/job.go b/cli/azd/extensions/azure.ai.dataset/internal/cmd/job.go new file mode 100644 index 00000000000..05860a57c77 --- /dev/null +++ b/cli/azd/extensions/azure.ai.dataset/internal/cmd/job.go @@ -0,0 +1,197 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "fmt" + + "azureaidataset/internal/pkg/gen_api" + + "github.com/spf13/cobra" +) + +// The job group nests under `dataset` rather than sitting at the root, because +// generation runs as two independent long-running resources — one for datasets, +// one for evaluators — sharing no collection. The evaluator half lives in +// `azure.ai.evaluations`; a shared top-level `job show ` would have to guess +// which endpoint to call from an id prefix that is not a documented contract. + +func newJobCommand() *cobra.Command { + cmd := &cobra.Command{ + Use: "job", + Short: "Inspect, cancel and delete dataset generation jobs.", + Long: "Inspect, cancel and delete dataset generation jobs.\n\n" + + "This is the resume path for `dataset generate`: a job started with " + + "--no-wait, or one whose client was interrupted, is reattached to here " + + "rather than restarted.", + } + cmd.AddCommand( + newJobListCommand(), + newJobShowCommand(), + newJobCancelCommand(), + newJobDeleteCommand(), + ) + return cmd +} + +func newJobListCommand() *cobra.Command { + var endpointFlg string + + cmd := &cobra.Command{ + Use: "list", + Short: "List the project's dataset generation jobs.", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + ctx := cmd.Context() + dc, err := newDatasetContext(ctx, endpointFlg) + if err != nil { + return err + } + defer dc.Close() + + out, err := dc.genClient.ListDataGenerationJobs(ctx, ProjectEndpointAPIVersion) + if err != nil { + return fmt.Errorf("listing dataset generation jobs: %w", err) + } + jobs := out.Data + + if isJSON(cmd) { + return emitJSONList(cmd.OutOrStdout(), jobs) + } + if len(jobs) == 0 { + fmt.Fprintln(cmd.OutOrStdout(), "No dataset generation jobs found.") + return nil + } + table := make([][]string, 0, len(jobs)) + for _, j := range jobs { + table = append(table, []string{j.ID, j.Status}) + } + return emitTable(cmd.OutOrStdout(), []string{"JOB ID", "STATUS"}, table) + }, + } + + cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") + return cmd +} + +func newJobShowCommand() *cobra.Command { + var endpointFlg string + + cmd := &cobra.Command{ + Use: "show ", + Short: "Show a dataset generation job.", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + jobID := args[0] + + ctx := cmd.Context() + dc, err := newDatasetContext(ctx, endpointFlg) + if err != nil { + return err + } + defer dc.Close() + + job, err := dc.genClient.GetDataGenerationJob(ctx, jobID, ProjectEndpointAPIVersion) + if err != nil { + return jobLookupError(jobID, err) + } + + if isJSON(cmd) { + return emitJSON(cmd.OutOrStdout(), job) + } + fmt.Fprintf(cmd.OutOrStdout(), "%s %s\n", job.ID, job.Status) + if job.Error != nil && job.Error.Message != "" { + fmt.Fprintf(cmd.OutOrStdout(), "error: %s\n", job.Error.Message) + } + return nil + }, + } + + cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") + return cmd +} + +func newJobCancelCommand() *cobra.Command { + var endpointFlg string + + cmd := &cobra.Command{ + Use: "cancel ", + Short: "Cancel an in-flight dataset generation job.", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + jobID := args[0] + + ctx := cmd.Context() + dc, err := newDatasetContext(ctx, endpointFlg) + if err != nil { + return err + } + defer dc.Close() + + canceled, err := dc.genClient.CancelDataGenerationJob(ctx, jobID, ProjectEndpointAPIVersion) + if err != nil { + return jobLookupError(jobID, err) + } + + if isJSON(cmd) { + return emitJSON(cmd.OutOrStdout(), canceled) + } + fmt.Fprintf(cmd.OutOrStdout(), "Cancelled dataset generation job %s (%s)\n", + jobID, canceled.Status) + return nil + }, + } + + cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") + return cmd +} + +func newJobDeleteCommand() *cobra.Command { + var endpointFlg string + + cmd := &cobra.Command{ + Use: "delete ", + Short: "Delete a dataset generation job record.", + Long: "Delete a dataset generation job record.\n\n" + + "The dataset the job produced is already registered as its own version " + + "and is not affected.", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + jobID := args[0] + + ctx := cmd.Context() + dc, err := newDatasetContext(ctx, endpointFlg) + if err != nil { + return err + } + defer dc.Close() + + if err := dc.genClient.DeleteDataGenerationJob(ctx, jobID, ProjectEndpointAPIVersion); err != nil { + return jobLookupError(jobID, err) + } + + if isJSON(cmd) { + return emitJSON(cmd.OutOrStdout(), map[string]string{ + "id": jobID, "kind": "dataset", "status": "deleted", + }) + } + fmt.Fprintf(cmd.OutOrStdout(), "Deleted dataset generation job %s\n", jobID) + return nil + }, + } + + cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") + return cmd +} + +// jobLookupError names the evaluator group, because the two job types share an +// id shape and reaching for the wrong one is the likely mistake. +func jobLookupError(jobID string, err error) error { + if gen_api.IsNotFound(err) { + return fmt.Errorf( + "no dataset generation job %q in this project; if it generated an "+ + "evaluator, use `azd ai eval evaluator job` instead", jobID) + } + return fmt.Errorf("reading dataset generation job %s: %w", jobID, err) +} diff --git a/cli/azd/extensions/azure.ai.dataset/internal/cmd/output.go b/cli/azd/extensions/azure.ai.dataset/internal/cmd/output.go new file mode 100644 index 00000000000..143b49e91ba --- /dev/null +++ b/cli/azd/extensions/azure.ai.dataset/internal/cmd/output.go @@ -0,0 +1,86 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "encoding/json" + "fmt" + "io" + "strings" + "text/tabwriter" + + "github.com/spf13/cobra" +) + +const outputJSON = "json" + +// Progress markers from the azd style guide, so the extension's lines sit +// alongside core's without a second vocabulary. +const ( + doneMark = "(✓) Done:" // finished successfully + skippedMark = "(-) Skipped:" // intentionally not done, not a failure + failedMark = "(x) Failed:" // the step did not complete +) + +// outputFormat reads the inherited -o/--output flag. +func outputFormat(cmd *cobra.Command) string { + if cmd == nil { + return "" + } + v, err := cmd.Flags().GetString("output") + if err != nil { + return "" + } + return strings.ToLower(v) +} + +// isJSON reports whether the command should emit machine-readable output. +func isJSON(cmd *cobra.Command) bool { + return outputFormat(cmd) == outputJSON +} + +// emitJSON writes v as indented JSON. +func emitJSON(w io.Writer, v any) error { + enc := json.NewEncoder(w) + enc.SetIndent("", " ") + return enc.Encode(v) +} + +// emitJSONList writes items as a JSON array. +// +// List commands emit a bare array rather than the envelope the service replied +// with. The envelopes disagree with each other — the OpenAI-shaped APIs wrap +// results in `data`, the ARM-shaped ones in `value` — so passing them through +// would make a caller's parsing depend on which service happens to back a given +// command. They also carry paging fields that this extension does not follow, +// which would suggest there is more to fetch when there is not. +// +// A nil slice encodes as `null`, so it is normalized to an empty array: a +// caller iterating the result should see no elements, not a type error. +func emitJSONList[T any](w io.Writer, items []T) error { + if items == nil { + items = []T{} + } + return emitJSON(w, items) +} + +// emitTable writes a simple aligned table. Rows must match the header width. +func emitTable(w io.Writer, headers []string, rows [][]string) error { + tw := tabwriter.NewWriter(w, 0, 0, 3, ' ', 0) + if _, err := fmt.Fprintln(tw, strings.Join(headers, "\t")); err != nil { + return err + } + for _, row := range rows { + if _, err := fmt.Fprintln(tw, strings.Join(row, "\t")); err != nil { + return err + } + } + return tw.Flush() +} + +// requireFlag returns an error naming the missing flag, used when --no-prompt +// prevents asking for a required value. +func requireFlag(name string) error { + return fmt.Errorf("--%s is required (running with --no-prompt)", name) +} diff --git a/cli/azd/extensions/azure.ai.dataset/internal/cmd/root.go b/cli/azd/extensions/azure.ai.dataset/internal/cmd/root.go new file mode 100644 index 00000000000..e96c9d00927 --- /dev/null +++ b/cli/azd/extensions/azure.ai.dataset/internal/cmd/root.go @@ -0,0 +1,62 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "fmt" + + "github.com/azure/azure-dev/cli/azd/pkg/azdext" + "github.com/fatih/color" + "github.com/spf13/cobra" +) + +// NewRootCommand builds the `azd ai dataset` command tree. +func NewRootCommand() *cobra.Command { + rootCmd, _ := azdext.NewExtensionRootCommand(azdext.ExtensionCommandOptions{ + Name: "dataset", + Use: "dataset [options]", + Short: fmt.Sprintf( + "Create, generate and version Foundry datasets from your terminal. %s", + color.YellowString("(Beta)"), + ), + }) + rootCmd.SilenceUsage = true + rootCmd.SilenceErrors = true + rootCmd.CompletionOptions.DisableDefaultCmd = true + + // The data-plane clients trace requests through the standard logger, which + // Go writes to stderr, so it has to be silenced unless debug was asked for. + // The SDK's own hook is chained rather than replaced, and cobra ignores + // PersistentPreRun entirely once PersistentPreRunE is set. + sdkPreRun := rootCmd.PersistentPreRunE + rootCmd.PersistentPreRunE = func(cmd *cobra.Command, args []string) error { + if sdkPreRun != nil { + if err := sdkPreRun(cmd, args); err != nil { + return err + } + } + setupDebugLogging(cmd.Flags()) + return nil + } + + rootCmd.AddCommand( + newDatasetCreateCommand(), + newDatasetUpdateCommand(), + newDatasetGenerateCommand(), + newDatasetListCommand(), + newDatasetShowCommand(), + newDatasetDeleteCommand(), + newDatasetVersionsCommand(), + newJobCommand(), + ) + + // The manifest declares the `metadata` capability, which azd uses to + // discover this extension's command tree. Without the command registered, + // that discovery fails with "unknown command". + rootCmd.AddCommand(azdext.NewMetadataCommand("1.0", "azure.ai.dataset", func() *cobra.Command { + return rootCmd + })) + + return rootCmd +} diff --git a/cli/azd/extensions/azure.ai.dataset/internal/cmd/surface_test.go b/cli/azd/extensions/azure.ai.dataset/internal/cmd/surface_test.go new file mode 100644 index 00000000000..200ef14b5ec --- /dev/null +++ b/cli/azd/extensions/azure.ai.dataset/internal/cmd/surface_test.go @@ -0,0 +1,130 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "strings" + "testing" + + "github.com/spf13/cobra" + "github.com/spf13/pflag" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// walk visits every command in the tree, skipping the ones azd contributes. +func walk(t *testing.T, cmd *cobra.Command, path []string, visit func(string, *cobra.Command)) { + t.Helper() + for _, child := range cmd.Commands() { + name := strings.Fields(child.Use)[0] + switch name { + case "help", "completion", "listen", "metadata": + continue + } + full := append(append([]string{}, path...), name) + visit(strings.Join(full, " "), child) + walk(t, child, full, visit) + } +} + +// The command tree is the spec's `azd ai dataset` table. The groups moved here +// from azure.ai.evaluations, so this is also what says the move was complete. +func TestCommandTreeMatchesTheSpec(t *testing.T) { + want := []string{ + "create", + "delete", + "generate", + "job", + "job cancel", + "job delete", + "job list", + "job show", + "list", + "show", + "update", + "versions", + "versions list", + } + + var got []string + walk(t, NewRootCommand(), nil, func(path string, _ *cobra.Command) { + got = append(got, path) + }) + + assert.ElementsMatch(t, want, got, + "the command tree changed; update the spec's command table with it") +} + +// Flag names are shared vocabulary across the Foundry extensions. A command +// that invents its own spelling for something the others already name is the +// kind of difference nobody notices until a user types the one they learned +// somewhere else. +func TestFlagVocabularyIsShared(t *testing.T) { + forbidden := map[string]string{ + "--out-file": "--output-file", + "--out-dir": "--output-dir", + "--file": "--from-file", + "--out": "--output-file", + "--dir": "--output-dir", + } + + walk(t, NewRootCommand(), nil, func(path string, cmd *cobra.Command) { + cmd.LocalFlags().VisitAll(func(f *pflag.Flag) { + if want, bad := forbidden["--"+f.Name]; bad { + t.Errorf("%s declares --%s; use %s", path, f.Name, want) + } + }) + }) +} + +// `-o json` and `--no-prompt` come from the azd extension SDK's root command, +// so every command inherits them — until one declares a flag by the same name, +// which silently shadows the global. +func TestNoCommandShadowsAGlobalFlag(t *testing.T) { + global := []string{"output", "no-prompt", "environment", "cwd", "debug"} + + walk(t, NewRootCommand(), nil, func(path string, cmd *cobra.Command) { + for _, name := range global { + assert.Nilf(t, cmd.LocalFlags().Lookup(name), + "%s declares its own --%s, which shadows the global one", path, name) + } + }) +} + +// Every command here reaches the service, so the shared Foundry resolver has to +// be reachable from all of them. +func TestServiceCommandsTakeProjectEndpoint(t *testing.T) { + walk(t, NewRootCommand(), nil, func(path string, cmd *cobra.Command) { + if cmd.RunE == nil { + return + } + assert.NotNil(t, cmd.Flags().Lookup("project-endpoint"), + "%s reaches the service, so it must accept --project-endpoint", path) + }) +} + +// The spec says --from "selects one or more" of the sources, so it has to be +// repeatable. Declared as a plain string it would still accept every documented +// single-source invocation and silently keep only the last of a repeated one. +func TestGenerateFromTakesMoreThanOneSource(t *testing.T) { + flag := find(t, "generate").Flags().Lookup("from") + require.NotNil(t, flag, "generate must offer --from") + + assert.Equal(t, "stringSlice", flag.Value.Type(), + "--from selects one or more sources, so it cannot be a single string") + + for _, source := range generateSources { + assert.Containsf(t, flag.Usage, source, + "--from accepts %q, so its help has to say so", source) + } +} + +func find(t *testing.T, path string) *cobra.Command { + t.Helper() + cmd, _, err := NewRootCommand().Find(strings.Fields(path)) + require.NoError(t, err, "no such command: %s", path) + require.Equal(t, strings.Fields(path)[len(strings.Fields(path))-1], + strings.Fields(cmd.Use)[0], "resolved the wrong command for %s", path) + return cmd +} diff --git a/cli/azd/extensions/azure.ai.dataset/internal/pkg/dataset_api/download_test.go b/cli/azd/extensions/azure.ai.dataset/internal/pkg/dataset_api/download_test.go new file mode 100644 index 00000000000..9ac6f21e92b --- /dev/null +++ b/cli/azd/extensions/azure.ai.dataset/internal/pkg/dataset_api/download_test.go @@ -0,0 +1,47 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package dataset_api + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +// A dataset's URI points at either the blob or the container holding it, +// depending on how it was created, and nothing in the payload says which: +// isSingleFile is true either way. Uploaded datasets end in the file name; +// generated ones end in the container. Downloading a container returns 409. +func TestLooksLikeBlobURI(t *testing.T) { + uploaded := "https://acct.blob.core.windows.net:443/container-guid/azd-smoke-golden.jsonl" + generated := "https://acct.blob.core.windows.net/asayedahme-420d0b21-956c-513b-bb18-f60bfbf5e724" + + require.True(t, looksLikeBlobURI(uploaded), "an uploaded dataset names its file") + require.False(t, looksLikeBlobURI(generated), "a generated dataset names its container") +} + +// A SAS token on the URI must not change the answer. +func TestLooksLikeBlobURIIgnoresQuery(t *testing.T) { + require.True(t, looksLikeBlobURI( + "https://acct.blob.core.windows.net/c/data.jsonl?sv=2021&sig=abc")) + require.False(t, looksLikeBlobURI( + "https://acct.blob.core.windows.net/c?sv=2021&sig=abc")) + require.False(t, looksLikeBlobURI("https://acct.blob.core.windows.net/c/")) +} + +// An evaluation dataset is JSONL, so that is preferred when a container holds +// more than one file. +func TestPickDatasetBlobPrefersJSONL(t *testing.T) { + require.Equal(t, "data.jsonl", + pickDatasetBlob([]string{"_meta.json", "data.jsonl", "readme.txt"})) + require.Equal(t, "data.JSONL", + pickDatasetBlob([]string{"data.JSONL"}), "the extension match is case-insensitive") +} + +// With nothing recognisable, any real file beats returning nothing. +func TestPickDatasetBlobFallsBackToAnyFile(t *testing.T) { + require.Equal(t, "data.csv", pickDatasetBlob([]string{"data.csv"})) + require.Empty(t, pickDatasetBlob([]string{"folder/"})) + require.Empty(t, pickDatasetBlob(nil)) +} diff --git a/cli/azd/extensions/azure.ai.dataset/internal/pkg/dataset_api/list.go b/cli/azd/extensions/azure.ai.dataset/internal/pkg/dataset_api/list.go new file mode 100644 index 00000000000..19b548c3e05 --- /dev/null +++ b/cli/azd/extensions/azure.ai.dataset/internal/pkg/dataset_api/list.go @@ -0,0 +1,103 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package dataset_api + +import ( + "context" + "fmt" + "net/http" + "net/url" + "strconv" + "strings" +) + +// DatasetList is the paged response returned when listing datasets or the +// versions of one dataset. +type DatasetList struct { + Value []Dataset `json:"value"` + NextLink string `json:"nextLink,omitempty"` +} + +// ListDatasets returns the datasets registered on the project. +func (c *DatasetClient) ListDatasets(ctx context.Context, apiVersion string) (*DatasetList, error) { + return doRequestTyped[DatasetList](c, ctx, http.MethodGet, pathDatasets, nil, nil, apiVersion) +} + +// ListDatasetVersions returns every version of a single dataset. +func (c *DatasetClient) ListDatasetVersions( + ctx context.Context, + name string, + apiVersion string, +) (*DatasetList, error) { + path := fmt.Sprintf("%s/%s/versions", pathDatasets, url.PathEscape(name)) + return doRequestTyped[DatasetList](c, ctx, http.MethodGet, path, nil, nil, apiVersion) +} + +// DeleteDatasetVersion removes a single dataset version. +func (c *DatasetClient) DeleteDatasetVersion( + ctx context.Context, + name string, + version string, + apiVersion string, +) error { + path := fmt.Sprintf( + "%s/%s/versions/%s", + pathDatasets, url.PathEscape(name), url.PathEscape(version), + ) + _, err := c.doRequest(ctx, http.MethodDelete, path, nil, nil, apiVersion) + return err +} + +// VersionOrder returns a sortable value for a version string, matching the +// decimal convention NextVersion produces ("1.0", "2.0"). Unparseable versions +// sort lowest. +func VersionOrder(version string) float64 { + v := strings.TrimSpace(version) + if v == "" { + return -1 + } + if f, err := strconv.ParseFloat(v, 64); err == nil { + return f + } + // Fall back to trailing digits, e.g. "v3" -> 3. + i := len(v) + for i > 0 && v[i-1] >= '0' && v[i-1] <= '9' { + i-- + } + if i == len(v) { + return -1 + } + if n, err := strconv.Atoi(v[i:]); err == nil { + return float64(n) + } + return -1 +} + +// VersionGreater reports whether a is a strictly newer version than b. +// +// Both must be orderable; when either is not, the answer is false so an +// unparseable version never triggers a drift failure on its own. +func VersionGreater(a, b string) bool { + orderA, orderB := VersionOrder(a), VersionOrder(b) + if orderA < 0 || orderB < 0 { + return false + } + return orderA > orderB +} + +// LatestVersion returns the highest version in the list, falling back to the +// last entry when none of the versions can be ordered. +func LatestVersion(datasets []Dataset) string { + best := "" + bestOrder := -2.0 + for _, d := range datasets { + if o := VersionOrder(d.Version); o > bestOrder { + bestOrder, best = o, d.Version + } + } + if best == "" && len(datasets) > 0 { + return datasets[len(datasets)-1].Version + } + return best +} diff --git a/cli/azd/extensions/azure.ai.dataset/internal/pkg/dataset_api/models.go b/cli/azd/extensions/azure.ai.dataset/internal/pkg/dataset_api/models.go new file mode 100644 index 00000000000..13fe6da2d9d --- /dev/null +++ b/cli/azd/extensions/azure.ai.dataset/internal/pkg/dataset_api/models.go @@ -0,0 +1,210 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package dataset_api + +import ( + "fmt" + "math" + "os" + "path/filepath" + "strconv" + "strings" +) + +// CreateDatasetRequest is the request body for creating (uploading) a dataset. +type CreateDatasetRequest struct { + Name string `json:"name"` + Version string `json:"version"` + Format string `json:"format"` + Content string `json:"content"` +} + +// Dataset is the response for dataset operations. +// +// The field spelling is not consistent across the surface: the live +// project-endpoint GET returns camelCase (dataUri, isSingleFile), while other +// paths have used snake_case (data_uri, blob_uri, content_uri). Both spellings +// are accepted here because binding only one silently yields an empty URI, +// which then fails much later at download time. +type Dataset struct { + ID string `json:"id,omitempty"` + Name string `json:"name"` + Version string `json:"version"` + Type string `json:"type,omitempty"` + Format string `json:"format,omitempty"` + + // camelCase spellings (project endpoint). + DataURICamel string `json:"dataUri,omitempty"` + BlobURICamel string `json:"blobUri,omitempty"` + ContentURICamel string `json:"contentUri,omitempty"` + IsSingleFile bool `json:"isSingleFile,omitempty"` + ConnectionName string `json:"connectionName,omitempty"` + + // snake_case spellings. + BlobURI string `json:"blob_uri,omitempty"` + DataURI string `json:"data_uri,omitempty"` + ContentURI string `json:"content_uri,omitempty"` +} + +// ResolvedBlobURI returns the first URI the service supplied, across both +// spellings. An empty result means the dataset carries no downloadable URI and +// the caller must fetch a credential instead. +func (d *Dataset) ResolvedBlobURI() string { + for _, candidate := range []string{ + d.BlobURI, d.BlobURICamel, + d.DataURI, d.DataURICamel, + d.ContentURI, d.ContentURICamel, + } { + if candidate != "" { + return candidate + } + } + return "" +} + +// DatasetCredential is the response for dataset credential (SAS token) requests. +// The API returns a nested structure with blobReference and blobReferenceForConsumption. +type DatasetCredential struct { + // Flat fields (legacy format). + BlobURI string `json:"blob_uri,omitempty"` + SAS string `json:"sas,omitempty"` + SASUri string `json:"sas_uri,omitempty"` + + // Nested fields (current API format). + BlobReference *BlobReference `json:"blobReference,omitempty"` + BlobReferenceConsumption *BlobReference `json:"blobReferenceForConsumption,omitempty"` +} + +// BlobReference represents a blob storage reference with credentials. +type BlobReference struct { + BlobURI string `json:"blobUri,omitempty"` + StorageAccountARM string `json:"storageAccountArmId,omitempty"` + Credential *BlobCredential `json:"credential,omitempty"` +} + +// BlobCredential holds SAS credential details for blob access. +type BlobCredential struct { + Type string `json:"type,omitempty"` + SASUri string `json:"sasUri,omitempty"` + SASPath string `json:"sas,omitempty"` +} + +// ResolvedDownloadURI returns the URL to download the dataset. +// Prefers blobReferenceForConsumption.credential.sasUri (current API), +// then blobReference.credential.sasUri, then flat sas_uri, then blob_uri + sas. +func (c *DatasetCredential) ResolvedDownloadURI() string { + // Current API format: nested blob references. + if c.BlobReferenceConsumption != nil && c.BlobReferenceConsumption.Credential != nil { + if uri := c.BlobReferenceConsumption.Credential.SASUri; uri != "" { + return uri + } + } + if c.BlobReference != nil && c.BlobReference.Credential != nil { + if uri := c.BlobReference.Credential.SASUri; uri != "" { + return uri + } + } + // Legacy flat format. + if c.SASUri != "" { + return c.SASUri + } + if c.BlobURI != "" && c.SAS != "" { + return c.BlobURI + "?" + c.SAS + } + return c.BlobURI +} + +// PendingUploadResponse is returned by the startPendingUpload endpoint. +// It contains a SAS URI for uploading blob data and the blob container URI. +type PendingUploadResponse struct { + BlobReference *BlobReference `json:"blobReference,omitempty"` + BlobReferenceConsumption *BlobReference `json:"blobReferenceForConsumption,omitempty"` + PendingUploadID *string `json:"pendingUploadId,omitempty"` + PendingUploadType string `json:"pendingUploadType,omitempty"` + Version string `json:"version,omitempty"` +} + +// ResolvedUploadURI returns the SAS URI for uploading blobs. +func (p *PendingUploadResponse) ResolvedUploadURI() string { + if p.BlobReference != nil && p.BlobReference.Credential != nil { + if uri := p.BlobReference.Credential.SASUri; uri != "" { + return uri + } + } + return "" +} + +// ResolvedBlobURI returns the blob container URI (without SAS) for the finalize request. +func (p *PendingUploadResponse) ResolvedBlobURI() string { + if p.BlobReference != nil { + return p.BlobReference.BlobURI + } + return "" +} + +// FinalizeDatasetRequest is the request body for finalizing a dataset version +// after blob upload. +type FinalizeDatasetRequest struct { + Name string `json:"name"` + Version string `json:"version"` + Description string `json:"description"` + Type string `json:"type"` + IsReference bool `json:"isReference"` + DataURI string `json:"dataUri"` +} + +// NextVersion computes the next dataset version string. +// +// Rules: +// 1. Empty → "1.0" +// 2. Parsable as a decimal number → increment by 1, format as "N.0" +// 3. Ends with trailing digits → increment the trailing numeric part +// 4. Otherwise → append ".1" +func NextVersion(current string) string { + current = strings.TrimSpace(current) + if current == "" { + return "1.0" + } + + // Try parsing as a decimal number (e.g. "1", "1.0", "2.0"). + if f, err := strconv.ParseFloat(current, 64); err == nil { + return strconv.FormatFloat(math.Floor(f)+1, 'f', 1, 64) + } + + // Find trailing digits and increment them. + i := len(current) - 1 + for i >= 0 && current[i] >= '0' && current[i] <= '9' { + i-- + } + if i < len(current)-1 { + prefix := current[:i+1] + n, err := strconv.Atoi(current[i+1:]) + if err == nil { + return prefix + strconv.Itoa(n+1) + } + } + + return current + ".1" +} + +// ReadFirstJSONLFile finds and reads the first .jsonl file in a directory. +func ReadFirstJSONLFile(dir string) (string, error) { + entries, err := os.ReadDir(dir) + if err != nil { + return "", fmt.Errorf("reading directory: %w", err) + } + for _, e := range entries { + if e.IsDir() { + continue + } + if filepath.Ext(e.Name()) == ".jsonl" { + data, err := os.ReadFile(filepath.Join(dir, e.Name())) //nolint:gosec // local artifact path + if err != nil { + return "", fmt.Errorf("reading %s: %w", e.Name(), err) + } + return string(data), nil + } + } + return "", fmt.Errorf("no .jsonl file found in %s", dir) +} diff --git a/cli/azd/extensions/azure.ai.dataset/internal/pkg/dataset_api/operations.go b/cli/azd/extensions/azure.ai.dataset/internal/pkg/dataset_api/operations.go new file mode 100644 index 00000000000..562515d97b2 --- /dev/null +++ b/cli/azd/extensions/azure.ai.dataset/internal/pkg/dataset_api/operations.go @@ -0,0 +1,622 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package dataset_api + +import ( + "bytes" + "context" + "encoding/json" + "encoding/xml" + "errors" + "fmt" + "io" + "log" + "net/http" + "net/url" + "path" + "strings" + + "azureaidataset/internal/version" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/streaming" + "github.com/azure/azure-dev/cli/azd/pkg/azsdk" +) + +// API path prefix for dataset endpoints. +const pathDatasets = "/datasets" + +// DatasetClient provides methods for dataset upload, download, and metadata retrieval. +type DatasetClient struct { + endpoint string + pipeline runtime.Pipeline +} + +// NewDatasetClient creates a new DatasetClient. +func NewDatasetClient(endpoint string, cred azcore.TokenCredential) *DatasetClient { + userAgent := fmt.Sprintf("azd-ext-azure-ai-evaluations/%s", version.Version) + + clientOptions := &policy.ClientOptions{ + Logging: policy.LogOptions{ + AllowedHeaders: []string{"X-Ms-Correlation-Request-Id", "X-Request-Id"}, + IncludeBody: false, + }, + PerCallPolicies: []policy.Policy{ + runtime.NewBearerTokenPolicy(cred, []string{"https://ai.azure.com/.default"}, nil), + azsdk.NewMsCorrelationPolicy(), + azsdk.NewUserAgentPolicy(userAgent), + }, + } + + pipeline := runtime.NewPipeline( + "azure-ai-datasets", + "v1.0.0", + runtime.PipelineOptions{}, + clientOptions, + ) + + return &DatasetClient{ + endpoint: endpoint, + pipeline: pipeline, + } +} + +// NewDatasetClientFromPipeline creates a DatasetClient with a pre-built pipeline. +// This is intended for tests that need to bypass auth policies. +func NewDatasetClientFromPipeline(endpoint string, pipeline runtime.Pipeline) *DatasetClient { + return &DatasetClient{ + endpoint: endpoint, + pipeline: pipeline, + } +} + +// CreateDataset registers a dataset with inline content (upload). +func (c *DatasetClient) CreateDataset( + ctx context.Context, + request *CreateDatasetRequest, + apiVersion string, +) (*Dataset, error) { + return doRequestTyped[Dataset](c, ctx, http.MethodPost, pathDatasets, nil, request, apiVersion) +} + +// UploadNextVersion registers the next version of a dataset, discovering the +// current one from the service when currentVersion is empty. +// +// Prefer this over UploadNewVersion. That function derives the next version +// from whatever it is handed, so an empty value restarts at 1.0 and the +// service rejects the pending upload with a 409 +// TemporaryDataReferencesForExistingAsset as soon as 1.0 exists. Callers +// almost always mean "the version after whatever is registered", which is what +// this does. +// +// The version listing is eventually consistent — it returns nothing for a +// second or two after a version is created — so an empty listing cannot be +// trusted to mean the dataset is new. A conflict is therefore treated as a +// stale read: the listing is re-read, and when it is still behind, the version +// just refused is taken as proof that it exists and the next one is tried. +// Trusting the listing alone left a second upload issued moments after the +// first reporting a 409 to the user for a publish that should simply have +// added a version. +func (c *DatasetClient) UploadNextVersion( + ctx context.Context, + name string, + currentVersion string, + localDir string, + apiVersion string, +) (*Dataset, error) { + if currentVersion == "" { + currentVersion = c.latestRegisteredVersion(ctx, name, apiVersion) + } + + var err error + for range versionConflictAttempts { + var ds *Dataset + ds, err = c.UploadNewVersion(ctx, name, currentVersion, localDir, apiVersion) + if err == nil || !IsVersionConflict(err) { + return ds, err + } + + // The version derived from currentVersion is taken, so it exists + // whatever the listing says. Prefer the listing when it has caught up + // and moved further ahead; otherwise step past what was just refused. + refused := NextVersion(currentVersion) + currentVersion = refused + if latest := c.latestRegisteredVersion(ctx, name, apiVersion); versionAtLeast(latest, refused) { + currentVersion = latest + } + } + return nil, err +} + +// versionConflictAttempts bounds the walk past versions the listing has not +// caught up with. Each attempt is one refused pending upload, so this is short. +const versionConflictAttempts = 4 + +// versionAtLeast reports whether a is a version at or beyond b. +func versionAtLeast(a, b string) bool { + if a == "" { + return false + } + return LatestVersion([]Dataset{{Version: a}, {Version: b}}) == a +} + +// latestRegisteredVersion returns the newest registered version, or empty when +// the dataset is unknown or the listing has not caught up yet. +func (c *DatasetClient) latestRegisteredVersion( + ctx context.Context, + name string, + apiVersion string, +) string { + list, err := c.ListDatasetVersions(ctx, name, apiVersion) + if err != nil || list == nil || len(list.Value) == 0 { + return "" + } + return LatestVersion(list.Value) +} + +// isVersionConflict reports whether the service refused the upload because the +// target version already exists. +func IsVersionConflict(err error) bool { + var respErr *azcore.ResponseError + if !errors.As(err, &respErr) { + return false + } + return respErr.StatusCode == http.StatusConflict +} + +// UploadNewVersion reads the first JSONL file from localDir, computes the next +// version from currentVersion, and uploads it as a new dataset version using +// the 3-step pending upload flow: +// 1. startPendingUpload → get SAS URI +// 2. Upload blob to SAS URI +// 3. Finalize dataset version with dataUri +func (c *DatasetClient) UploadNewVersion( + ctx context.Context, + name string, + currentVersion string, + localDir string, + apiVersion string, +) (*Dataset, error) { + return c.UploadVersion(ctx, name, NextVersion(currentVersion), localDir, apiVersion) +} + +// UploadVersion publishes the dataset at exactly this version. +// +// Separate from UploadNewVersion because its parameter is the version to +// count from, not the one to write: passing "1.0" there publishes 2.0. An +// author who declares a version means that version. +func (c *DatasetClient) UploadVersion( + ctx context.Context, + name string, + version string, + localDir string, + apiVersion string, +) (*Dataset, error) { + content, err := ReadFirstJSONLFile(localDir) + if err != nil { + return nil, fmt.Errorf("reading dataset from %s: %w", localDir, err) + } + + newVersion := version + + // Step 1: Start pending upload to get a SAS URI. + pending, err := c.StartPendingUpload(ctx, name, newVersion, apiVersion) + if err != nil { + return nil, fmt.Errorf("starting pending upload: %w", err) + } + + uploadURI := pending.ResolvedUploadURI() + if uploadURI == "" { + return nil, fmt.Errorf("no upload SAS URI returned from startPendingUpload") + } + + // Step 2: Upload the JSONL file to blob storage. + blobName := name + ".jsonl" + if err := c.UploadBlob(ctx, uploadURI, blobName, []byte(content)); err != nil { + return nil, fmt.Errorf("uploading blob: %w", err) + } + + // Step 3: Finalize the dataset version with the full blob URI. + dataURI := strings.TrimSuffix(pending.ResolvedBlobURI(), "/") + "/" + blobName + return c.FinalizeDatasetVersion(ctx, name, newVersion, dataURI, apiVersion) +} + +// StartPendingUpload initiates a pending upload for a dataset version. +// Returns the SAS URI and blob reference for uploading data. +func (c *DatasetClient) StartPendingUpload( + ctx context.Context, + name string, + version string, + apiVersion string, +) (*PendingUploadResponse, error) { + path := fmt.Sprintf( + "%s/%s/versions/%s/startPendingUpload", + pathDatasets, url.PathEscape(name), url.PathEscape(version), + ) + return doRequestTyped[PendingUploadResponse](c, ctx, http.MethodPost, path, nil, json.RawMessage(`{}`), apiVersion) +} + +// UploadBlob uploads data to a container SAS URI as a block blob. +func (c *DatasetClient) UploadBlob(ctx context.Context, containerSASUri, blobName string, data []byte) error { + u, err := url.Parse(containerSASUri) + if err != nil { + return fmt.Errorf("invalid container SAS URI: %w", err) + } + + // Append blob name to the container path. + u.Path = strings.TrimSuffix(u.Path, "/") + "/" + blobName + + req, err := http.NewRequestWithContext(ctx, http.MethodPut, u.String(), bytes.NewReader(data)) + if err != nil { + return fmt.Errorf("failed to create upload request: %w", err) + } + req.Header.Set("x-ms-blob-type", "BlockBlob") + req.Header.Set("Content-Type", "application/octet-stream") + + httpClient := &http.Client{} + resp, err := httpClient.Do(req) + if err != nil { + return fmt.Errorf("failed to upload blob: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusCreated && resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(resp.Body) + return fmt.Errorf("blob upload failed with status %d: %s", resp.StatusCode, string(body)) + } + + return nil +} + +// FinalizeDatasetVersion completes the dataset version after blob upload +// by sending the metadata (name, version, dataUri) to the API. +func (c *DatasetClient) FinalizeDatasetVersion( + ctx context.Context, + name string, + version string, + dataURI string, + apiVersion string, +) (*Dataset, error) { + path := fmt.Sprintf("%s/%s/versions/%s", pathDatasets, url.PathEscape(name), url.PathEscape(version)) + request := &FinalizeDatasetRequest{ + Name: name, + Version: version, + Type: "uri_file", + DataURI: dataURI, + } + return doRequestTyped[Dataset](c, ctx, http.MethodPut, path, nil, request, apiVersion) +} + +// GetDataset retrieves metadata for a dataset by name and version. +func (c *DatasetClient) GetDataset( + ctx context.Context, + name string, + version string, + apiVersion string, +) (*Dataset, error) { + path := fmt.Sprintf("%s/%s/versions/%s", pathDatasets, url.PathEscape(name), url.PathEscape(version)) + return doRequestTyped[Dataset](c, ctx, http.MethodGet, path, nil, nil, apiVersion) +} + +// GetDatasetCredential retrieves a SAS credential for downloading a dataset from blob storage. +func (c *DatasetClient) GetDatasetCredential( + ctx context.Context, + name string, + version string, + apiVersion string, +) (*DatasetCredential, error) { + path := fmt.Sprintf( + "%s/%s/versions/%s/credentials", + pathDatasets, url.PathEscape(name), url.PathEscape(version), + ) + return doRequestTyped[DatasetCredential](c, ctx, http.MethodPost, path, nil, nil, apiVersion) +} + +// DownloadDatasetContent fetches a dataset version's content, whether its URI +// names a blob or a container. +// +// The two differ by origin, not by any field: a dataset uploaded through +// startPendingUpload gets a URI ending in the file name, while one produced by +// a generation job gets the container it was written into, with isSingleFile +// true either way. Downloading the container directly returns a 409, so the +// blob inside has to be found first. +// +// A credential is always fetched, because the URI on the dataset carries no +// SAS token and an unauthenticated read fails. +func (c *DatasetClient) DownloadDatasetContent( + ctx context.Context, + name string, + version string, + apiVersion string, +) ([]byte, error) { + cred, err := c.GetDatasetCredential(ctx, name, version, apiVersion) + if err != nil { + return nil, fmt.Errorf("reading download credentials for %q: %w", name, err) + } + + sasURI := cred.ResolvedDownloadURI() + if sasURI == "" { + return nil, fmt.Errorf("no download URI returned for dataset %q", name) + } + + // A URI whose last path segment carries a file extension is the blob + // itself; anything else is the container holding it. + if looksLikeBlobURI(sasURI) { + data, err := c.DownloadDataset(ctx, sasURI) + if err == nil { + return data, nil + } + log.Printf("[dataset_api] direct download failed (%v); treating the URI as a container", err) + } + + names, err := c.ListContainerBlobs(ctx, sasURI) + if err != nil { + return nil, fmt.Errorf("listing the content of dataset %q: %w", name, err) + } + blobName := pickDatasetBlob(names) + if blobName == "" { + return nil, fmt.Errorf("dataset %q holds no downloadable file", name) + } + return c.DownloadBlob(ctx, sasURI, blobName) +} + +// looksLikeBlobURI reports whether the URI's final segment names a file. +func looksLikeBlobURI(raw string) bool { + u, err := url.Parse(raw) + if err != nil { + return false + } + last := path.Base(strings.TrimSuffix(u.Path, "/")) + return path.Ext(last) != "" +} + +// pickDatasetBlob chooses the file to read from a container, preferring JSONL +// since that is what an evaluation dataset is. +func pickDatasetBlob(names []string) string { + for _, n := range names { + if strings.EqualFold(path.Ext(n), ".jsonl") { + return n + } + } + for _, n := range names { + if n != "" && !strings.HasSuffix(n, "/") { + return n + } + } + return "" +} + +// DownloadDataset downloads dataset content from blob storage using a SAS-authenticated URL. +// Returns the raw content as bytes. The downloadURL should be the full URL with SAS token +// (e.g., from DatasetCredential.ResolvedDownloadURI()). +func (c *DatasetClient) DownloadDataset(ctx context.Context, downloadURL string) ([]byte, error) { + req, err := runtime.NewRequest(ctx, http.MethodGet, downloadURL) + if err != nil { + return nil, fmt.Errorf("failed to create download request: %w", err) + } + + // Use a plain HTTP client for blob downloads — the SAS token in the URL provides + // authentication, and Azure SDK pipeline policies (bearer token, correlation ID) + // should not be sent to Azure Blob Storage endpoints. + httpClient := &http.Client{} + resp, err := httpClient.Do(req.Raw()) + if err != nil { + return nil, fmt.Errorf("failed to download dataset from blob: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("blob download failed with status %d", resp.StatusCode) + } + + data, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("failed to read dataset content: %w", err) + } + + log.Printf("[dataset_api] downloaded %d bytes", len(data)) + return data, nil +} + +// ListContainerBlobs lists blobs in a container using a container-level SAS URI. +// The containerSASUri should include the SAS token (e.g., from credential.sasUri with sr=c). +// Returns a list of blob names found in the container. +func (c *DatasetClient) ListContainerBlobs(ctx context.Context, containerSASUri string) ([]string, error) { + // Parse the container URI and append list query parameters. + u, err := url.Parse(containerSASUri) + if err != nil { + return nil, fmt.Errorf("invalid container SAS URI: %w", err) + } + + q := u.Query() + q.Set("restype", "container") // cspell:ignore restype — Azure Storage API query parameter + q.Set("comp", "list") + u.RawQuery = q.Encode() + + log.Printf("[dataset_api] listing blobs: %s", u.Redacted()) + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil) + if err != nil { + return nil, fmt.Errorf("failed to create list request: %w", err) + } + + httpClient := &http.Client{} + resp, err := httpClient.Do(req) + if err != nil { + return nil, fmt.Errorf("failed to list container blobs: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("container list failed with status %d", resp.StatusCode) + } + + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("failed to read list response: %w", err) + } + + // Parse XML blob listing to extract blob names. + names := parseBlobNames(string(body)) + log.Printf("[dataset_api] found %d blobs in container", len(names)) + return names, nil +} + +// DownloadBlob downloads a single blob from a container using the container SAS URI +// and the blob name. Returns the blob content as bytes. +func (c *DatasetClient) DownloadBlob(ctx context.Context, containerSASUri, blobName string) ([]byte, error) { + u, err := url.Parse(containerSASUri) + if err != nil { + return nil, fmt.Errorf("invalid container SAS URI: %w", err) + } + + // Append blob name to the container path. + u.Path = strings.TrimSuffix(u.Path, "/") + "/" + blobName + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil) + if err != nil { + return nil, fmt.Errorf("failed to create blob download request: %w", err) + } + + httpClient := &http.Client{} + resp, err := httpClient.Do(req) + if err != nil { + return nil, fmt.Errorf("failed to download blob: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("blob download failed with status %d for %s", resp.StatusCode, blobName) + } + + data, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("failed to read blob content: %w", err) + } + + log.Printf("[dataset_api] downloaded blob %s (%d bytes)", blobName, len(data)) + return data, nil +} + +// parseBlobNames extracts blob names from the Azure Blob Storage XML list response +// using proper XML parsing against the EnumerationResults schema. +func parseBlobNames(xmlBody string) []string { + type blob struct { + Name string `xml:"Name"` + } + type blobs struct { + Blob []blob `xml:"Blob"` + } + type enumerationResults struct { + Blobs blobs `xml:"Blobs"` + } + + var result enumerationResults + if err := xml.Unmarshal([]byte(xmlBody), &result); err != nil { + return nil + } + + names := make([]string, 0, len(result.Blobs.Blob)) + for _, b := range result.Blobs.Blob { + if b.Name != "" { + names = append(names, b.Name) + } + } + return names +} + +// doRequest performs an HTTP request against the dataset API and returns the raw response body. +func (c *DatasetClient) doRequest( + ctx context.Context, + method string, + path string, + query map[string]string, + body any, + apiVersion string, +) ([]byte, error) { + u, err := url.Parse(c.endpoint) + if err != nil { + return nil, fmt.Errorf("invalid endpoint URL: %w", err) + } + + u.Path += path + q := u.Query() + if apiVersion != "" { + q.Set("api-version", apiVersion) + } + for k, v := range query { + q.Set(k, v) + } + u.RawQuery = q.Encode() + + req, err := runtime.NewRequest(ctx, method, u.String()) + if err != nil { + return nil, fmt.Errorf("failed to create request: %w", err) + } + + log.Printf("[dataset_api] %s %s", method, u.Redacted()) + + if body != nil { + payload, err := json.Marshal(body) + if err != nil { + return nil, fmt.Errorf("failed to marshal request: %w", err) + } + if err := req.SetBody(streaming.NopCloser(bytes.NewReader(payload)), "application/json"); err != nil { + return nil, fmt.Errorf("failed to set request body: %w", err) + } + } + + resp, err := c.pipeline.Do(req) + if err != nil { + return nil, fmt.Errorf("HTTP request failed: %w", err) + } + defer resp.Body.Close() + + respBody, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("failed to read response body: %w", err) + } + + log.Printf("[dataset_api] response status: %d", resp.StatusCode) + + // 204 belongs here for the same reason it does in eval_api: a delete that + // removed the version answers No Content, and rejecting that reports every + // successful delete as an error. + if !runtime.HasStatusCode(resp, + http.StatusOK, http.StatusCreated, http.StatusAccepted, http.StatusNoContent) { + resp.Body = io.NopCloser(bytes.NewReader(respBody)) + return nil, runtime.NewResponseError(resp) + } + + return respBody, nil +} + +// doRequestTyped performs an HTTP request and unmarshals the response into T. +func doRequestTyped[T any]( + c *DatasetClient, + ctx context.Context, + method string, + path string, + query map[string]string, + body any, + apiVersion string, +) (*T, error) { + respBody, err := c.doRequest(ctx, method, path, query, body, apiVersion) + if err != nil { + return nil, err + } + + if len(respBody) == 0 { + return new(T), nil + } + + var result T + if err := json.Unmarshal(respBody, &result); err != nil { + return nil, fmt.Errorf("failed to parse response: %w", err) + } + + return &result, nil +} diff --git a/cli/azd/extensions/azure.ai.dataset/internal/pkg/dataset_api/upload_version_test.go b/cli/azd/extensions/azure.ai.dataset/internal/pkg/dataset_api/upload_version_test.go new file mode 100644 index 00000000000..164cc6d415b --- /dev/null +++ b/cli/azd/extensions/azure.ai.dataset/internal/pkg/dataset_api/upload_version_test.go @@ -0,0 +1,157 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package dataset_api + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "sync" + "testing" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// uploadServer answers the three-step publish, refusing any version in taken +// and reporting whatever the listing is told to report. +type uploadServer struct { + mu sync.Mutex + taken map[string]bool + listing []string + attempts []string +} + +func (s *uploadServer) handler(t *testing.T, base func() string) http.HandlerFunc { + t.Helper() + return func(w http.ResponseWriter, r *http.Request) { + s.mu.Lock() + defer s.mu.Unlock() + w.Header().Set("Content-Type", "application/json") + + switch { + case strings.HasSuffix(r.URL.Path, "/startPendingUpload"): + version := strings.Split(r.URL.Path, "/versions/")[1] + version = strings.TrimSuffix(version, "/startPendingUpload") + s.attempts = append(s.attempts, version) + if s.taken[version] { + w.WriteHeader(http.StatusConflict) + _, _ = w.Write([]byte(`{"error":{"code":"Conflict"}}`)) + return + } + require.NoError(t, json.NewEncoder(w).Encode(map[string]any{ + "blobReference": map[string]any{ + "blobUri": base() + "/c", + "storageAccountArmId": "id", + "credential": map[string]any{"sasUri": base() + "/c?sig=x"}, + }, + })) + + case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/versions"): + values := []map[string]any{} + for _, v := range s.listing { + values = append(values, map[string]any{"name": "ds", "version": v}) + } + require.NoError(t, json.NewEncoder(w).Encode(map[string]any{"value": values})) + + case r.Method == http.MethodPut: + version := r.URL.Path[strings.LastIndex(r.URL.Path, "/")+1:] + s.taken[version] = true + require.NoError(t, json.NewEncoder(w).Encode(map[string]any{ + "name": "ds", "version": version, + })) + + default: + // The blob PUT. + w.WriteHeader(http.StatusCreated) + } + } +} + +// The version listing lags a publish, so a second upload can be told the +// dataset is new and restart at a version that already exists. Trusting the +// listing alone surfaced that 409 to the user for a publish that should simply +// have added a version. +func TestUploadNextVersionWalksPastAStaleListing(t *testing.T) { + server := &uploadServer{taken: map[string]bool{"1.0": true}} + // The listing has not caught up: it still reports nothing at all. + httpServer := func() *httptest.Server { + var s *httptest.Server + s = httptest.NewServer(server.handler(t, func() string { return s.URL })) + return s + }() + t.Cleanup(httpServer.Close) + + client := NewDatasetClientFromPipeline( + httpServer.URL, runtime.NewPipeline("test", "v1", runtime.PipelineOptions{}, nil)) + + dir := t.TempDir() + require.NoError(t, os.WriteFile( + filepath.Join(dir, "rows.jsonl"), []byte("{\"query\":\"q\"}\n"), 0o600)) + + ds, err := client.UploadNextVersion(context.Background(), "ds", "", dir, "2025-11-15-preview") + require.NoError(t, err, "a stale listing must not surface as a conflict") + assert.Equal(t, "2.0", ds.Version) + assert.Equal(t, []string{"1.0", "2.0"}, server.attempts, + "the version just refused is proof it exists, so the next one is tried") +} + +// When the listing has caught up and is further ahead than the refused +// version, it is the better answer: it skips versions somebody else published. +func TestUploadNextVersionPrefersACaughtUpListing(t *testing.T) { + server := &uploadServer{ + taken: map[string]bool{"1.0": true, "2.0": true, "3.0": true}, + listing: []string{"1.0", "2.0", "3.0"}, + } + httpServer := func() *httptest.Server { + var s *httptest.Server + s = httptest.NewServer(server.handler(t, func() string { return s.URL })) + return s + }() + t.Cleanup(httpServer.Close) + + client := NewDatasetClientFromPipeline( + httpServer.URL, runtime.NewPipeline("test", "v1", runtime.PipelineOptions{}, nil)) + + dir := t.TempDir() + require.NoError(t, os.WriteFile( + filepath.Join(dir, "rows.jsonl"), []byte("{\"query\":\"q\"}\n"), 0o600)) + + ds, err := client.UploadNextVersion(context.Background(), "ds", "", dir, "2025-11-15-preview") + require.NoError(t, err) + assert.Equal(t, "4.0", ds.Version) +} + +// A service that refuses everything must end in the conflict rather than +// looping: an unbounded walk would hammer the service on a real failure. +func TestUploadNextVersionGivesUpBounded(t *testing.T) { + server := &uploadServer{taken: map[string]bool{}} + for _, v := range []string{"1.0", "2.0", "3.0", "4.0", "5.0", "6.0"} { + server.taken[v] = true + } + httpServer := func() *httptest.Server { + var s *httptest.Server + s = httptest.NewServer(server.handler(t, func() string { return s.URL })) + return s + }() + t.Cleanup(httpServer.Close) + + client := NewDatasetClientFromPipeline( + httpServer.URL, runtime.NewPipeline("test", "v1", runtime.PipelineOptions{}, nil)) + + dir := t.TempDir() + require.NoError(t, os.WriteFile( + filepath.Join(dir, "rows.jsonl"), []byte("{\"query\":\"q\"}\n"), 0o600)) + + _, err := client.UploadNextVersion(context.Background(), "ds", "", dir, "2025-11-15-preview") + require.Error(t, err) + assert.True(t, IsVersionConflict(err)) + assert.Len(t, server.attempts, versionConflictAttempts) +} diff --git a/cli/azd/extensions/azure.ai.dataset/internal/pkg/dataset_api/uri_test.go b/cli/azd/extensions/azure.ai.dataset/internal/pkg/dataset_api/uri_test.go new file mode 100644 index 00000000000..e6cc46c2e3d --- /dev/null +++ b/cli/azd/extensions/azure.ai.dataset/internal/pkg/dataset_api/uri_test.go @@ -0,0 +1,75 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package dataset_api + +import ( + "encoding/json" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// The service spells these fields inconsistently, and a URI read from the +// wrong spelling comes back empty rather than wrong — which is how the dataset +// URI went unbound the first time. +func TestDatasetResolvedBlobURI_AcceptsEitherSpelling(t *testing.T) { + cases := map[string]string{ + `{"dataUri":"https://x/y.jsonl"}`: "https://x/y.jsonl", + `{"data_uri":"https://x/y.jsonl"}`: "https://x/y.jsonl", + `{"blobUri":"https://x/b.jsonl"}`: "https://x/b.jsonl", + `{"contentUri":"https://x/c.jsonl"}`: "https://x/c.jsonl", + } + for body, want := range cases { + var ds Dataset + require.NoError(t, json.Unmarshal([]byte(body), &ds), body) + assert.Equal(t, want, ds.ResolvedBlobURI(), body) + } + + var none Dataset + require.NoError(t, json.Unmarshal([]byte(`{"name":"x"}`), &none)) + assert.Empty(t, none.ResolvedBlobURI(), + "no URI means the caller has to fetch a credential, not that the dataset is unreadable") +} + +// An upload needs the SAS-bearing URI to write to and the plain one to +// finalize with. Confusing them fails at different stages, so both are read +// from their own place. +func TestPendingUploadURIs(t *testing.T) { + var p PendingUploadResponse + require.NoError(t, json.Unmarshal([]byte(`{ + "blobReference": { + "blobUri": "https://acct.blob.core.windows.net/container", + "credential": { "sasUri": "https://acct.blob.core.windows.net/container?sig=abc" } + } + }`), &p)) + + assert.Equal(t, "https://acct.blob.core.windows.net/container?sig=abc", p.ResolvedUploadURI(), + "the upload target carries the SAS") + assert.Equal(t, "https://acct.blob.core.windows.net/container", p.ResolvedBlobURI(), + "the finalize URI does not") + + var empty PendingUploadResponse + assert.Empty(t, empty.ResolvedUploadURI()) + assert.Empty(t, empty.ResolvedBlobURI()) +} + +// Credentials arrive in two shapes and the consumption one takes precedence, +// because that is the one scoped for reading. +func TestCredentialResolvedDownloadURI(t *testing.T) { + var c DatasetCredential + require.NoError(t, json.Unmarshal([]byte(`{ + "blobReferenceForConsumption": { "credential": { "sasUri": "https://acct/read?sig=r" } }, + "blobReference": { "credential": { "sasUri": "https://acct/write?sig=w" } } + }`), &c)) + assert.Equal(t, "https://acct/read?sig=r", c.ResolvedDownloadURI()) + + var legacy DatasetCredential + require.NoError(t, json.Unmarshal([]byte(`{"sas_uri":"https://acct/legacy?sig=l"}`), &legacy)) + assert.Equal(t, "https://acct/legacy?sig=l", legacy.ResolvedDownloadURI(), + "the flat spelling is still honoured") + + var none DatasetCredential + assert.Empty(t, none.ResolvedDownloadURI()) +} diff --git a/cli/azd/extensions/azure.ai.dataset/internal/pkg/dataset_api/version_test.go b/cli/azd/extensions/azure.ai.dataset/internal/pkg/dataset_api/version_test.go new file mode 100644 index 00000000000..052a63ae504 --- /dev/null +++ b/cli/azd/extensions/azure.ai.dataset/internal/pkg/dataset_api/version_test.go @@ -0,0 +1,68 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package dataset_api + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +// Drift detection compares the version on the service with the one recorded at +// the last deploy, so the ordering has to be numeric rather than lexical: +// "10.0" is newer than "9.0" even though it sorts earlier as a string. +func TestVersionGreater(t *testing.T) { + cases := []struct { + a, b string + want bool + }{ + {"2.0", "1.0", true}, + {"1.0", "2.0", false}, + {"1.0", "1.0", false}, + {"10.0", "9.0", true}, + {"9.0", "10.0", false}, + {"v3", "v2", true}, + } + + for _, tc := range cases { + require.Equal(t, tc.want, VersionGreater(tc.a, tc.b), + "VersionGreater(%q, %q)", tc.a, tc.b) + } +} + +// An unorderable version must never trigger a drift failure on its own: the +// deploy would be blocked with no way for the author to reason about it. +func TestVersionGreaterIgnoresUnorderable(t *testing.T) { + require.False(t, VersionGreater("draft", "1.0")) + require.False(t, VersionGreater("1.0", "draft")) + require.False(t, VersionGreater("", "1.0")) + require.False(t, VersionGreater("1.0", "")) +} + +// The two upload entry points read their version argument differently, and the +// difference is the whole point: UploadNewVersion counts from it, UploadVersion +// writes it. Passing "1.0" to the counting one publishes 2.0, which is not what +// an author who wrote version: "1.0" asked for. +func TestNextVersionCountsFromTheArgument(t *testing.T) { + if got := NextVersion("1.0"); got != "2.0" { + t.Fatalf("NextVersion(1.0) = %q, want 2.0", got) + } + if got := NextVersion("1"); got != "2.0" { + t.Fatalf("NextVersion(1) = %q, want 2.0", got) + } + // An unknown current version starts the sequence rather than guessing. + if got := NextVersion(""); got != "1.0" { + t.Fatalf("NextVersion(empty) = %q, want 1.0", got) + } +} + +func TestLatestVersionOrdersNumerically(t *testing.T) { + got := LatestVersion([]Dataset{{Version: "1.0"}, {Version: "10.0"}, {Version: "2.0"}}) + if got != "10.0" { + t.Fatalf("LatestVersion = %q, want 10.0 (numeric, not lexical)", got) + } + if LatestVersion(nil) != "" { + t.Fatal("LatestVersion(nil) should be empty") + } +} diff --git a/cli/azd/extensions/azure.ai.dataset/internal/pkg/gen_api/errors.go b/cli/azd/extensions/azure.ai.dataset/internal/pkg/gen_api/errors.go new file mode 100644 index 00000000000..bd68ce76e6c --- /dev/null +++ b/cli/azd/extensions/azure.ai.dataset/internal/pkg/gen_api/errors.go @@ -0,0 +1,49 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package gen_api + +import ( + "errors" + "net/http" + "strings" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore" +) + +// IsConflict reports whether the service refused because the resource is busy. +func IsConflict(err error) bool { + var respErr *azcore.ResponseError + if !errors.As(err, &respErr) { + return false + } + return respErr.StatusCode == http.StatusConflict +} + +// IsNotFound reports whether the service answered 404. +func IsNotFound(err error) bool { + var respErr *azcore.ResponseError + if !errors.As(err, &respErr) { + return false + } + return respErr.StatusCode == http.StatusNotFound +} + +// IsTransientError reports whether err is worth retrying: throttling, a server +// fault, or a dropped connection. +func IsTransientError(err error) bool { + if err == nil { + return false + } + + var respErr *azcore.ResponseError + if errors.As(err, &respErr) { + return respErr.StatusCode == http.StatusTooManyRequests || + respErr.StatusCode >= http.StatusInternalServerError + } + + msg := err.Error() + return strings.Contains(msg, "connection reset") || + strings.Contains(msg, "connection refused") || + strings.Contains(msg, "EOF") +} diff --git a/cli/azd/extensions/azure.ai.dataset/internal/pkg/gen_api/generation.go b/cli/azd/extensions/azure.ai.dataset/internal/pkg/gen_api/generation.go new file mode 100644 index 00000000000..fd7af65c614 --- /dev/null +++ b/cli/azd/extensions/azure.ai.dataset/internal/pkg/gen_api/generation.go @@ -0,0 +1,180 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package gen_api + +import ( + "path/filepath" + "strings" + "time" +) + +// --------------------------------------------------------------------------- +// Generation source building +// --------------------------------------------------------------------------- + +// TraceOptions holds optional trace inclusion parameters for generation sources. +type TraceOptions struct { + Days int +} + +// WithoutAgentSource returns the sources with the agent entry removed. +// +// Agent-seeded data generation currently fails server-side for every agent, +// while the same request carrying only the prompt succeeds, so this is what a +// retry falls back to. +func WithoutAgentSource(sources []GenerationSource) []GenerationSource { + kept := make([]GenerationSource, 0, len(sources)) + for _, s := range sources { + if s.Type == "agent" { + continue + } + kept = append(kept, s) + } + return kept +} + +// HasPromptSource reports whether anything remains to generate from. +func HasPromptSource(sources []GenerationSource) bool { + for _, s := range sources { + if s.Type == "prompt" && s.Prompt != "" { + return true + } + } + return false +} + +// BuildGenerationSources emits the sources the caller selected, in a stable +// order, along with the ones it asked for and nothing could be built from. +// +// kinds is what --from named. An empty kinds means "whatever this plan has to +// offer" and reports nothing missing: the caller expressed no preference, so +// there is nothing to disappoint. Naming a kind explicitly is a request, and a +// request that cannot be built is worth saying out loud rather than quietly +// submitting a job seeded from less than was asked for. +func BuildGenerationSources( + kinds []string, + agentName, version, instruction string, + traces *TraceOptions, +) (sources []GenerationSource, unbuildable []string) { + want := map[string]bool{} + for _, k := range kinds { + want[k] = true + } + // Empty kinds selects everything available; a populated one selects only + // what it names. + selected := func(kind string) bool { + return len(want) == 0 || want[kind] + } + // asked distinguishes "the default swept this up" from "the user typed it", + // which is what decides whether an empty-handed source is an error. + asked := func(kind string) bool { return want[kind] } + + // The agent is settled first because whether it was built decides whether + // its instructions have anything to be the instructions of. + var agentSource *GenerationSource + if selected("agent") { + switch { + case agentName != "": + agentSource = &GenerationSource{Type: "agent", AgentName: agentName} + if version != "" { + agentSource.AgentVersion = version + } + case asked("agent"): + unbuildable = append(unbuildable, "agent") + } + } + + // Generating from an agent means generating from its instructions, so they + // travel with it as a prompt. That is also the only shape the service + // currently honours: the agent source alone fails for every agent, and the + // prompt is what the retry in generateDataset falls back to. Without this, + // `--from agent` would be a request that always fails. + promptCarriesTheAgent := agentSource != nil && asked("agent") + if selected("prompt") || promptCarriesTheAgent { + switch { + case instruction != "": + sources = append(sources, GenerationSource{ + Type: "prompt", + Prompt: instruction, + }) + case asked("prompt"): + unbuildable = append(unbuildable, "prompt") + } + } + + if agentSource != nil { + sources = append(sources, *agentSource) + } + + if selected("traces") { + // A window narrows the request; it does not authorize it. Asking for + // traces without one means every trace the agent has. + switch { + case traces != nil && traces.Days > 0: + sources = append(sources, GenerationSource{ + Type: "traces", + AgentName: agentName, + StartTime: time.Now().AddDate(0, 0, -traces.Days).Unix(), + }) + case asked("traces"): + sources = append(sources, GenerationSource{ + Type: "traces", + AgentName: agentName, + }) + } + } + + // The service takes a file's rows through the dataset upload path, not + // through a generation source, so there is nothing here to build one from. + if asked("file") { + unbuildable = append(unbuildable, "file") + } + + return sources, unbuildable +} + +// --------------------------------------------------------------------------- +// Request builders +// --------------------------------------------------------------------------- + +// NewDataGenerationJobRequest builds a DataGenerationJobRequest from the +// provided parameters. Currently, it's always "simple_qna" type with multiple sources +func NewDataGenerationJobRequest( + name, evalModel string, + maxSamples int, + sources []GenerationSource, +) *DataGenerationJobRequest { + return &DataGenerationJobRequest{ + Inputs: DataGenerationInputs{ + Name: name, + Scenario: "evaluation", + Options: DataGenerationOptions{ + Type: "simple_qna", + MaxSamples: maxSamples, + ModelOptions: ModelOptions{ + Model: evalModel, + }, + }, + Sources: sources, + }, + } +} + +// --------------------------------------------------------------------------- +// Dataset name detection +// --------------------------------------------------------------------------- + +// IsDatasetName returns true when the value looks like a registered dataset +// name rather than a local file path. A name has no path separators and no +// common data-file extension (.jsonl, .json, .csv). +func IsDatasetName(value string) bool { + if value == "" { + return false + } + if strings.ContainsAny(value, "/\\") { + return false + } + ext := strings.ToLower(filepath.Ext(value)) + return ext != ".jsonl" && ext != ".json" && ext != ".csv" +} diff --git a/cli/azd/extensions/azure.ai.dataset/internal/pkg/gen_api/generation_test.go b/cli/azd/extensions/azure.ai.dataset/internal/pkg/gen_api/generation_test.go new file mode 100644 index 00000000000..d6cce82e5e8 --- /dev/null +++ b/cli/azd/extensions/azure.ai.dataset/internal/pkg/gen_api/generation_test.go @@ -0,0 +1,232 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package gen_api + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// kindsOf reduces the built sources to what --from talks about, which is the +// only part these tests are asserting on. +func kindsOf(sources []GenerationSource) []string { + kinds := make([]string, 0, len(sources)) + for _, s := range sources { + kinds = append(kinds, s.Type) + } + return kinds +} + +// Naming a source is a request to send that one, not a hint. Everything the +// plan could otherwise have offered stays out of the request. +func TestBuildGenerationSources_SendsOnlyWhatFromNamed(t *testing.T) { + sources, unbuildable := BuildGenerationSources( + []string{"traces"}, + "support-agent", "3", "answer support questions", + &TraceOptions{Days: 7}, + ) + + assert.Equal(t, []string{"traces"}, kindsOf(sources)) + assert.Empty(t, unbuildable) +} + +// Generating from an agent means generating from its instructions, so asking +// for the agent carries them. It is also the only shape the service honours: +// the agent source on its own fails for every agent, so a `--from agent` that +// dropped the prompt would be a request that always fails. +func TestBuildGenerationSources_AgentCarriesItsInstructions(t *testing.T) { + sources, unbuildable := BuildGenerationSources( + []string{"agent"}, "support-agent", "3", "answer support questions", nil, + ) + + assert.Equal(t, []string{"prompt", "agent"}, kindsOf(sources)) + assert.Equal(t, "answer support questions", sources[0].Prompt) + assert.Empty(t, unbuildable) +} + +// The instructions ride along with the agent; they do not stand in for it. An +// agent nobody named is still nothing to generate from. +func TestBuildGenerationSources_InstructionsDoNotSubstituteForTheAgent(t *testing.T) { + sources, unbuildable := BuildGenerationSources( + []string{"agent"}, "", "", "answer support questions", nil, + ) + + assert.Empty(t, sources) + assert.Equal(t, []string{"agent"}, unbuildable) +} + +// The agent name travels with the traces source: it is what scopes the query +// to this agent's conversations rather than the whole project's. +func TestBuildGenerationSources_TracesCarryTheAgent(t *testing.T) { + sources, _ := BuildGenerationSources( + []string{"traces"}, "support-agent", "", "", &TraceOptions{Days: 7}, + ) + + require.Len(t, sources, 1) + assert.Equal(t, "support-agent", sources[0].AgentName) +} + +// A day window narrows the trace query; it is not what authorizes it. The +// documented `dataset generate --from traces` carries no window, and it +// has to mean "every trace" rather than "no traces". +func TestBuildGenerationSources_TracesWithoutAWindowAreUnbounded(t *testing.T) { + sources, unbuildable := BuildGenerationSources( + []string{"traces"}, "support-agent", "", "", nil, + ) + + require.Len(t, sources, 1) + assert.Equal(t, "traces", sources[0].Type) + assert.Zero(t, sources[0].StartTime, + "an absent window must leave start_time off the wire, not pin it to now") + assert.Empty(t, unbuildable) +} + +func TestBuildGenerationSources_TraceWindowBecomesAStartTime(t *testing.T) { + sources, _ := BuildGenerationSources( + []string{"traces"}, "support-agent", "", "", &TraceOptions{Days: 7}, + ) + + require.Len(t, sources, 1) + want := time.Now().AddDate(0, 0, -7).Unix() + assert.InDelta(t, want, sources[0].StartTime, 60) +} + +// No --from is no preference, so the plan sends everything it happens to have. +func TestBuildGenerationSources_EmptyFromSendsWhatThePlanHas(t *testing.T) { + sources, unbuildable := BuildGenerationSources( + nil, "support-agent", "3", "answer support questions", &TraceOptions{Days: 7}, + ) + + assert.Equal(t, []string{"prompt", "agent", "traces"}, kindsOf(sources)) + assert.Empty(t, unbuildable) +} + +// Expressing no preference cannot disappoint one, so an empty --from reports +// nothing missing however little the plan turns out to hold. +func TestBuildGenerationSources_EmptyFromNeverReportsMissingSources(t *testing.T) { + sources, unbuildable := BuildGenerationSources(nil, "", "", "", nil) + + assert.Empty(t, sources) + assert.Empty(t, unbuildable) +} + +// Asking for a source the plan cannot build has to surface, because the job is +// billed and what comes back looks the same either way. +func TestBuildGenerationSources_ReportsWhatItCouldNotBuild(t *testing.T) { + tests := []struct { + name string + kinds []string + agentName string + instruction string + want []string + }{ + { + name: "prompt without an instruction", + kinds: []string{"prompt"}, + want: []string{"prompt"}, + }, + { + name: "agent without a target", + kinds: []string{"agent"}, + want: []string{"agent"}, + }, + { + name: "file is not a generation source at all", + kinds: []string{"file"}, + want: []string{"file"}, + }, + { + name: "several at once", + kinds: []string{"prompt", "agent"}, + want: []string{"agent", "prompt"}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + sources, unbuildable := BuildGenerationSources( + tt.kinds, tt.agentName, "", tt.instruction, nil, + ) + + assert.Empty(t, sources) + assert.Equal(t, tt.want, unbuildable) + }) + } +} + +// A request that names two sources and can only build one still reports the +// one it could not, rather than being satisfied by the other's success. +func TestBuildGenerationSources_OneBuiltSourceDoesNotExcuseAMissingOne(t *testing.T) { + sources, unbuildable := BuildGenerationSources( + []string{"agent", "prompt"}, "support-agent", "", "", nil, + ) + + assert.Equal(t, []string{"agent"}, kindsOf(sources)) + assert.Equal(t, []string{"prompt"}, unbuildable) +} + +// `file` is only unbuildable when it was asked for. The default sweep must not +// invent a complaint about a source nobody named. +func TestBuildGenerationSources_FileIsOnlyReportedWhenAskedFor(t *testing.T) { + _, unbuildable := BuildGenerationSources( + nil, "support-agent", "", "instruction", &TraceOptions{Days: 7}, + ) + + assert.Empty(t, unbuildable) +} + +func TestBuildGenerationSources_AgentVersionIsOptional(t *testing.T) { + withVersion, _ := BuildGenerationSources([]string{"agent"}, "support-agent", "3", "", nil) + require.Len(t, withVersion, 1) + assert.Equal(t, "3", withVersion[0].AgentVersion) + + withoutVersion, _ := BuildGenerationSources([]string{"agent"}, "support-agent", "", "", nil) + require.Len(t, withoutVersion, 1) + assert.Empty(t, withoutVersion[0].AgentVersion) +} + +// The retry that saves the documented flow: agent-seeded generation fails +// server-side for every agent, and the same request without the agent source +// succeeds. +func TestWithoutAgentSource(t *testing.T) { + sources := []GenerationSource{ + {Type: "prompt", Prompt: "be helpful"}, + {Type: "agent", AgentName: "support"}, + {Type: "traces", AgentName: "support"}, + } + + kept := WithoutAgentSource(sources) + + assert.Equal(t, []string{"prompt", "traces"}, kindsOf(kept)) + assert.Len(t, sources, 3, "the original must not be modified; it is retried from") +} + +// The retry only happens when something is left to generate from, so this is +// what stops a second billed job that would fail the same way. +func TestHasPromptSource(t *testing.T) { + assert.True(t, HasPromptSource([]GenerationSource{{Type: "prompt", Prompt: "x"}})) + assert.False(t, HasPromptSource([]GenerationSource{{Type: "prompt"}}), + "an empty prompt is nothing to generate from") + assert.False(t, HasPromptSource([]GenerationSource{{Type: "agent", AgentName: "s"}})) + assert.False(t, HasPromptSource(nil)) +} + +// The request body is what the service validates, so the fields it keys on are +// pinned rather than left to whatever the builder happens to set. +func TestNewDataGenerationJobRequest(t *testing.T) { + sources := []GenerationSource{{Type: "prompt", Prompt: "be helpful"}} + + req := NewDataGenerationJobRequest("support-regression", "gpt-4o", 15, sources) + + require.NotNil(t, req) + assert.Equal(t, "support-regression", req.Inputs.Name) + assert.Equal(t, "evaluation", req.Inputs.Scenario) + assert.Equal(t, "simple_qna", req.Inputs.Options.Type) + assert.Equal(t, 15, req.Inputs.Options.MaxSamples) + assert.Equal(t, "gpt-4o", req.Inputs.Options.ModelOptions.Model) + assert.Equal(t, sources, req.Inputs.Sources) +} diff --git a/cli/azd/extensions/azure.ai.dataset/internal/pkg/gen_api/models.go b/cli/azd/extensions/azure.ai.dataset/internal/pkg/gen_api/models.go new file mode 100644 index 00000000000..de6ca795b21 --- /dev/null +++ b/cli/azd/extensions/azure.ai.dataset/internal/pkg/gen_api/models.go @@ -0,0 +1,144 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package gen_api + +import ( + "encoding/json" + "strings" +) + +// This is the data-generation half of the evaluation service's API. The +// evaluator half stays with `azure.ai.evaluations`, because only that extension +// generates evaluators. azd extensions share no code, so the shapes both need +// are spelled out in each rather than imported. + +// DataGenerationJobRequest is the request body for CreateDataGenerationJob. +type DataGenerationJobRequest struct { + Inputs DataGenerationInputs `json:"inputs"` +} + +// DataGenerationInputs holds the inputs for a data generation job. +type DataGenerationInputs struct { + Name string `json:"name"` + Scenario string `json:"scenario"` + Options DataGenerationOptions `json:"options"` + Sources []GenerationSource `json:"sources"` +} + +// DataGenerationOptions holds configuration for data generation. +type DataGenerationOptions struct { + Type string `json:"type"` + MaxSamples int `json:"max_samples"` + ModelOptions ModelOptions `json:"model_options"` +} + +// ModelOptions holds the model selection for generation. +type ModelOptions struct { + Model string `json:"model"` +} + +// GenerationSource describes a source used for dataset generation. +type GenerationSource struct { + Type string `json:"type"` + Prompt string `json:"prompt,omitempty"` + AgentName string `json:"agent_name,omitempty"` + AgentVersion string `json:"agent_version,omitempty"` + StartTime int64 `json:"start_time,omitempty"` +} + +// Agent is the part of a catalog agent that describes what it does. +// +// An agent is returned with its versions inlined rather than as a list, and +// only `latest` is populated on a plain read. +type Agent struct { + Name string `json:"name"` + Versions struct { + Latest *AgentVersion `json:"latest"` + } `json:"versions"` +} + +// AgentVersion is one published revision of an agent. +type AgentVersion struct { + Version string `json:"version"` + Definition struct { + Model string `json:"model"` + Instructions string `json:"instructions"` + } `json:"definition"` +} + +// Instructions returns the newest version's system prompt, or "" when the agent +// has no published version. +func (a *Agent) Instructions() string { + if a == nil || a.Versions.Latest == nil { + return "" + } + return strings.TrimSpace(a.Versions.Latest.Definition.Instructions) +} + +// GenerationJob is the response for data generation job operations. +type GenerationJob struct { + ID string `json:"id"` + Status string `json:"status"` + Result json.RawMessage `json:"result,omitempty"` + Error *JobError `json:"error,omitempty"` +} + +// JobError captures error details from a failed generation job. +type JobError struct { + Code string `json:"code,omitempty"` + Message string `json:"message,omitempty"` +} + +// GenerationJobList is the listing envelope the job routes answer with. It is +// `data`, not the `value` the dataset routes use. +type GenerationJobList struct { + Data []GenerationJob `json:"data"` +} + +// ResolvedNameVersion extracts the name and version from the generation job +// result. An empty name means there is no result to read; an empty version +// means the service left it to be resolved as `latest`. +func (j *GenerationJob) ResolvedNameVersion() (string, string) { + name := j.resultStringField("name") + if name == "" { + return "", "" + } + version := j.resultStringField("version") + if version == "" { + version = "latest" + } + return name, version +} + +// resultStringField reads a string field out of the raw Result JSON, trying a +// top-level key before the nested outputs[0] shape the service also returns. +func (j *GenerationJob) resultStringField(key string) string { + if len(j.Result) == 0 { + return "" + } + var m map[string]json.RawMessage + if err := json.Unmarshal(j.Result, &m); err != nil { + return "" + } + + if raw, ok := m[key]; ok { + var s string + if err := json.Unmarshal(raw, &s); err == nil && s != "" { + return s + } + } + + if rawOutputs, ok := m["outputs"]; ok { + var outputs []map[string]json.RawMessage + if err := json.Unmarshal(rawOutputs, &outputs); err == nil && len(outputs) > 0 { + if raw, ok := outputs[0][key]; ok { + var s string + if err := json.Unmarshal(raw, &s); err == nil { + return s + } + } + } + } + return "" +} diff --git a/cli/azd/extensions/azure.ai.dataset/internal/pkg/gen_api/operations.go b/cli/azd/extensions/azure.ai.dataset/internal/pkg/gen_api/operations.go new file mode 100644 index 00000000000..5071b2e2e61 --- /dev/null +++ b/cli/azd/extensions/azure.ai.dataset/internal/pkg/gen_api/operations.go @@ -0,0 +1,224 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package gen_api + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "log" + "net/http" + "net/url" + + "azureaidataset/internal/version" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/streaming" + "github.com/azure/azure-dev/cli/azd/pkg/azsdk" +) + +const ( + pathDataGenerationJobs = "/data_generation_jobs" + pathAgents = "/agents" +) + +// Client talks to the evaluation service's data-generation routes. +// +// Datasets are registered through the dataset API, but they are *generated* by +// the evaluation service, so this extension speaks to both. +type Client struct { + endpoint string + pipeline runtime.Pipeline +} + +// NewClient creates a Client for the given project endpoint. +func NewClient(endpoint string, cred azcore.TokenCredential) *Client { + userAgent := fmt.Sprintf("azd-ext-azure-ai-dataset/%s", version.Version) + + clientOptions := &policy.ClientOptions{ + Logging: policy.LogOptions{ + AllowedHeaders: []string{"X-Ms-Correlation-Request-Id", "X-Request-Id"}, + IncludeBody: false, + }, + PerCallPolicies: []policy.Policy{ + runtime.NewBearerTokenPolicy(cred, []string{"https://ai.azure.com/.default"}, nil), + azsdk.NewMsCorrelationPolicy(), + azsdk.NewUserAgentPolicy(userAgent), + }, + } + + pipeline := runtime.NewPipeline( + "azure-ai-dataset", + "v1.0.0", + runtime.PipelineOptions{}, + clientOptions, + ) + + return &Client{endpoint: endpoint, pipeline: pipeline} +} + +// NewClientFromPipeline creates a Client with a pre-built pipeline, for tests +// that need to bypass auth policies. +func NewClientFromPipeline(endpoint string, pipeline runtime.Pipeline) *Client { + return &Client{endpoint: endpoint, pipeline: pipeline} +} + +// CreateDataGenerationJob starts a dataset generation job. +func (c *Client) CreateDataGenerationJob( + ctx context.Context, + request *DataGenerationJobRequest, + apiVersion string, +) (*GenerationJob, error) { + return doRequestTyped[GenerationJob]( + c, ctx, http.MethodPost, pathDataGenerationJobs, request, apiVersion) +} + +// GetDataGenerationJob gets the current state of a dataset generation job. +func (c *Client) GetDataGenerationJob( + ctx context.Context, + operationID string, + apiVersion string, +) (*GenerationJob, error) { + path := pathDataGenerationJobs + "/" + url.PathEscape(operationID) + return doRequestTyped[GenerationJob](c, ctx, http.MethodGet, path, nil, apiVersion) +} + +// ListDataGenerationJobs returns the project's dataset generation jobs. +func (c *Client) ListDataGenerationJobs( + ctx context.Context, + apiVersion string, +) (*GenerationJobList, error) { + return doRequestTyped[GenerationJobList]( + c, ctx, http.MethodGet, pathDataGenerationJobs, nil, apiVersion) +} + +// CancelDataGenerationJob stops a dataset generation job. +// +// The separator is a colon, not a path segment: `{id}/cancel` is a 404 while +// `{id}:cancel` reaches the action. The empty object is what carries a content +// type, without which the route answers 415. +func (c *Client) CancelDataGenerationJob( + ctx context.Context, + operationID string, + apiVersion string, +) (*GenerationJob, error) { + path := pathDataGenerationJobs + "/" + url.PathEscape(operationID) + ":cancel" + return doRequestTyped[GenerationJob]( + c, ctx, http.MethodPost, path, json.RawMessage(`{}`), apiVersion) +} + +// DeleteDataGenerationJob discards the job record. The dataset the job produced +// is already registered and is not affected. +func (c *Client) DeleteDataGenerationJob( + ctx context.Context, + operationID string, + apiVersion string, +) error { + path := pathDataGenerationJobs + "/" + url.PathEscape(operationID) + _, err := c.doRequest(ctx, http.MethodDelete, path, nil, apiVersion) + return err +} + +// GetAgent reads an agent from the project's catalog. +// +// Only the newest version is returned, which is the one generation is seeded +// from: the point is to describe what the agent does now. +func (c *Client) GetAgent( + ctx context.Context, + name string, + apiVersion string, +) (*Agent, error) { + path := pathAgents + "/" + url.PathEscape(name) + return doRequestTyped[Agent](c, ctx, http.MethodGet, path, nil, apiVersion) +} + +func (c *Client) doRequest( + ctx context.Context, + method string, + path string, + body any, + apiVersion string, +) ([]byte, error) { + u, err := url.Parse(c.endpoint) + if err != nil { + return nil, fmt.Errorf("invalid endpoint URL: %w", err) + } + + u.Path += path + q := u.Query() + if apiVersion != "" { + q.Set("api-version", apiVersion) + } + u.RawQuery = q.Encode() + + req, err := runtime.NewRequest(ctx, method, u.String()) + if err != nil { + return nil, fmt.Errorf("failed to create request: %w", err) + } + + log.Printf("[gen_api] %s %s", method, u.Redacted()) + + if body != nil { + payload, err := json.Marshal(body) + if err != nil { + return nil, fmt.Errorf("failed to marshal request: %w", err) + } + if err := req.SetBody(streaming.NopCloser(bytes.NewReader(payload)), "application/json"); err != nil { + return nil, fmt.Errorf("failed to set request body: %w", err) + } + } + + resp, err := c.pipeline.Do(req) + if err != nil { + return nil, fmt.Errorf("HTTP request failed: %w", err) + } + defer resp.Body.Close() + + respBody, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("failed to read response body: %w", err) + } + + log.Printf("[gen_api] response status: %d", resp.StatusCode) + + // 204 belongs here: a delete that removed the resource answers No Content, + // and treating that as a failure reports every successful delete as an + // error. + if !runtime.HasStatusCode(resp, + http.StatusOK, http.StatusCreated, http.StatusAccepted, http.StatusNoContent) { + resp.Body = io.NopCloser(bytes.NewReader(respBody)) + return nil, runtime.NewResponseError(resp) + } + + return respBody, nil +} + +func doRequestTyped[T any]( + c *Client, + ctx context.Context, + method string, + path string, + body any, + apiVersion string, +) (*T, error) { + respBody, err := c.doRequest(ctx, method, path, body, apiVersion) + if err != nil { + return nil, err + } + + if len(respBody) == 0 { + return new(T), nil + } + + var result T + if err := json.Unmarshal(respBody, &result); err != nil { + return nil, fmt.Errorf("failed to parse response: %w", err) + } + + return &result, nil +} diff --git a/cli/azd/extensions/azure.ai.dataset/internal/pkg/gen_api/poller.go b/cli/azd/extensions/azure.ai.dataset/internal/pkg/gen_api/poller.go new file mode 100644 index 00000000000..825b980ec2e --- /dev/null +++ b/cli/azd/extensions/azure.ai.dataset/internal/pkg/gen_api/poller.go @@ -0,0 +1,193 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package gen_api + +import ( + "context" + "fmt" + "log" + "strings" + "time" +) + +// --------------------------------------------------------------------------- +// JobStatus — typed status with terminal/failed semantics +// --------------------------------------------------------------------------- + +// JobStatus represents the normalized status of a generation job. +type JobStatus string + +const ( + JobStatusRunning JobStatus = "running" + JobStatusCompleted JobStatus = "completed" + JobStatusSucceeded JobStatus = "succeeded" + JobStatusFailed JobStatus = "failed" + JobStatusCancelled JobStatus = "cancelled" + JobStatusCanceled JobStatus = "canceled" +) + +// ParseJobStatus normalizes a raw status string into a JobStatus. +// An empty string is treated as "running". +func ParseJobStatus(s string) JobStatus { + if s == "" { + return JobStatusRunning + } + return JobStatus(strings.ToLower(s)) +} + +// IsTerminal returns true when the status represents a final state. +func (s JobStatus) IsTerminal() bool { + switch s { + case JobStatusCompleted, JobStatusSucceeded, JobStatusFailed, JobStatusCancelled, JobStatusCanceled: + return true + } + return false +} + +// IsFailed returns true when the status represents a failure or cancellation. +func (s JobStatus) IsFailed() bool { + switch s { + case JobStatusFailed, JobStatusCancelled, JobStatusCanceled: + return true + } + return false +} + +// String returns the status as a plain string. +func (s JobStatus) String() string { + return string(s) +} + +// --------------------------------------------------------------------------- +// JobFailedError — returned when a polled job reaches a failed state +// --------------------------------------------------------------------------- + +// JobFailedError is returned when a generation job reaches a failed terminal state. +type JobFailedError struct { + Job *GenerationJob + Status JobStatus +} + +func (e *JobFailedError) Error() string { + if e.Job != nil && e.Job.Error != nil && e.Job.Error.Message != "" { + return fmt.Sprintf("job failed with status %q: %s", e.Status, e.Job.Error.Message) + } + return fmt.Sprintf("job failed with status %q", e.Status) +} + +// --------------------------------------------------------------------------- +// PollerTimeoutError — returned when polling exhausts all attempts +// --------------------------------------------------------------------------- + +// PollerTimeoutError is returned when a generation job has not reached a +// terminal state within the configured number of polling attempts. +type PollerTimeoutError struct { + OperationID string + Attempts int +} + +func (e *PollerTimeoutError) Error() string { + return fmt.Sprintf( + "operation %s did not complete within %d attempts", + e.OperationID, e.Attempts, + ) +} + +// --------------------------------------------------------------------------- +// GetJobFunc — callback type for fetching job state +// --------------------------------------------------------------------------- + +// GetJobFunc fetches the current state of a generation job by operation ID. +type GetJobFunc func(ctx context.Context, operationID, apiVersion string) (*GenerationJob, error) + +// --------------------------------------------------------------------------- +// PollerOptions — configurable polling behavior +// --------------------------------------------------------------------------- + +// PollerOptions configures the polling interval and attempt limit. +type PollerOptions struct { + Interval time.Duration + MaxAttempts int +} + +// DefaultPollerOptions returns sensible defaults: 2 s interval, 300 attempts (~10 min). +func DefaultPollerOptions() PollerOptions { + return PollerOptions{ + Interval: 2 * time.Second, + MaxAttempts: 300, + } +} + +// --------------------------------------------------------------------------- +// Poller — polls a generation job until it reaches a terminal state +// --------------------------------------------------------------------------- + +// Poller polls a GenerationJob until it reaches a terminal status. +type Poller struct { + OperationID string + APIVersion string + GetJob GetJobFunc + Options PollerOptions + // OnPoll is called after each successful poll with the latest status. + // Callers can use this for progress reporting (e.g. debug logging). + OnPoll func(status JobStatus) +} + +// NewPoller creates a Poller with default options. +func NewPoller(operationID, apiVersion string, getJob GetJobFunc) *Poller { + return &Poller{ + OperationID: operationID, + APIVersion: apiVersion, + GetJob: getJob, + Options: DefaultPollerOptions(), + } +} + +// Poll blocks until the job reaches a terminal state, the context is +// cancelled, or the maximum number of attempts is exhausted. +// +// On success it returns the completed GenerationJob. +// On failure it returns a *JobFailedError (which wraps the job for inspection). +// On timeout it returns a plain error. +func (p *Poller) Poll(ctx context.Context) (*GenerationJob, error) { + if p.OperationID == "" { + return nil, fmt.Errorf("operation ID is empty") + } + + for range p.Options.MaxAttempts { + select { + case <-ctx.Done(): + return nil, ctx.Err() + case <-time.After(p.Options.Interval): + } + + job, err := p.GetJob(ctx, p.OperationID, p.APIVersion) + if err != nil { + if IsTransientError(err) { + log.Printf("[poller] transient error polling %s, will retry: %v", p.OperationID, err) + continue + } + return nil, err + } + + status := ParseJobStatus(job.Status) + log.Printf("[poller] operationID=%s status=%s", p.OperationID, status) + + if p.OnPoll != nil { + p.OnPoll(status) + } + + if status.IsTerminal() { + if status.IsFailed() { + return nil, &JobFailedError{Job: job, Status: status} + } + return job, nil + } + } + + return nil, &PollerTimeoutError{ + OperationID: p.OperationID, + Attempts: p.Options.MaxAttempts, + } +} diff --git a/cli/azd/extensions/azure.ai.dataset/internal/version/version.go b/cli/azd/extensions/azure.ai.dataset/internal/version/version.go new file mode 100644 index 00000000000..e7279d11fba --- /dev/null +++ b/cli/azd/extensions/azure.ai.dataset/internal/version/version.go @@ -0,0 +1,11 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package version + +var ( + // Populated at build time. + Version = "dev" + Commit = "none" + BuildDate = "unknown" +) diff --git a/cli/azd/extensions/azure.ai.dataset/main.go b/cli/azd/extensions/azure.ai.dataset/main.go new file mode 100644 index 00000000000..9f66d1148be --- /dev/null +++ b/cli/azd/extensions/azure.ai.dataset/main.go @@ -0,0 +1,14 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package main + +import ( + "azureaidataset/internal/cmd" + + "github.com/azure/azure-dev/cli/azd/pkg/azdext" +) + +func main() { + azdext.Run(cmd.NewRootCommand()) +} diff --git a/cli/azd/extensions/azure.ai.dataset/version.txt b/cli/azd/extensions/azure.ai.dataset/version.txt new file mode 100644 index 00000000000..e864273872d --- /dev/null +++ b/cli/azd/extensions/azure.ai.dataset/version.txt @@ -0,0 +1 @@ +1.0.0-beta.1 \ No newline at end of file diff --git a/eng/pipelines/release-ext-azure-ai-dataset.yml b/eng/pipelines/release-ext-azure-ai-dataset.yml new file mode 100644 index 00000000000..dbcebc827c1 --- /dev/null +++ b/eng/pipelines/release-ext-azure-ai-dataset.yml @@ -0,0 +1,45 @@ +# Continuous deployment trigger +trigger: + branches: + include: + - main + paths: + include: + - cli/azd/extensions/azure.ai.dataset + - /eng/pipelines/templates/stages/release-azd-extension.yml + - /eng/pipelines/templates/jobs/build-azd-extension.yml + - /eng/pipelines/templates/jobs/cross-build-azd-extension.yml + - /eng/pipelines/templates/variables/image.yml + +pr: + paths: + include: + - cli/azd/extensions/azure.ai.dataset + - eng/pipelines/release-ext-azure-ai-dataset.yml + - /eng/pipelines/templates/stages/release-azd-extension.yml + - eng/pipelines/templates/steps/publish-cli.yml + exclude: + - cli/azd/docs/** + +parameters: + - name: PublishToRegistry + displayName: Publish to registry + type: string + # Scheduled (nightly) runs override this in the shared templates; the runtime + # parameter default must be a literal because it renders before variables exist. + default: stable + values: + - stable + - dev + - nightly + +extends: + template: /eng/pipelines/templates/stages/1es-redirect.yml + parameters: + stages: + - template: /eng/pipelines/templates/stages/release-azd-extension.yml + parameters: + AzdExtensionId: azure.ai.dataset + SanitizedExtensionId: azure-ai-dataset + AzdExtensionDirectory: cli/azd/extensions/azure.ai.dataset + PublishToRegistry: ${{ parameters.PublishToRegistry }} From f1efd4c43f30c4b43e8af62a684c4956f1492a35 Mon Sep 17 00:00:00 2001 From: mohessie Date: Thu, 6 Aug 2026 04:34:24 +0300 Subject: [PATCH 116/320] Tidy the dataset module's dependencies The manifest was copied from azure.ai.evaluations, so it required what that extension needs rather than what this one does. yaml and protobuf are gone entirely -- there is no eval configuration to parse and no service-target provider -- and uuid and protobuf drop to indirect. --- cli/azd/extensions/azure.ai.dataset/go.mod | 7 +++---- cli/azd/extensions/azure.ai.dataset/go.sum | 2 -- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/cli/azd/extensions/azure.ai.dataset/go.mod b/cli/azd/extensions/azure.ai.dataset/go.mod index d995ace863a..32ac3b07c56 100644 --- a/cli/azd/extensions/azure.ai.dataset/go.mod +++ b/cli/azd/extensions/azure.ai.dataset/go.mod @@ -7,11 +7,9 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.14.0-beta.3 github.com/azure/azure-dev/cli/azd v1.28.0 github.com/fatih/color v1.18.0 - github.com/google/uuid v1.6.0 github.com/spf13/cobra v1.10.1 + github.com/spf13/pflag v1.0.10 github.com/stretchr/testify v1.11.1 - go.yaml.in/yaml/v3 v3.0.4 - google.golang.org/protobuf v1.36.11 ) require ( @@ -52,6 +50,7 @@ require ( github.com/gofrs/flock v0.12.1 // indirect github.com/golang-jwt/jwt/v5 v5.3.0 // indirect github.com/golobby/container/v3 v3.3.2 // indirect + github.com/google/uuid v1.6.0 // indirect github.com/gorilla/css v1.0.1 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/invopop/jsonschema v0.13.0 // indirect @@ -78,7 +77,6 @@ require ( github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 // indirect github.com/sethvargo/go-retry v0.3.0 // indirect github.com/spf13/cast v1.10.0 // indirect - github.com/spf13/pflag v1.0.10 // indirect github.com/theckman/yacspin v0.13.12 // indirect github.com/wk8/go-ordered-map/v2 v2.1.8 // indirect github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect @@ -102,5 +100,6 @@ require ( golang.org/x/time v0.9.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20260401024825-9d38bb4040a9 // indirect google.golang.org/grpc v1.80.0 // indirect + google.golang.org/protobuf v1.36.11 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/cli/azd/extensions/azure.ai.dataset/go.sum b/cli/azd/extensions/azure.ai.dataset/go.sum index 81c10e45793..1753277fdf9 100644 --- a/cli/azd/extensions/azure.ai.dataset/go.sum +++ b/cli/azd/extensions/azure.ai.dataset/go.sum @@ -249,8 +249,6 @@ go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= -go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= -go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.53.0 h1:QZ4Muo8THX6CizN2vPPd5fBGHyogrdK9fG4wLPFUsto= From 9c4dad5a3ccf1509b347dee77cb8f67a1520bacc Mon Sep 17 00:00:00 2001 From: mohessie Date: Thu, 6 Aug 2026 04:52:34 +0300 Subject: [PATCH 117/320] Cover the new extension, and drop what the move left behind The extension shipped with a surface test and the tests that came across with the packages, which left the code written for the move itself at zero: the validators, the artifact path, the --from default, the unbuildable-source refusal, and the two failure explanations. Those are all decisions a user meets as an error message, so they are the ones worth pinning. newDatasetCommand was dead on arrival. It grouped the subcommands under a `dataset` parent, which is what the eval extension needs and this one does not -- here `dataset` is the namespace, so root.go registers the subcommands directly. Nothing referenced it. gen_api gets its response parsing covered. ResolvedNameVersion reads the generated dataset's name and version out of the job result, and the service answers in two shapes; reading only one leaves the download looking for a dataset the job never named. Agent.Instructions trims, because surrounding whitespace would otherwise travel into the prompt. ParseJobStatus lowercases, without which a status the service capitalizes reads as still-running and the poller waits out its budget on a finished job. And the suggestion guard, which is here because its absence cost something in the extension these commands came from: three printed instructions pointed at a namespace nobody served, and the check there matched on the wrong prefix so none of them failed. Renaming `job` to `jobs` makes this one fail by name. Coverage 28.3% -> 38.6%; gen_api 21.9% -> 52.8%. --- .../azure.ai.dataset/internal/cmd/dataset.go | 18 -- .../internal/cmd/helpers_test.go | 177 +++++++++++++++++ .../internal/cmd/surface_test.go | 66 +++++++ .../internal/pkg/gen_api/models_test.go | 179 ++++++++++++++++++ 4 files changed, 422 insertions(+), 18 deletions(-) create mode 100644 cli/azd/extensions/azure.ai.dataset/internal/cmd/helpers_test.go create mode 100644 cli/azd/extensions/azure.ai.dataset/internal/pkg/gen_api/models_test.go diff --git a/cli/azd/extensions/azure.ai.dataset/internal/cmd/dataset.go b/cli/azd/extensions/azure.ai.dataset/internal/cmd/dataset.go index 97d2cb8ff9f..ad0b323e56f 100644 --- a/cli/azd/extensions/azure.ai.dataset/internal/cmd/dataset.go +++ b/cli/azd/extensions/azure.ai.dataset/internal/cmd/dataset.go @@ -20,24 +20,6 @@ import ( // publish, and so the one that exists for every dataset that exists at all. const firstDatasetVersion = "1" -func newDatasetCommand() *cobra.Command { - cmd := &cobra.Command{ - Use: "dataset", - Short: "Manage Foundry datasets.", - } - cmd.AddCommand( - newDatasetCreateCommand(), - newDatasetUpdateCommand(), - newDatasetGenerateCommand(), - newDatasetListCommand(), - newDatasetShowCommand(), - newDatasetDeleteCommand(), - newDatasetVersionsCommand(), - newJobCommand(), - ) - return cmd -} - // newDatasetCreateCommand builds `dataset create `, which registers a // dataset that does not exist yet. func newDatasetCreateCommand() *cobra.Command { diff --git a/cli/azd/extensions/azure.ai.dataset/internal/cmd/helpers_test.go b/cli/azd/extensions/azure.ai.dataset/internal/cmd/helpers_test.go new file mode 100644 index 00000000000..ae4a253b342 --- /dev/null +++ b/cli/azd/extensions/azure.ai.dataset/internal/cmd/helpers_test.go @@ -0,0 +1,177 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "errors" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// A sample count the service would reject is refused here, before a job is +// submitted and billed. +func TestValidateSampleSize(t *testing.T) { + assert.NoError(t, validateSampleSize(0), "unset means take the default") + assert.NoError(t, validateSampleSize(minSampleSize)) + assert.NoError(t, validateSampleSize(maxSampleSize)) + assert.NoError(t, validateSampleSize(100)) + + for _, n := range []int{1, minSampleSize - 1, maxSampleSize + 1, -5} { + err := validateSampleSize(n) + require.Errorf(t, err, "%d is out of range", n) + assert.Contains(t, err.Error(), "15") + assert.Contains(t, err.Error(), "1000", "the message has to name the range it enforces") + } +} + +// --from names a source the service has a path for. A typo caught here costs +// nothing; the same typo reaching the service costs a job. +func TestValidateGenerateSource(t *testing.T) { + assert.NoError(t, validateGenerateSource(""), "unset means take the default") + for _, s := range generateSources { + assert.NoErrorf(t, validateGenerateSource(s), "%q is a documented source", s) + } + + err := validateGenerateSource("tracez") + require.Error(t, err) + assert.Contains(t, err.Error(), "tracez") + for _, s := range generateSources { + assert.Containsf(t, err.Error(), s, "the refusal has to list %q as an option", s) + } +} + +// The generated file lands where `azd ai eval init` scaffolds, so a generated +// dataset is already where an evaluation configuration expects it. +func TestArtifactPath(t *testing.T) { + assert.Equal(t, + filepath.Join("evals", "datasets", "support-regression.jsonl"), + artifactPath(defaultOutputDir, "support-regression")) + assert.Equal(t, + filepath.Join("out", "x.jsonl"), + artifactPath("out", "x")) +} + +// The spec's default: traces when the project has Application Insights +// connected, otherwise the agent. The connection string is how a project says +// it collects traces at all, so asking for traces without one would submit a +// billed job against nothing. +func TestDefaultGenerationSource(t *testing.T) { + assert.Equal(t, []string{generateFromTraces}, + defaultGenerationSource("InstrumentationKey=00000000-0000-0000-0000-000000000000")) + assert.Equal(t, []string{generateFromAgent}, defaultGenerationSource("")) +} + +// The one difference between create and update, and the only thing stopping a +// create from silently publishing version 2 of someone else's dataset. +func TestCheckAssetExistence(t *testing.T) { + assert.NoError(t, checkAssetExistence("create", "dataset", "x", false)) + assert.NoError(t, checkAssetExistence("update", "dataset", "x", true)) + + err := checkAssetExistence("create", "dataset", "x", true) + require.Error(t, err) + assert.Contains(t, err.Error(), "update", "the error has to name the verb that works") + + err = checkAssetExistence("update", "dataset", "x", false) + require.Error(t, err) + assert.Contains(t, err.Error(), "create") +} + +// Generation is billed and the file is checked in, so overwriting one is a +// decision the caller makes rather than a side effect. +func TestRefuseExistingArtifact(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "support.jsonl") + + assert.NoError(t, refuseExistingArtifact(path, false), "nothing there yet") + + require.NoError(t, os.WriteFile(path, []byte("{}"), 0o600)) + + err := refuseExistingArtifact(path, false) + require.Error(t, err) + assert.Contains(t, err.Error(), "--force") + assert.Contains(t, err.Error(), "--output-dir", "both ways out have to be named") + + assert.NoError(t, refuseExistingArtifact(path, true), "--force is the way through") +} + +// --from is a request, and one the plan cannot honour has to stop the command +// rather than quietly submit a job seeded from less than was asked for. +func TestRefuseUnbuildableSources(t *testing.T) { + assert.NoError(t, refuseUnbuildableSources(nil)) + assert.NoError(t, refuseUnbuildableSources([]string{})) + + tests := []struct{ kind, says string }{ + {generateFromPrompt, "--agent-instruction"}, + {generateFromAgent, "--target"}, + {generateFromFile, "azd ai dataset create"}, + } + for _, tt := range tests { + t.Run(tt.kind, func(t *testing.T) { + err := refuseUnbuildableSources([]string{tt.kind}) + require.Error(t, err) + assert.Contains(t, err.Error(), tt.says, + "the error has to name the way out, not just the problem") + }) + } + + both := refuseUnbuildableSources([]string{generateFromPrompt, generateFromAgent}) + require.Error(t, both) + assert.Contains(t, both.Error(), "--agent-instruction") + assert.Contains(t, both.Error(), "--target", + "two unhonoured sources are two things to fix, so both are reported at once") +} + +// The service's system error says only that something went wrong and to try +// again, which sends users into a retry loop against a deterministic failure. +func TestExplainDataGenerationFailure(t *testing.T) { + systemErr := errors.New("DataGenerationJobSystemError: Something went wrong during data generation") + + explained := explainDataGenerationFailure(systemErr, "support-agent") + require.Error(t, explained) + assert.Contains(t, explained.Error(), "support-agent") + assert.Contains(t, explained.Error(), "--agent-instruction", + "the explanation has to name the way around it") + assert.ErrorIs(t, explained, systemErr, "the original must still be reachable") + + assert.NoError(t, explainDataGenerationFailure(nil, "support-agent")) + assert.Equal(t, systemErr, explainDataGenerationFailure(systemErr, ""), + "with no agent named there is nothing to explain") + + other := errors.New("connection reset") + assert.Equal(t, other, explainDataGenerationFailure(other, "support-agent"), + "an unrelated failure must not be blamed on the agent") +} + +func TestIsAgentSeededGenerationFailure(t *testing.T) { + assert.True(t, isAgentSeededGenerationFailure(errors.New("DataGenerationJobSystemError"))) + assert.True(t, isAgentSeededGenerationFailure( + errors.New("Something went wrong during data generation"))) + assert.False(t, isAgentSeededGenerationFailure(errors.New("429 Too Many Requests"))) + assert.False(t, isAgentSeededGenerationFailure(nil)) +} + +// The two job types share an id shape, so reaching for the wrong group is the +// likely mistake and the error has to say where the other one is. +func TestJobLookupErrorPointsAtTheEvaluatorGroup(t *testing.T) { + err := jobLookupError("dgj_01", errors.New("boom")) + require.Error(t, err) + assert.Contains(t, err.Error(), "dgj_01") + + // A non-404 is reported as itself rather than as a wrong-group guess. + assert.NotContains(t, err.Error(), "azd ai eval evaluator job") +} + +// A window narrows a traces request; its absence is not a request for none. +func TestGenerationPlanTraceOptions(t *testing.T) { + assert.Nil(t, generationPlan{}.traceOptions()) + assert.Nil(t, generationPlan{TraceDays: -1}.traceOptions()) + + opts := generationPlan{TraceDays: 7}.traceOptions() + require.NotNil(t, opts) + assert.Equal(t, 7, opts.Days) +} diff --git a/cli/azd/extensions/azure.ai.dataset/internal/cmd/surface_test.go b/cli/azd/extensions/azure.ai.dataset/internal/cmd/surface_test.go index 200ef14b5ec..9f1a9161fd5 100644 --- a/cli/azd/extensions/azure.ai.dataset/internal/cmd/surface_test.go +++ b/cli/azd/extensions/azure.ai.dataset/internal/cmd/surface_test.go @@ -4,6 +4,10 @@ package cmd import ( + "io/fs" + "os" + "path/filepath" + "regexp" "strings" "testing" @@ -120,6 +124,68 @@ func TestGenerateFromTakesMoreThanOneSource(t *testing.T) { } } +// Messages that tell a user what to run next have to name a command that +// exists. +// +// In the extension these commands moved from, three suggestions pointed at +// `azd ai dataset ...` while that namespace was served by nobody, and the +// check there matched on the wrong prefix so none of them failed. This +// extension's namespace is `ai.dataset`; anything it suggests under +// `azd ai dataset` has to resolve here, and a suggestion under another +// namespace is one it cannot make. +func TestSuggestedCommandsExist(t *testing.T) { + pattern := regexp.MustCompile("azd ai ([a-z][a-z0-9-]*(?: [a-z][a-z0-9-]*)*)") + + err := filepath.WalkDir("../..", func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() || !strings.HasSuffix(path, ".go") || strings.HasSuffix(path, "_test.go") { + return nil + } + body, err := os.ReadFile(path) + if err != nil { + return err + } + for _, line := range strings.Split(string(body), "\n") { + if strings.HasPrefix(strings.TrimSpace(line), "//") { + continue + } + for _, m := range pattern.FindAllStringSubmatch(line, -1) { + words := strings.Fields(m[1]) + if len(words) == 0 { + continue + } + // A suggestion pointing at a sibling extension is that + // extension's contract, not this one's, and cannot be resolved + // from here. `eval` is the only one this extension names. + if words[0] == "eval" { + continue + } + assert.Equalf(t, "dataset", words[0], + "%s suggests `azd ai %s`, which is neither this extension's "+ + "namespace nor a sibling it knows about", path, m[1]) + words = words[1:] + + // Trim trailing prose: "job show" is a command, "job show and + // then" is a sentence that begins with one. + for len(words) > 0 { + if resolved, _, e := NewRootCommand().Find(words); e == nil { + if strings.Fields(resolved.Use)[0] == words[len(words)-1] { + break + } + } + words = words[:len(words)-1] + } + assert.NotEmptyf(t, words, + "%s suggests `azd ai %s`, which is not a command", path, m[1]) + } + } + return nil + }) + require.NoError(t, err) +} + func find(t *testing.T, path string) *cobra.Command { t.Helper() cmd, _, err := NewRootCommand().Find(strings.Fields(path)) diff --git a/cli/azd/extensions/azure.ai.dataset/internal/pkg/gen_api/models_test.go b/cli/azd/extensions/azure.ai.dataset/internal/pkg/gen_api/models_test.go new file mode 100644 index 00000000000..ff3b17aa425 --- /dev/null +++ b/cli/azd/extensions/azure.ai.dataset/internal/pkg/gen_api/models_test.go @@ -0,0 +1,179 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package gen_api + +import ( + "encoding/json" + "errors" + "net/http" + "testing" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// The generated dataset's name and version come back inside the job result, +// and the service answers in two shapes. Reading only one leaves the download +// looking for a dataset the job never named. +func TestGenerationJob_ResolvedNameVersion(t *testing.T) { + tests := []struct { + name string + result string + wantName string + wantVersion string + }{ + { + name: "top-level fields", + result: `{"name":"support-regression","version":"3"}`, + wantName: "support-regression", + wantVersion: "3", + }, + { + name: "nested under outputs", + result: `{"outputs":[{"name":"support-regression","version":"2"}]}`, + wantName: "support-regression", + wantVersion: "2", + }, + { + name: "a name with no version resolves as latest", + result: `{"name":"support-regression"}`, + wantName: "support-regression", + wantVersion: "latest", + }, + { + name: "top-level wins over outputs", + result: `{"name":"top","outputs":[{"name":"nested"}]}`, + wantName: "top", + wantVersion: "latest", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + job := &GenerationJob{Result: json.RawMessage(tt.result)} + + gotName, gotVersion := job.ResolvedNameVersion() + + assert.Equal(t, tt.wantName, gotName) + assert.Equal(t, tt.wantVersion, gotVersion) + }) + } +} + +// No name means no result to read, and both halves come back empty so a caller +// checking either sees the same thing. +func TestGenerationJob_ResolvedNameVersion_NoResult(t *testing.T) { + for _, result := range []string{``, `{}`, `not json`, `{"version":"3"}`} { + job := &GenerationJob{Result: json.RawMessage(result)} + + name, version := job.ResolvedNameVersion() + + assert.Emptyf(t, name, "result %q", result) + assert.Emptyf(t, version, "result %q", result) + } +} + +// The agent's newest instructions are what generation is seeded from, so an +// agent with no published version has nothing to offer rather than an error. +func TestAgent_Instructions(t *testing.T) { + var agent *Agent + assert.Empty(t, agent.Instructions(), "a nil agent is not a panic") + + agent = &Agent{Name: "support"} + assert.Empty(t, agent.Instructions(), "no published version, nothing to read") + + agent.Versions.Latest = &AgentVersion{Version: "3"} + agent.Versions.Latest.Definition.Instructions = "\n Answer politely.\n" + assert.Equal(t, "Answer politely.", agent.Instructions(), + "surrounding whitespace would travel into the prompt") +} + +// A status the service spells differently must not read as still-running, or +// the poller waits out its budget on a job that finished. +func TestParseJobStatus(t *testing.T) { + assert.Equal(t, JobStatusRunning, ParseJobStatus(""), "unset means still running") + assert.Equal(t, JobStatusCompleted, ParseJobStatus("Completed")) + assert.Equal(t, JobStatusSucceeded, ParseJobStatus("SUCCEEDED")) + assert.Equal(t, JobStatusFailed, ParseJobStatus("failed")) +} + +func TestJobStatus_TerminalAndFailed(t *testing.T) { + terminal := []JobStatus{ + JobStatusCompleted, JobStatusSucceeded, + JobStatusFailed, JobStatusCancelled, JobStatusCanceled, + } + for _, s := range terminal { + assert.Truef(t, s.IsTerminal(), "%s is a final state", s) + } + assert.False(t, JobStatusRunning.IsTerminal()) + + // Both spellings of cancelled count as a failure, because the service uses + // one and the other is what half the callers will type. + for _, s := range []JobStatus{JobStatusFailed, JobStatusCancelled, JobStatusCanceled} { + assert.Truef(t, s.IsFailed(), "%s did not produce an artifact", s) + } + for _, s := range []JobStatus{JobStatusCompleted, JobStatusSucceeded, JobStatusRunning} { + assert.Falsef(t, s.IsFailed(), "%s is not a failure", s) + } +} + +// The poller reports why it gave up, and a job that failed with a service +// message has to carry that message rather than only its status. +func TestJobFailedError(t *testing.T) { + bare := &JobFailedError{Status: JobStatusFailed} + assert.Contains(t, bare.Error(), "failed") + + withMessage := &JobFailedError{ + Status: JobStatusFailed, + Job: &GenerationJob{Error: &JobError{Message: "quota exceeded"}}, + } + assert.Contains(t, withMessage.Error(), "quota exceeded", + "the service said why; repeating only the status loses it") +} + +// A transient failure is retried and a terminal one is not, so the difference +// decides whether a caller waits or is told. +func TestIsTransientError(t *testing.T) { + assert.False(t, IsTransientError(nil)) + + for _, code := range []int{http.StatusTooManyRequests, http.StatusInternalServerError, + http.StatusBadGateway, http.StatusServiceUnavailable} { + err := &azcore.ResponseError{StatusCode: code} + assert.Truef(t, IsTransientError(err), "%d is worth retrying", code) + } + for _, code := range []int{http.StatusBadRequest, http.StatusNotFound, http.StatusConflict} { + err := &azcore.ResponseError{StatusCode: code} + assert.Falsef(t, IsTransientError(err), "%d will not change on a retry", code) + } + + for _, msg := range []string{"connection reset by peer", "connection refused", "unexpected EOF"} { + assert.Truef(t, IsTransientError(errors.New(msg)), "%q is a dropped connection", msg) + } + assert.False(t, IsTransientError(errors.New("invalid dataset name"))) +} + +func TestIsNotFoundAndIsConflict(t *testing.T) { + notFound := &azcore.ResponseError{StatusCode: http.StatusNotFound} + conflict := &azcore.ResponseError{StatusCode: http.StatusConflict} + + assert.True(t, IsNotFound(notFound)) + assert.False(t, IsNotFound(conflict)) + assert.False(t, IsNotFound(errors.New("boom"))) + + assert.True(t, IsConflict(conflict)) + assert.False(t, IsConflict(notFound)) + assert.False(t, IsConflict(nil)) +} + +// Wrapped errors have to keep answering, because the client wraps everything it +// returns with context about the call. +func TestIsNotFound_ThroughAWrap(t *testing.T) { + wrapped := errors.Join( + errors.New("reading dataset \"x\""), + &azcore.ResponseError{StatusCode: http.StatusNotFound}, + ) + + require.True(t, IsNotFound(wrapped)) +} From a6850d761aceeadc2574ee5bae3f0af8b18066e4 Mon Sep 17 00:00:00 2001 From: mohessie Date: Thu, 6 Aug 2026 05:04:21 +0300 Subject: [PATCH 118/320] Build the portal links the spec describes, and stop suggesting a dead flag Auditing behaviour against the spec's output conventions found the whole portal-link feature missing. The spec gives it a table of URL shapes, says the prefix comes from AZURE_AI_PROJECT_ID, and says `-o json` carries it as portal_url. None of that existed: nothing called NewPortalPrefix, neither env key was read anywhere, and portal_url appeared in no response. portal_urls.go was dead code in production -- which I made worse last commit by testing it, entrenching a package no command used. What the commands did print is `Report:`, the service's own report_url. That is a different link and it stays; the portal line is built here and closes the detail view in cyan, as the sibling extensions do. run start, run show and dataset show carry it now, and OpenAIEvalRun gains a PortalURL the extension populates so the terminal and `-o json` answer from one place. Resolution stays best effort, matching the agents extension: a missing or unparseable AZURE_AI_PROJECT_ID drops the line rather than failing a command that already did its work. Separately, `run start --no-wait` closed with Reattach with: azd ai eval run show --eval-id and --eval-id was removed earlier in this branch. The command resolved, so the suggestion check passed; the flag did not exist, so the one line a user is told to paste was the one guaranteed to fail. TestSuggestedFlagsExist now checks the flags in a suggestion against what that command declares, and restoring --eval-id makes it fail naming the file and line. Not done here: `evaluator show` emits raw JSON with no text form, so it has nowhere to put a Portal: line without making its output unparseable. The spec says show is a detail view with `-o json` as the alternative; that is a restructure rather than an addition. --- .../internal/cmd/context.go | 41 +++++++ .../internal/cmd/dataset.go | 10 +- .../internal/cmd/output.go | 13 +++ .../internal/cmd/portal_test.go | 100 ++++++++++++++++++ .../azure.ai.evaluations/internal/cmd/run.go | 4 +- .../internal/cmd/run_ops.go | 2 + .../internal/cmd/surface_test.go | 56 ++++++++++ .../internal/pkg/eval_api/models.go | 3 + 8 files changed, 226 insertions(+), 3 deletions(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/portal_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/context.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/context.go index 1c897a0f906..65c3e87181c 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/context.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/context.go @@ -7,6 +7,7 @@ import ( "context" "errors" "fmt" + "log" "os" "strings" @@ -175,6 +176,46 @@ func (ec *evalContext) Close() { } } +// projectARMIDEnvKey holds the project's ARM resource ID, which is what the +// Foundry portal addresses a project by. azd provisioning writes it, and the +// agents extension reads the same key to build the same links. +const projectARMIDEnvKey = "AZURE_AI_PROJECT_ID" + +// portalPrefix builds the Foundry portal prefix for this project, or nil when +// the project cannot be addressed. +// +// Best effort by design: a portal link is a convenience on top of a command +// that already did its work, so a missing or unparseable resource ID drops the +// line rather than failing the command that earned it. +func (ec *evalContext) portalPrefix(ctx context.Context) *eval_api.PortalPrefix { + armID := ec.getEnvValue(ctx, projectARMIDEnvKey) + if armID == "" { + return nil + } + prefix, err := eval_api.NewPortalPrefix(armID) + if err != nil { + log.Printf("[portal] %s is not a project resource ID: %v", projectARMIDEnvKey, err) + return nil + } + return prefix +} + +// withPortalLink stamps a run with its portal URL, so the terminal and `-o json` +// answer with the same link from one place. +func (ec *evalContext) withPortalLink( + ctx context.Context, + evalID string, + run *eval_api.OpenAIEvalRun, +) *eval_api.OpenAIEvalRun { + if run == nil || evalID == "" || run.ID == "" { + return run + } + if prefix := ec.portalPrefix(ctx); prefix != nil { + run.PortalURL = prefix.EvalRunURL(evalID, run.ID) + } + return run +} + // azd environment keys written by this extension. const ( envKeyEvalID = "EVAL_ID" diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset.go index 05a42c354e1..e9735a0b032 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset.go @@ -277,10 +277,16 @@ func newDatasetShowCommand() *cobra.Command { if isJSON(cmd) { return emitJSON(cmd.OutOrStdout(), ds) } - return emitTable(cmd.OutOrStdout(), + if err := emitTable(cmd.OutOrStdout(), []string{"NAME", "VERSION", "FORMAT", "URI"}, [][]string{{ds.Name, ds.Version, ds.Format, ds.ResolvedBlobURI()}}, - ) + ); err != nil { + return err + } + if prefix := ec.portalPrefix(ctx); prefix != nil { + writePortalLink(cmd.OutOrStdout(), prefix.DatasetURL(ds.Name, ds.Version)) + } + return nil }, } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/output.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/output.go index 143b49e91ba..07578515781 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/output.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/output.go @@ -10,6 +10,7 @@ import ( "strings" "text/tabwriter" + "github.com/fatih/color" "github.com/spf13/cobra" ) @@ -23,6 +24,18 @@ const ( failedMark = "(x) Failed:" // the step did not complete ) +// writePortalLink closes a detail view with the asset's portal URL. +// +// Last line and cyan, matching the sibling extensions, and silent when there is +// no URL — the link is a convenience on top of work already done, so its +// absence must not look like a failure. +func writePortalLink(w io.Writer, url string) { + if url == "" { + return + } + fmt.Fprintf(w, "Portal: %s\n", color.CyanString(url)) +} + // outputFormat reads the inherited -o/--output flag. func outputFormat(cmd *cobra.Command) string { if cmd == nil { diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/portal_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/portal_test.go new file mode 100644 index 00000000000..2d40dfc8533 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/portal_test.go @@ -0,0 +1,100 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "bytes" + "encoding/json" + "strings" + "testing" + + "azureaieval/internal/pkg/eval_api" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// The portal link is the last line of a detail view, and it is the one thing a +// user clicks to see the run they just waited for. +func TestWritePortalLink(t *testing.T) { + var buf bytes.Buffer + writePortalLink(&buf, "https://ai.azure.com/nextgen/r/x,y,,z,p/build/evaluations/e/run/r") + + out := buf.String() + assert.Contains(t, out, "Portal: ") + assert.Contains(t, out, "/build/evaluations/e/run/r") + assert.True(t, strings.HasSuffix(out, "\n"), "it closes the view, so it ends the line") +} + +// Resolution is best effort: the link is a convenience on top of work already +// done, so having none must print nothing rather than an empty label that +// reads like a failure. +func TestWritePortalLink_SilentWithoutAURL(t *testing.T) { + var buf bytes.Buffer + writePortalLink(&buf, "") + + assert.Empty(t, buf.String()) +} + +// `-o json` carries the same link the terminal prints, so a pipeline reading +// JSON is not the one consumer that cannot find the run in the portal. +func TestRunPortalURLTravelsInJSON(t *testing.T) { + run := &eval_api.OpenAIEvalRun{ + ID: "evalrun_1", + Status: "completed", + PortalURL: "https://ai.azure.com/nextgen/r/x,y,,z,p/build/evaluations/eval_1/run/evalrun_1", + } + + var buf bytes.Buffer + require.NoError(t, emitJSON(&buf, run)) + + var decoded map[string]any + require.NoError(t, json.Unmarshal(buf.Bytes(), &decoded)) + assert.Equal(t, run.PortalURL, decoded["portal_url"], + "the key is portal_url, which is what the spec tells consumers to read") +} + +// A run with no portal link must not carry an empty key, or a consumer cannot +// tell "no link" from "link is the empty string". +func TestRunWithoutPortalURLOmitsTheKey(t *testing.T) { + var buf bytes.Buffer + require.NoError(t, emitJSON(&buf, &eval_api.OpenAIEvalRun{ID: "evalrun_1"})) + + var decoded map[string]any + require.NoError(t, json.Unmarshal(buf.Bytes(), &decoded)) + assert.NotContains(t, decoded, "portal_url") +} + +// The portal URL is built from the eval and run ids, which is what makes the +// link land on the run rather than the eval's list of them. +func TestPortalRunURLShape(t *testing.T) { + prefix, err := eval_api.NewPortalPrefix( + "/subscriptions/00000000-1111-2222-3333-444444444444/resourceGroups/rg/" + + "providers/Microsoft.CognitiveServices/accounts/acct/projects/proj") + require.NoError(t, err) + + assert.True(t, strings.HasSuffix( + prefix.EvalRunURL("eval_1", "evalrun_9"), + "/build/evaluations/eval_1/run/evalrun_9")) +} + +// The report line and the portal line are different things: one is the +// service's own link, the other is built here. A run carrying both prints both. +func TestRenderRunPrintsReportAndPortalSeparately(t *testing.T) { + run := &eval_api.OpenAIEvalRun{ + ID: "evalrun_1", + Status: "completed", + ReportURL: "https://service.example/report/1", + PortalURL: "https://ai.azure.com/nextgen/r/x,y,,z,p/build/evaluations/e/run/r", + } + + var buf bytes.Buffer + require.NoError(t, renderRun(&buf, run, nil)) + + out := buf.String() + assert.Contains(t, out, "Report: https://service.example/report/1") + assert.Contains(t, out, "Portal: ") + assert.Less(t, strings.Index(out, "Report:"), strings.Index(out, "Portal:"), + "the portal link closes the view") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go index c3591d523f9..efd5ae895ab 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go @@ -187,7 +187,7 @@ func buildRunCommand(use, short string) *cobra.Command { return emitJSON(out, startedRun(run, evalID, group)) } fmt.Fprintf(out, "Started run %s (status: %s)\n", run.ID, run.Status) - fmt.Fprintf(out, "Reattach with: azd ai eval run show %s --eval-id %s\n", run.ID, evalID) + fmt.Fprintf(out, "Reattach with: azd ai eval run show %s --eval %s\n", run.ID, evalID) return nil } @@ -195,6 +195,7 @@ func buildRunCommand(use, short string) *cobra.Command { if err != nil { return err } + final = ec.withPortalLink(ctx, evalID, final) if isJSON(cmd) { if err := emitJSON(out, final); err != nil { @@ -769,6 +770,7 @@ func renderRun( if run.ReportURL != "" { fmt.Fprintf(out, "Report: %s\n", run.ReportURL) } + writePortalLink(out, run.PortalURL) return nil } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go index cea3b01bb4d..2efd578269d 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go @@ -122,6 +122,7 @@ func newRunShowCommand() *cobra.Command { if err != nil { return err } + run = ec.withPortalLink(ctx, evalID, run) // Reattaching to a run started asynchronously: the pipeline that // gates on it is often not the one that started it. @@ -161,6 +162,7 @@ func newRunShowCommand() *cobra.Command { if run.ReportURL != "" { fmt.Fprintf(out, " report : %s\n", run.ReportURL) } + writePortalLink(out, run.PortalURL) if gateOnStatus { if err := runCompleted(run); err != nil { return err diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/surface_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/surface_test.go index bf6be972f2a..993044751ed 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/surface_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/surface_test.go @@ -351,6 +351,62 @@ func TestTaggedSuitesNameFlagsThatExist(t *testing.T) { require.NoError(t, err) } +// A suggestion that names a flag has to name one the suggested command takes. +// +// `run start --no-wait` closed with "Reattach with: azd ai eval run show +// --eval-id " for the whole life of the branch that removed --eval-id. +// The command resolved, so the suggestion check passed; the flag did not exist, +// so the one line a user is told to paste was the one guaranteed to fail. +func TestSuggestedFlagsExist(t *testing.T) { + // `azd ai eval ... --flag`, with the flag anywhere after it. + suggestion := regexp.MustCompile(`azd ai eval ((?:[a-z][a-z0-9-]*\s+)+)([^"'\n]*)`) + flagName := regexp.MustCompile(`--([a-z][a-z0-9-]*)`) + + err := filepath.WalkDir("../..", func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() || !strings.HasSuffix(path, ".go") || strings.HasSuffix(path, "_test.go") { + return nil + } + body, err := os.ReadFile(path) + if err != nil { + return err + } + for i, line := range strings.Split(string(body), "\n") { + if strings.HasPrefix(strings.TrimSpace(line), "//") { + continue + } + for _, m := range suggestion.FindAllStringSubmatch(line, -1) { + flags := flagName.FindAllStringSubmatch(m[2], -1) + if len(flags) == 0 { + continue + } + // Longest command prefix that resolves; the rest is arguments. + words := strings.Fields(m[1]) + for len(words) > 0 { + if resolved, _, e := NewRootCommand().Find(words); e == nil && + strings.Fields(resolved.Use)[0] == words[len(words)-1] { + break + } + words = words[:len(words)-1] + } + if len(words) == 0 { + continue // the command itself is checked above + } + cmd, _, _ := NewRootCommand().Find(words) + for _, f := range flags { + assert.NotNilf(t, cmd.Flags().Lookup(f[1]), + "%s:%d suggests `azd ai eval %s --%s`, which that command does not accept", + path, i+1, strings.Join(words, " "), f[1]) + } + } + } + return nil + }) + require.NoError(t, err) +} + // find resolves a command path, failing the test when it does not exist. func find(t *testing.T, path string) *cobra.Command { t.Helper() diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/models.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/models.go index ceed191847a..8f6df9d6bf9 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/models.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/models.go @@ -419,6 +419,9 @@ type OpenAIEvalRun struct { DataSource *EvalRunDataSource `json:"data_source,omitempty"` Metadata map[string]string `json:"metadata,omitempty"` ReportURL string `json:"report_url,omitempty"` + // PortalURL is built by the extension, not returned by the service, so that + // `-o json` carries the same link the terminal prints. + PortalURL string `json:"portal_url,omitempty"` // Result summary ResultCounts *EvalRunResultCounts `json:"result_counts,omitempty"` From aca6aacfe77b6e1e5f8b0c1878d614fb79465718 Mon Sep 17 00:00:00 2001 From: mohessie Date: Thu, 6 Aug 2026 05:17:43 +0300 Subject: [PATCH 119/320] Resolve the project endpoint the way the spec says, in five levels not three The spec says commands use "the sibling Foundry extensions' five-level cascade verbatim" and lists it: flag, active azd env (FOUNDRY_PROJECT_ENDPOINT then AZURE_AI_PROJECT_ENDPOINT), global config extensions.ai-agents.project.context.endpoint, host env (same two keys), then a structured error. Both extensions did three of those. AZURE_AI_PROJECT_ENDPOINT appeared nowhere in either, and the global config level did not exist -- so a project set up with `azd ai project set`, or one using the AZURE_AI_ spelling, resolved to "no Foundry project endpoint found" while holding one. The cascade is azure.ai.toolboxes' projectctx package, copied rather than imported because extensions share no code, along with the exterrors package it validates through. Copying is what "verbatim" costs here; the alternative is four extensions drifting apart on the one thing they all have to agree about. It also brings the spec's other requirement, which the old code did not have: an invalid value at any level is a hard error rather than a silent fall to the next. Falling through is how a typo'd endpoint in an azd env becomes a confusing 401 against the wrong project. The suggestion guards learn about sibling namespaces, because the copied validator points at `azd ai project set` and that is a real command in a real extension. Listed explicitly rather than wildcarded, so a typo in a namespace still fails. --- .../azure.ai.dataset/internal/cmd/context.go | 24 +- .../internal/cmd/surface_test.go | 11 +- .../internal/exterrors/codes.go | 71 +++++ .../internal/exterrors/errors.go | 155 +++++++++++ .../internal/foundry/projectctx/resolver.go | 152 ++++++++++ .../foundry/projectctx/resolver_test.go | 259 ++++++++++++++++++ .../internal/foundry/projectctx/store.go | 40 +++ .../internal/foundry/projectctx/types.go | 81 ++++++ .../internal/foundry/projectctx/validator.go | 113 ++++++++ .../internal/cmd/context.go | 23 +- .../internal/cmd/surface_test.go | 29 +- .../internal/exterrors/codes.go | 71 +++++ .../internal/exterrors/errors.go | 155 +++++++++++ .../internal/foundry/projectctx/resolver.go | 152 ++++++++++ .../foundry/projectctx/resolver_test.go | 259 ++++++++++++++++++ .../internal/foundry/projectctx/store.go | 40 +++ .../internal/foundry/projectctx/types.go | 81 ++++++ .../internal/foundry/projectctx/validator.go | 113 ++++++++ 18 files changed, 1789 insertions(+), 40 deletions(-) create mode 100644 cli/azd/extensions/azure.ai.dataset/internal/exterrors/codes.go create mode 100644 cli/azd/extensions/azure.ai.dataset/internal/exterrors/errors.go create mode 100644 cli/azd/extensions/azure.ai.dataset/internal/foundry/projectctx/resolver.go create mode 100644 cli/azd/extensions/azure.ai.dataset/internal/foundry/projectctx/resolver_test.go create mode 100644 cli/azd/extensions/azure.ai.dataset/internal/foundry/projectctx/store.go create mode 100644 cli/azd/extensions/azure.ai.dataset/internal/foundry/projectctx/types.go create mode 100644 cli/azd/extensions/azure.ai.dataset/internal/foundry/projectctx/validator.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/exterrors/codes.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/exterrors/errors.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/foundry/projectctx/resolver.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/foundry/projectctx/resolver_test.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/foundry/projectctx/store.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/foundry/projectctx/types.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/foundry/projectctx/validator.go diff --git a/cli/azd/extensions/azure.ai.dataset/internal/cmd/context.go b/cli/azd/extensions/azure.ai.dataset/internal/cmd/context.go index f8637c2b073..0ad178f0f67 100644 --- a/cli/azd/extensions/azure.ai.dataset/internal/cmd/context.go +++ b/cli/azd/extensions/azure.ai.dataset/internal/cmd/context.go @@ -7,9 +7,10 @@ import ( "context" "errors" "fmt" - "os" + "log" "strings" + "azureaidataset/internal/foundry/projectctx" "azureaidataset/internal/pkg/dataset_api" "azureaidataset/internal/pkg/gen_api" @@ -56,24 +57,15 @@ func newDatasetContext(ctx context.Context, endpointFlag string) (*datasetContex // The environment name is resolved regardless of where the endpoint came // from: it is what cached version numbers are read from and written to. - azdEndpoint, envName := lookupEndpointFromAzd(ctx, azdClient) + _, envName := lookupEndpointFromAzd(ctx, azdClient) dc.envName = envName - if endpointFlag != "" { - dc.endpoint = endpointFlag - } else { - dc.endpoint = azdEndpoint - } - if dc.endpoint == "" { - dc.endpoint = os.Getenv(projectEndpointEnvKey) - } - if dc.endpoint == "" { - return nil, fmt.Errorf( - "no Foundry project endpoint found; pass --project-endpoint or set %s "+ - "in the azd environment (azd env set %s )", - projectEndpointEnvKey, projectEndpointEnvKey) + resolved, err := projectctx.Resolve(ctx, projectctx.ResolveOpts{FlagValue: endpointFlag}) + if err != nil { + return nil, err } - dc.endpoint = strings.TrimSuffix(dc.endpoint, "/") + dc.endpoint = strings.TrimSuffix(resolved.Endpoint, "/") + log.Printf("[endpoint] resolved from %s", resolved.Source) cred, err := azidentity.NewAzureDeveloperCLICredential( &azidentity.AzureDeveloperCLICredentialOptions{}, diff --git a/cli/azd/extensions/azure.ai.dataset/internal/cmd/surface_test.go b/cli/azd/extensions/azure.ai.dataset/internal/cmd/surface_test.go index 9f1a9161fd5..b625e22dede 100644 --- a/cli/azd/extensions/azure.ai.dataset/internal/cmd/surface_test.go +++ b/cli/azd/extensions/azure.ai.dataset/internal/cmd/surface_test.go @@ -158,8 +158,9 @@ func TestSuggestedCommandsExist(t *testing.T) { } // A suggestion pointing at a sibling extension is that // extension's contract, not this one's, and cannot be resolved - // from here. `eval` is the only one this extension names. - if words[0] == "eval" { + // from here. Listed rather than wildcarded so a typo in a + // namespace still fails. + if siblingNamespaces[words[0]] { continue } assert.Equalf(t, "dataset", words[0], @@ -186,6 +187,12 @@ func TestSuggestedCommandsExist(t *testing.T) { require.NoError(t, err) } +// siblingNamespaces are the other Foundry extensions this one points users at. +var siblingNamespaces = map[string]bool{ + "eval": true, // registering a generated dataset in an eval configuration + "project": true, // `azd ai project set` owns the shared endpoint context +} + func find(t *testing.T, path string) *cobra.Command { t.Helper() cmd, _, err := NewRootCommand().Find(strings.Fields(path)) diff --git a/cli/azd/extensions/azure.ai.dataset/internal/exterrors/codes.go b/cli/azd/extensions/azure.ai.dataset/internal/exterrors/codes.go new file mode 100644 index 00000000000..58fc300148c --- /dev/null +++ b/cli/azd/extensions/azure.ai.dataset/internal/exterrors/codes.go @@ -0,0 +1,71 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package exterrors + +// Error codes for user cancellation. +const ( + CodeCancelled = "cancelled" +) + +// Error codes for validation failures (user input, manifests, flags). +const ( + CodeInvalidParameter = "invalid_parameter" + CodeInvalidPositionalArg = "invalid_positional_arg" +) + +// Error codes for dependency failures (missing resources, services, env values). +const ( + CodeAzdClientFailed = "azd_client_failed" + CodeMissingProjectEndpoint = "missing_project_endpoint" +) + +// Error codes for auth failures. +const ( + CodeNotLoggedIn = "not_logged_in" + CodeLoginExpired = "login_expired" + CodeAuthFailed = "auth_failed" +) + +// Error codes for toolbox operations. +const ( + CodeToolboxNotFound = "toolbox_not_found" + CodeToolboxVersionNotFound = "toolbox_version_not_found" + CodeInvalidToolboxName = "invalid_toolbox_name" + CodeMissingUpdateField = "missing_update_field" + CodeDefaultVersionDelete = "default_version_delete" + CodeOnlyVersionDelete = "only_version_delete" + CodeMissingForceFlag = "missing_force_flag" + CodeUnsupportedConnectionCategory = "unsupported_connection_category" + CodeMissingIndex = "missing_index" + CodeUnsupportedIndexFlag = "unsupported_index_flag" + CodeMissingInstanceName = "missing_instance_name" + CodeUnsupportedInstanceNameFlag = "unsupported_instance_name_flag" + CodeInvalidSkillName = "invalid_skill_name" + CodeInvalidSkillSpec = "invalid_skill_spec" + CodeDuplicateSkill = "duplicate_skill" + CodeSkillNotInToolbox = "skill_not_in_toolbox" + CodeSkillAlreadyAttached = "skill_already_attached" + CodeDuplicateConnection = "duplicate_connection" + CodeDuplicateToolName = "duplicate_tool_name" + CodeMissingToolType = "missing_tool_type" + CodeConnectionNotFound = "connection_not_found" + CodeConnectionNotInToolbox = "connection_not_in_toolbox" + CodeConnectionMissingTarget = "connection_missing_target" + CodeLastToolRemoval = "last_tool_removal" + CodePendingToolboxStoreFailed = "pending_toolbox_store_failed" +) + +// Operation names for [ServiceFromAzure] errors. +// These are prefixed to the Azure error code (e.g., "get_toolbox.NotFound"). +const ( + OpCreateToolboxVersion = "create_toolbox_version" + OpGetToolbox = "get_toolbox" + OpDeleteToolbox = "delete_toolbox" + OpDeleteToolboxVersion = "delete_toolbox_version" + OpSetDefaultVersion = "set_default_version" + OpListToolboxes = "list_toolboxes" + OpGetToolboxVersion = "get_toolbox_version" + OpListToolboxVersions = "list_toolbox_versions" + OpResolveProjectConnection = "resolve_project_connection" +) diff --git a/cli/azd/extensions/azure.ai.dataset/internal/exterrors/errors.go b/cli/azd/extensions/azure.ai.dataset/internal/exterrors/errors.go new file mode 100644 index 00000000000..4e3c02b6731 --- /dev/null +++ b/cli/azd/extensions/azure.ai.dataset/internal/exterrors/errors.go @@ -0,0 +1,155 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +// Package exterrors provides structured error helpers for the azure.ai.toolboxes +// extension. +// +// Use plain Go errors until the current code can confidently choose a final +// category, code, and suggestion. At that point, create a structured error with +// one of the helpers in this package or with [ServiceFromAzure] for Azure SDK +// failures. +// +// Once an error is structured, usually return it unchanged. Avoid wrapping a +// structured error with [fmt.Errorf] and %w for extra context: azd serializes +// the structured error's own message and metadata, not the outer wrapper text. +package exterrors + +import ( + "context" + "errors" + "fmt" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore" + "github.com/azure/azure-dev/cli/azd/pkg/azdext" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +// --------------------------------------------------------------------------- +// Structured error factories +// --------------------------------------------------------------------------- + +// Validation returns a validation [azdext.LocalError] for user input / flag errors. +func Validation(code, message, suggestion string) error { + return &azdext.LocalError{ + Message: message, + Code: code, + Category: azdext.LocalErrorCategoryValidation, + Suggestion: suggestion, + } +} + +// Dependency returns a dependency [azdext.LocalError] for missing resources or services. +func Dependency(code, message, suggestion string) error { + return &azdext.LocalError{ + Message: message, + Code: code, + Category: azdext.LocalErrorCategoryDependency, + Suggestion: suggestion, + } +} + +// Auth returns an auth [azdext.LocalError] for authentication/authorization failures. +func Auth(code, message, suggestion string) error { + return &azdext.LocalError{ + Message: message, + Code: code, + Category: azdext.LocalErrorCategoryAuth, + Suggestion: suggestion, + } +} + +// User returns a user-action [azdext.LocalError] (e.g. cancellation). No suggestion. +func User(code, message string) error { + return &azdext.LocalError{ + Message: message, + Code: code, + Category: azdext.LocalErrorCategoryUser, + } +} + +// Internal returns an internal [azdext.LocalError] for unexpected extension failures. +func Internal(code, message string) error { + return &azdext.LocalError{ + Message: message, + Code: code, + Category: azdext.LocalErrorCategoryInternal, + } +} + +// Cancelled returns a user cancellation error. +func Cancelled(message string) error { + return User(CodeCancelled, message) +} + +// --------------------------------------------------------------------------- +// Azure error converters +// --------------------------------------------------------------------------- + +// ServiceFromAzure wraps an [azcore.ResponseError] into an [azdext.ServiceError] +// with operation context. If the error is not an azcore.ResponseError, it +// returns a generic internal [azdext.LocalError]. +func ServiceFromAzure(err error, operation string) error { + var respErr *azcore.ResponseError + if errors.As(err, &respErr) { + serviceName := "" + if respErr.RawResponse != nil && respErr.RawResponse.Request != nil { + serviceName = respErr.RawResponse.Request.Host + } + code := respErr.ErrorCode + if code == "" { + code = fmt.Sprintf("%d", respErr.StatusCode) + } + return &azdext.ServiceError{ + Message: fmt.Sprintf("%s: %s", operation, respErr.Error()), + ErrorCode: fmt.Sprintf("%s.%s", operation, code), + StatusCode: respErr.StatusCode, + ServiceName: serviceName, + } + } + if IsCancellation(err) { + return Cancelled(fmt.Sprintf("%s was cancelled", operation)) + } + return Internal(operation, fmt.Sprintf("%s: %s", operation, err.Error())) +} + +// FromPrompt wraps a gRPC error from an azd host Prompt call into a structured +// error. Auth errors (Unauthenticated) are classified as Auth errors with a +// re-auth suggestion; cancellations as User cancellations; other errors are +// returned wrapped with the provided context message. +func FromPrompt(err error, contextMsg string) error { + if err == nil { + return nil + } + + if IsCancellation(err) { + return Cancelled(contextMsg) + } + + st, ok := status.FromError(err) + if ok && st.Code() == codes.Unauthenticated { + return Auth( + CodeAuthFailed, + fmt.Sprintf("%s: %s", contextMsg, st.Message()), + "run `azd auth login` to authenticate", + ) + } + + return fmt.Errorf("%s: %w", contextMsg, err) +} + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +// IsCancellation reports whether err represents user cancellation +// ([context.Canceled] or gRPC [codes.Canceled]). +func IsCancellation(err error) bool { + if errors.Is(err, context.Canceled) { + return true + } + if st, ok := status.FromError(err); ok && st.Code() == codes.Canceled { + return true + } + return false +} diff --git a/cli/azd/extensions/azure.ai.dataset/internal/foundry/projectctx/resolver.go b/cli/azd/extensions/azure.ai.dataset/internal/foundry/projectctx/resolver.go new file mode 100644 index 00000000000..3c6c3129a50 --- /dev/null +++ b/cli/azd/extensions/azure.ai.dataset/internal/foundry/projectctx/resolver.go @@ -0,0 +1,152 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package projectctx + +import ( + "context" + "errors" + "os" + + "github.com/azure/azure-dev/cli/azd/pkg/azdext" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +// ReadAzdHostedSourcesFunc is a package-level seam so tests can stub the +// daemon-backed lookup without spinning up a real azd gRPC server. +var ReadAzdHostedSourcesFunc = readAzdHostedSources + +// readAzdHostedSources dials the azd daemon (if reachable) and reads both the +// active environment's project endpoint and the global-config project context +// in a single client lifetime. The active-env read prefers +// FOUNDRY_PROJECT_ENDPOINT and falls back to AZURE_AI_PROJECT_ENDPOINT (the key +// `azd ai agent init` / `azd add` persist). Errors talking to the daemon are +// returned only for non-Unavailable cases on the config read — Unavailable is +// treated as "no daemon" and the caller falls through to subsequent levels. +func readAzdHostedSources(ctx context.Context) (AzdHostedSources, error) { + var out AzdHostedSources + + azdClient, err := azdext.NewAzdClient() + if err != nil { + // No azd client at all => no hosted sources, not an error. + return out, nil + } + defer azdClient.Close() + + if envResp, err := azdClient.Environment().GetCurrent( + ctx, &azdext.EmptyRequest{}, + ); err == nil { + for _, key := range []string{foundryEnvKey, azureAiEnvKey} { + envVal, valErr := azdClient.Environment().GetValue(ctx, &azdext.GetEnvRequest{ + EnvName: envResp.Environment.Name, + Key: key, + }) + if valErr == nil && envVal.Value != "" { + out.EnvValue = envVal.Value + out.EnvName = envResp.Environment.Name + break + } + } + } + + state, found, cfgErr := getProjectContext(ctx, azdClient) + if cfgErr != nil { + // A gRPC Unavailable code means the azd daemon is not reachable; + // treat it the same as azdClient creation failing and fall through. + // Any other error (e.g. parse failure) is a hard error. + if !containsGRPCCode(cfgErr, codes.Unavailable) { + return out, cfgErr + } + } else { + out.CfgState = state + out.CfgFound = found + } + + return out, nil +} + +// containsGRPCCode walks the error chain looking for a gRPC status with the +// specified code. fmt.Errorf("%w", ...) wraps errors without forwarding the +// GRPCStatus() method, so we must unwrap manually. +// +// Note: only follows errors.Unwrap chains; errors.Join multi-wraps are not traversed. +func containsGRPCCode(err error, code codes.Code) bool { + for ; err != nil; err = errors.Unwrap(err) { + if st, ok := status.FromError(err); ok && st.Code() == code { + return true + } + } + return false +} + +// Resolve resolves a Foundry project endpoint using the 5-level cascade: +// +// 1. --project-endpoint flag +// 2. Active azd env value (FOUNDRY_PROJECT_ENDPOINT, then AZURE_AI_PROJECT_ENDPOINT) +// 3. Global config: extensions.ai-agents.project.context.endpoint (read-only; +// owned by azure.ai.agents) +// 4. Host environment variable (FOUNDRY_PROJECT_ENDPOINT, then AZURE_AI_PROJECT_ENDPOINT) +// 5. Structured error with actionable suggestion +// +// Invalid values at any level produce a hard validation error (no silent fallback). +func Resolve(ctx context.Context, opts ResolveOpts) (*Resolved, error) { + // Level 1: explicit flag. + if opts.FlagValue != "" { + normalized, _, err := Validate(opts.FlagValue) + if err != nil { + return nil, err + } + return &Resolved{Endpoint: normalized, Source: SourceFlag}, nil + } + + // Levels 2 + 3: azd-hosted sources (active env, then global config). + sources, err := ReadAzdHostedSourcesFunc(ctx) + if err != nil { + return nil, err + } + + // Level 2: active azd environment's FOUNDRY_PROJECT_ENDPOINT (with the + // AZURE_AI_PROJECT_ENDPOINT fallback applied in readAzdHostedSources). + if sources.EnvValue != "" { + normalized, _, err := Validate(sources.EnvValue) + if err != nil { + return nil, err + } + return &Resolved{ + Endpoint: normalized, + Source: SourceAzdEnv, + AzdEnvName: sources.EnvName, + }, nil + } + + // Level 3: global config (~/.azd/config.json). + if sources.CfgFound && sources.CfgState.Endpoint != "" { + normalized, _, err := Validate(sources.CfgState.Endpoint) + if err != nil { + return nil, err + } + return &Resolved{ + Endpoint: normalized, + Source: SourceGlobalConfig, + SetAt: sources.CfgState.SetAt, + }, nil + } + + // Level 4: host environment variable (FOUNDRY_PROJECT_ENDPOINT, then the + // AZURE_AI_PROJECT_ENDPOINT fallback). + for _, key := range []string{foundryEnvKey, azureAiEnvKey} { + envVal := os.Getenv(key) + if envVal == "" { + continue + } + normalized, _, err := Validate(envVal) + if err != nil { + return nil, err + } + return &Resolved{Endpoint: normalized, Source: SourceFoundryEnv}, nil + } + + // Level 5: structured error. + return nil, NoEndpointError() +} diff --git a/cli/azd/extensions/azure.ai.dataset/internal/foundry/projectctx/resolver_test.go b/cli/azd/extensions/azure.ai.dataset/internal/foundry/projectctx/resolver_test.go new file mode 100644 index 00000000000..f21bbbd60fe --- /dev/null +++ b/cli/azd/extensions/azure.ai.dataset/internal/foundry/projectctx/resolver_test.go @@ -0,0 +1,259 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package projectctx + +import ( + "context" + "errors" + "testing" + + "azureaidataset/internal/exterrors" + + "github.com/azure/azure-dev/cli/azd/pkg/azdext" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// withHostedSources installs a stub for ReadAzdHostedSourcesFunc for the +// duration of the test and restores the production value on cleanup. Tests +// using this MUST NOT run in parallel because the seam is a package-level var. +func withHostedSources(t *testing.T, sources AzdHostedSources, err error) { + t.Helper() + orig := ReadAzdHostedSourcesFunc + ReadAzdHostedSourcesFunc = func(context.Context) (AzdHostedSources, error) { + return sources, err + } + t.Cleanup(func() { ReadAzdHostedSourcesFunc = orig }) +} + +// isolateFromAzdDaemon installs an empty hosted-sources stub and clears +// AZD_SERVER so any code path that bypasses the seam cannot reach a real +// daemon. After calling this, the resolver only sees the flag and the +// FOUNDRY_PROJECT_ENDPOINT / AZURE_AI_PROJECT_ENDPOINT host env vars. +func isolateFromAzdDaemon(t *testing.T) { + t.Helper() + t.Setenv("AZD_SERVER", "") + withHostedSources(t, AzdHostedSources{}, nil) +} + +func TestResolve_FlagWins(t *testing.T) { + // Even with FOUNDRY_PROJECT_ENDPOINT and azd-hosted sources set, the flag wins. + t.Setenv("FOUNDRY_PROJECT_ENDPOINT", "https://env.services.ai.azure.com/api/projects/env-proj") + withHostedSources(t, AzdHostedSources{ + EnvValue: "https://azdenv.services.ai.azure.com/api/projects/p", + EnvName: "dev", + }, nil) + + result, err := Resolve(t.Context(), ResolveOpts{ + FlagValue: "https://flag.services.ai.azure.com/api/projects/flag-proj", + }) + require.NoError(t, err) + assert.Equal(t, "https://flag.services.ai.azure.com/api/projects/flag-proj", result.Endpoint) + assert.Equal(t, SourceFlag, result.Source) +} + +func TestResolve_AzdEnvWinsOverConfigAndFoundryEnv(t *testing.T) { + // EnvValue here stands in for whichever active-env key readAzdHostedSources + // resolved (FOUNDRY_PROJECT_ENDPOINT, or the AZURE_AI_PROJECT_ENDPOINT + // fallback); either way level 2 wins over global config and the host env. + t.Setenv("FOUNDRY_PROJECT_ENDPOINT", "https://foundry.services.ai.azure.com/api/projects/p") + withHostedSources(t, AzdHostedSources{ + EnvValue: " HTTPS://Azdenv.Services.AI.Azure.com/api/projects/p/ ", + EnvName: "dev", + CfgState: State{ + Endpoint: "https://cfg.services.ai.azure.com/api/projects/p", + SetAt: "2025-01-01T00:00:00Z", + }, + CfgFound: true, + }, nil) + + result, err := Resolve(t.Context(), ResolveOpts{}) + require.NoError(t, err) + assert.Equal(t, "https://azdenv.services.ai.azure.com/api/projects/p", result.Endpoint) + assert.Equal(t, SourceAzdEnv, result.Source) + assert.Equal(t, "dev", result.AzdEnvName) +} + +func TestResolve_AzdEnvInvalidIsHardError(t *testing.T) { + // Level 2 invalid values are hard errors (no silent fallback to lower levels). + t.Setenv("FOUNDRY_PROJECT_ENDPOINT", "https://foundry.services.ai.azure.com/api/projects/p") + withHostedSources(t, AzdHostedSources{ + EnvValue: "http://not-https.services.ai.azure.com/api/projects/p", + EnvName: "dev", + }, nil) + + _, err := Resolve(t.Context(), ResolveOpts{}) + require.Error(t, err) + + var localErr *azdext.LocalError + require.ErrorAs(t, err, &localErr) + assert.Contains(t, localErr.Message, "https") +} + +func TestResolve_GlobalConfigWinsOverFoundryEnv(t *testing.T) { + t.Setenv("FOUNDRY_PROJECT_ENDPOINT", "https://foundry.services.ai.azure.com/api/projects/p") + withHostedSources(t, AzdHostedSources{ + CfgState: State{ + Endpoint: " HTTPS://Cfg.Services.AI.Azure.com/api/projects/p/ ", + SetAt: "2025-01-02T03:04:05Z", + }, + CfgFound: true, + }, nil) + + result, err := Resolve(t.Context(), ResolveOpts{}) + require.NoError(t, err) + assert.Equal(t, "https://cfg.services.ai.azure.com/api/projects/p", result.Endpoint) + assert.Equal(t, SourceGlobalConfig, result.Source) + assert.Equal(t, "2025-01-02T03:04:05Z", result.SetAt) +} + +func TestResolve_GlobalConfigInvalidIsHardError(t *testing.T) { + t.Setenv("FOUNDRY_PROJECT_ENDPOINT", "https://foundry.services.ai.azure.com/api/projects/p") + withHostedSources(t, AzdHostedSources{ + CfgState: State{ + Endpoint: "http://not-https.services.ai.azure.com/api/projects/p", + SetAt: "2025-01-02T03:04:05Z", + }, + CfgFound: true, + }, nil) + + _, err := Resolve(t.Context(), ResolveOpts{}) + require.Error(t, err) + + var localErr *azdext.LocalError + require.ErrorAs(t, err, &localErr) + assert.Contains(t, localErr.Message, "https") +} + +func TestResolve_HostedSourcesErrorPropagates(t *testing.T) { + // Non-recoverable errors from the hosted-source lookup must be surfaced + // and must not silently fall through to level 4. + t.Setenv("FOUNDRY_PROJECT_ENDPOINT", "https://foundry.services.ai.azure.com/api/projects/p") + sentinel := errors.New("boom") + withHostedSources(t, AzdHostedSources{}, sentinel) + + _, err := Resolve(t.Context(), ResolveOpts{}) + require.ErrorIs(t, err, sentinel) +} + +func TestResolve_FoundryEnvFallback(t *testing.T) { + isolateFromAzdDaemon(t) + t.Setenv("FOUNDRY_PROJECT_ENDPOINT", "https://env.services.ai.azure.com/api/projects/env-proj") + + result, err := Resolve(t.Context(), ResolveOpts{}) + require.NoError(t, err) + assert.Equal(t, "https://env.services.ai.azure.com/api/projects/env-proj", result.Endpoint) + assert.Equal(t, SourceFoundryEnv, result.Source) +} + +func TestResolve_AzureAiHostEnvFallback(t *testing.T) { + // When FOUNDRY_PROJECT_ENDPOINT is unset, the resolver falls back to the + // AZURE_AI_PROJECT_ENDPOINT host env var (the key azd ai agent init / azd + // add persist). See https://github.com/Azure/azure-dev/issues/8688. + isolateFromAzdDaemon(t) + t.Setenv("FOUNDRY_PROJECT_ENDPOINT", "") + t.Setenv("AZURE_AI_PROJECT_ENDPOINT", "https://azureai.services.ai.azure.com/api/projects/p") + + result, err := Resolve(t.Context(), ResolveOpts{}) + require.NoError(t, err) + assert.Equal(t, "https://azureai.services.ai.azure.com/api/projects/p", result.Endpoint) + assert.Equal(t, SourceFoundryEnv, result.Source) +} + +func TestResolve_FoundryHostEnvWinsOverAzureAi(t *testing.T) { + // With both host env vars set, FOUNDRY_PROJECT_ENDPOINT takes precedence. + isolateFromAzdDaemon(t) + t.Setenv("FOUNDRY_PROJECT_ENDPOINT", "https://foundry.services.ai.azure.com/api/projects/f") + t.Setenv("AZURE_AI_PROJECT_ENDPOINT", "https://azureai.services.ai.azure.com/api/projects/a") + + result, err := Resolve(t.Context(), ResolveOpts{}) + require.NoError(t, err) + assert.Equal(t, "https://foundry.services.ai.azure.com/api/projects/f", result.Endpoint) + assert.Equal(t, SourceFoundryEnv, result.Source) +} + +func TestResolve_FoundryEnvNormalized(t *testing.T) { + isolateFromAzdDaemon(t) + t.Setenv("FOUNDRY_PROJECT_ENDPOINT", " https://X.SERVICES.AI.AZURE.COM/api/projects/p/ ") + + result, err := Resolve(t.Context(), ResolveOpts{}) + require.NoError(t, err) + assert.Equal(t, "https://x.services.ai.azure.com/api/projects/p", result.Endpoint) + assert.Equal(t, SourceFoundryEnv, result.Source) +} + +func TestResolve_InvalidFlagRejected(t *testing.T) { + isolateFromAzdDaemon(t) + + _, err := Resolve(t.Context(), ResolveOpts{ + FlagValue: "http://not-https.services.ai.azure.com/api/projects/p", + }) + require.Error(t, err) + + var localErr *azdext.LocalError + require.ErrorAs(t, err, &localErr) + assert.Contains(t, localErr.Message, "https") +} + +func TestResolve_InvalidFoundryEnvRejected(t *testing.T) { + isolateFromAzdDaemon(t) + t.Setenv("FOUNDRY_PROJECT_ENDPOINT", "http://bad.services.ai.azure.com/api/projects/p") + + _, err := Resolve(t.Context(), ResolveOpts{}) + require.Error(t, err) + + var localErr *azdext.LocalError + require.ErrorAs(t, err, &localErr) + assert.Contains(t, localErr.Message, "https") +} + +func TestResolve_InvalidAzureAiHostEnvRejected(t *testing.T) { + // An invalid AZURE_AI_PROJECT_ENDPOINT fallback is a hard error, not a + // silent skip to level 5. + isolateFromAzdDaemon(t) + t.Setenv("FOUNDRY_PROJECT_ENDPOINT", "") + t.Setenv("AZURE_AI_PROJECT_ENDPOINT", "http://not-https.services.ai.azure.com/api/projects/p") + + _, err := Resolve(t.Context(), ResolveOpts{}) + require.Error(t, err) + + var localErr *azdext.LocalError + require.ErrorAs(t, err, &localErr) + assert.Contains(t, localErr.Message, "https") +} + +func TestResolve_NothingResolvable(t *testing.T) { + isolateFromAzdDaemon(t) + t.Setenv("FOUNDRY_PROJECT_ENDPOINT", "") + t.Setenv("AZURE_AI_PROJECT_ENDPOINT", "") + + _, err := Resolve(t.Context(), ResolveOpts{}) + require.Error(t, err) + + var localErr *azdext.LocalError + require.ErrorAs(t, err, &localErr) + assert.Equal(t, exterrors.CodeMissingProjectEndpoint, localErr.Code) + assert.Equal(t, azdext.LocalErrorCategoryDependency, localErr.Category) +} + +func TestResolve_CfgFoundButEndpointEmptyFallsThrough(t *testing.T) { + // CfgFound=true with Endpoint="" must not short-circuit; the resolver + // should continue to level 4 (FOUNDRY_PROJECT_ENDPOINT). + t.Setenv("FOUNDRY_PROJECT_ENDPOINT", "https://env.services.ai.azure.com/api/projects/p") + withHostedSources(t, AzdHostedSources{ + CfgState: State{Endpoint: "", SetAt: "2025-01-01T00:00:00Z"}, + CfgFound: true, + }, nil) + + result, err := Resolve(t.Context(), ResolveOpts{}) + require.NoError(t, err) + assert.Equal(t, "https://env.services.ai.azure.com/api/projects/p", result.Endpoint) + assert.Equal(t, SourceFoundryEnv, result.Source) +} + +func TestContainsGRPCCode_NonGRPCErrorReturnsFalse(t *testing.T) { + t.Parallel() + assert.False(t, containsGRPCCode(errors.New("plain"), 0)) + assert.False(t, containsGRPCCode(nil, 0)) +} diff --git a/cli/azd/extensions/azure.ai.dataset/internal/foundry/projectctx/store.go b/cli/azd/extensions/azure.ai.dataset/internal/foundry/projectctx/store.go new file mode 100644 index 00000000000..433e9ab8d41 --- /dev/null +++ b/cli/azd/extensions/azure.ai.dataset/internal/foundry/projectctx/store.go @@ -0,0 +1,40 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package projectctx + +import ( + "context" + "fmt" + + "github.com/azure/azure-dev/cli/azd/pkg/azdext" +) + +// projectContextConfigPath is the read-only UserConfig path for the persisted +// project context owned by azure.ai.agents. The toolboxes extension reads this +// key but never writes it (§ 6 of the design spec). +const projectContextConfigPath = "extensions.ai-agents.project.context" + +// getProjectContext reads the persisted project context from global config. +// Returns (state, true, nil) when present, (zero, false, nil) when absent. +func getProjectContext( + ctx context.Context, azdClient *azdext.AzdClient, +) (State, bool, error) { + ch, err := azdext.NewConfigHelper(azdClient) + if err != nil { + return State{}, false, fmt.Errorf("getProjectContext: %w", err) + } + + var state State + found, err := ch.GetUserJSON(ctx, projectContextConfigPath, &state) + if err != nil { + return State{}, false, + fmt.Errorf("getProjectContext: failed to read config: %w", err) + } + + if !found || state.Endpoint == "" { + return State{}, false, nil + } + + return state, true, nil +} diff --git a/cli/azd/extensions/azure.ai.dataset/internal/foundry/projectctx/types.go b/cli/azd/extensions/azure.ai.dataset/internal/foundry/projectctx/types.go new file mode 100644 index 00000000000..93bf43f3780 --- /dev/null +++ b/cli/azd/extensions/azure.ai.dataset/internal/foundry/projectctx/types.go @@ -0,0 +1,81 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +// Package projectctx encapsulates the Foundry project endpoint cascade and +// validation shared by every Foundry-extension command tree. +// +// This is the toolboxes-extension copy of the agent_context.go / project_endpoint.go / +// project_context_store.go logic in azure.ai.agents (see § 3.2 of the toolbox +// design spec). Semantics match the agents original verbatim; identifiers are +// exported because they cross the package boundary in this layout. +package projectctx + +const ( + // foundryEnvKey is the canonical project-endpoint key. It is read both from + // the active azd environment (level 2) and as a host environment variable + // (level 4). + foundryEnvKey = "FOUNDRY_PROJECT_ENDPOINT" + // azureAiEnvKey is the legacy/sibling project-endpoint key written by + // `azd ai agent init` and `azd add` (Bicep output). It is read as a fallback + // after foundryEnvKey at both the active-azd-env and host-env levels so the + // hosted-agent + toolbox workflow resolves without an extra manual step. + // See https://github.com/Azure/azure-dev/issues/8688. + azureAiEnvKey = "AZURE_AI_PROJECT_ENDPOINT" +) + +// EndpointSource identifies where a resolved project endpoint came from. +type EndpointSource string + +const ( + // SourceFlag means the endpoint came from the --project-endpoint flag. + SourceFlag EndpointSource = "flag" + // SourceAzdEnv means the endpoint came from the active azd environment's + // FOUNDRY_PROJECT_ENDPOINT (or, as a fallback, AZURE_AI_PROJECT_ENDPOINT) value. + SourceAzdEnv EndpointSource = "azdEnv" + // SourceGlobalConfig means the endpoint came from ~/.azd/config.json + // (extensions.ai-agents.project.context.endpoint — owned by azure.ai.agents + // and shared read-only with sibling extensions). + SourceGlobalConfig EndpointSource = "globalConfig" + // SourceFoundryEnv means the endpoint came from the FOUNDRY_PROJECT_ENDPOINT + // (or, as a fallback, AZURE_AI_PROJECT_ENDPOINT) host environment variable. + SourceFoundryEnv EndpointSource = "foundryEnv" +) + +// ResolveOpts controls the 5-level endpoint resolution cascade. +type ResolveOpts struct { + // FlagValue is the value of the --project-endpoint flag (level 1). + // Empty means the flag was not provided. + FlagValue string +} + +// Resolved holds the result of Resolve. +type Resolved struct { + Endpoint string + Source EndpointSource + AzdEnvName string + SetAt string // RFC3339 timestamp; only meaningful when Source == SourceGlobalConfig +} + +// AzdHostedSources holds the values the resolver reads from azd-managed +// sources (active env + ~/.azd/config.json). Returned as a single struct so +// tests can stub the whole lookup via ReadAzdHostedSourcesFunc. +type AzdHostedSources struct { + // EnvValue is the active-azd-env project endpoint: FOUNDRY_PROJECT_ENDPOINT + // if set, otherwise AZURE_AI_PROJECT_ENDPOINT, otherwise "" (not set / no + // active env / no azd client available). + EnvValue string + // EnvName is the active azd env name. Only meaningful when EnvValue != "". + EnvName string + // CfgState is the project context persisted in global config. + CfgState State + // CfgFound indicates whether a non-empty endpoint was found in global config. + CfgFound bool +} + +// State is the JSON shape stored at extensions.ai-agents.project.context in +// ~/.azd/config.json. This key is owned by azure.ai.agents; the toolboxes +// extension reads it but never writes it. +type State struct { + Endpoint string `json:"endpoint"` + SetAt string `json:"setAt"` +} diff --git a/cli/azd/extensions/azure.ai.dataset/internal/foundry/projectctx/validator.go b/cli/azd/extensions/azure.ai.dataset/internal/foundry/projectctx/validator.go new file mode 100644 index 00000000000..47e17e6163d --- /dev/null +++ b/cli/azd/extensions/azure.ai.dataset/internal/foundry/projectctx/validator.go @@ -0,0 +1,113 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package projectctx + +import ( + "fmt" + "net/url" + "strings" + + "azureaidataset/internal/exterrors" +) + +// foundryHostSuffixes is the authoritative list of accepted Foundry host suffixes. +var foundryHostSuffixes = []string{ + ".services.ai.azure.com", +} + +// projectEndpointPathPrefix is the expected path prefix for Foundry project endpoints. +const projectEndpointPathPrefix = "/api/projects/" + +// isFoundryHost reports whether the hostname ends with a recognized Foundry suffix. +func isFoundryHost(hostname string) bool { + h := strings.ToLower(hostname) + for _, suffix := range foundryHostSuffixes { + if strings.HasSuffix(h, suffix) { + return true + } + } + return false +} + +// Validate validates and normalizes a Foundry project endpoint URL. +// +// The URL must be an absolute https:// URL whose host ends with a recognized +// Foundry suffix. Whitespace is trimmed, trailing slashes are stripped, and +// the result is returned in normalized form. +// +// The second return value is true when the path does not look like +// /api/projects/ — callers may use this as a non-fatal warning. +func Validate(raw string) (normalized string, pathWarning bool, err error) { + raw = strings.TrimSpace(raw) + if raw == "" { + return "", false, exterrors.Validation( + exterrors.CodeInvalidParameter, + "project endpoint must not be empty", + "provide a Foundry project endpoint URL "+ + "(e.g. https://.services.ai.azure.com/api/projects/)", + ) + } + + u, parseErr := url.Parse(raw) + if parseErr != nil { + return "", false, exterrors.Validation( + exterrors.CodeInvalidParameter, + fmt.Sprintf("invalid project endpoint URL: %v", parseErr), + "provide a valid https:// Foundry project endpoint URL", + ) + } + + if !strings.EqualFold(u.Scheme, "https") { + return "", false, exterrors.Validation( + exterrors.CodeInvalidParameter, + "project endpoint must use https", + "provide an https:// URL", + ) + } + + host := u.Hostname() + if host == "" || !isFoundryHost(host) { + return "", false, exterrors.Validation( + exterrors.CodeInvalidParameter, + fmt.Sprintf( + "project endpoint host %q is not a recognized Foundry host (*%s)", + host, foundryHostSuffixes[0], + ), + "the host must end with "+foundryHostSuffixes[0], + ) + } + + if u.Port() != "" { + return "", false, exterrors.Validation( + exterrors.CodeInvalidParameter, + fmt.Sprintf("project endpoint host %q must not include a port", u.Host), + "remove the explicit port from the URL", + ) + } + + // Normalize: lowercase host, strip trailing slash. + path := strings.TrimRight(u.EscapedPath(), "/") + normalized = fmt.Sprintf("https://%s%s", strings.ToLower(host), path) + + // Warn when the path does not look like /api/projects/. + if !strings.HasPrefix(path, projectEndpointPathPrefix) || + strings.TrimPrefix(path, projectEndpointPathPrefix) == "" { + pathWarning = true + } + + return normalized, pathWarning, nil +} + +// NoEndpointError returns the structured dependency error used when no project +// endpoint could be resolved from any source. +func NoEndpointError() error { + return exterrors.Dependency( + exterrors.CodeMissingProjectEndpoint, + "no Foundry project endpoint resolved", + "persist a workspace default with `azd ai project set `, "+ + "or set FOUNDRY_PROJECT_ENDPOINT (or AZURE_AI_PROJECT_ENDPOINT) "+ + "in the active azd environment, "+ + "or export FOUNDRY_PROJECT_ENDPOINT (or AZURE_AI_PROJECT_ENDPOINT) in your shell", + ) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/context.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/context.go index 65c3e87181c..e5412a72bd7 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/context.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/context.go @@ -8,9 +8,9 @@ import ( "errors" "fmt" "log" - "os" "strings" + "azureaieval/internal/foundry/projectctx" "azureaieval/internal/pkg/dataset_api" "azureaieval/internal/pkg/eval_api" "azureaieval/internal/project" @@ -53,24 +53,15 @@ func newEvalContext(ctx context.Context, endpointFlag string) (*evalContext, err // from: it is what the cached eval and run ids are read from and // written to. Deriving it only when the endpoint came from azd meant // --project-endpoint silently disabled that cache. - azdEndpoint, envName := lookupEndpointFromAzd(ctx, azdClient) + _, envName := lookupEndpointFromAzd(ctx, azdClient) ec.envName = envName - if endpointFlag != "" { - ec.endpoint = endpointFlag - } else { - ec.endpoint = azdEndpoint - } - if ec.endpoint == "" { - ec.endpoint = os.Getenv(projectEndpointEnvKey) - } - if ec.endpoint == "" { - return nil, fmt.Errorf( - "no Foundry project endpoint found; pass --project-endpoint or set %s "+ - "in the azd environment (azd env set %s )", - projectEndpointEnvKey, projectEndpointEnvKey) + resolved, err := projectctx.Resolve(ctx, projectctx.ResolveOpts{FlagValue: endpointFlag}) + if err != nil { + return nil, err } - ec.endpoint = strings.TrimSuffix(ec.endpoint, "/") + ec.endpoint = strings.TrimSuffix(resolved.Endpoint, "/") + log.Printf("[endpoint] resolved from %s", resolved.Source) cred, err := azidentity.NewAzureDeveloperCLICredential( &azidentity.AzureDeveloperCLICredentialOptions{}, diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/surface_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/surface_test.go index 993044751ed..2a570ca1614 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/surface_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/surface_test.go @@ -407,6 +407,14 @@ func TestSuggestedFlagsExist(t *testing.T) { require.NoError(t, err) } +// siblingNamespaces are the other Foundry extensions this one points users at. +// Listed rather than wildcarded so a typo in a namespace still fails. +var siblingNamespaces = map[string]bool{ + "project": true, // `azd ai project set` owns the shared endpoint context + "dataset": true, // where the dataset commands go once they move + "agent": true, +} + // find resolves a command path, failing the test when it does not exist. func find(t *testing.T, path string) *cobra.Command { t.Helper() @@ -457,14 +465,23 @@ func TestSuggestedCommandsExist(t *testing.T) { } for _, m := range pattern.FindAllStringSubmatch(line, -1) { words := strings.Fields(m[1]) + if len(words) == 0 { + continue + } + + // A suggestion under a sibling's namespace is that extension's + // contract and cannot be resolved from here. Only the ones this + // extension actually points at are allowed, so a typo still + // fails rather than passing as "probably somebody else's". + if siblingNamespaces[words[0]] { + continue + } // `ai.eval` is this extension's namespace, so it is the only - // thing under `azd ai` that resolves here. Another namespace is - // a command this extension cannot suggest, whether or not some - // future extension serves it. - if len(words) == 0 || words[0] != "eval" { - t.Errorf("%s suggests `azd ai %s`, which is not this extension's "+ - "namespace; commands here are `azd ai eval ...`", path, m[1]) + // other thing under `azd ai` that can resolve. + if words[0] != "eval" { + t.Errorf("%s suggests `azd ai %s`, which is neither this "+ + "extension's namespace nor a sibling it knows about", path, m[1]) continue } words = words[1:] diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/exterrors/codes.go b/cli/azd/extensions/azure.ai.evaluations/internal/exterrors/codes.go new file mode 100644 index 00000000000..58fc300148c --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/exterrors/codes.go @@ -0,0 +1,71 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package exterrors + +// Error codes for user cancellation. +const ( + CodeCancelled = "cancelled" +) + +// Error codes for validation failures (user input, manifests, flags). +const ( + CodeInvalidParameter = "invalid_parameter" + CodeInvalidPositionalArg = "invalid_positional_arg" +) + +// Error codes for dependency failures (missing resources, services, env values). +const ( + CodeAzdClientFailed = "azd_client_failed" + CodeMissingProjectEndpoint = "missing_project_endpoint" +) + +// Error codes for auth failures. +const ( + CodeNotLoggedIn = "not_logged_in" + CodeLoginExpired = "login_expired" + CodeAuthFailed = "auth_failed" +) + +// Error codes for toolbox operations. +const ( + CodeToolboxNotFound = "toolbox_not_found" + CodeToolboxVersionNotFound = "toolbox_version_not_found" + CodeInvalidToolboxName = "invalid_toolbox_name" + CodeMissingUpdateField = "missing_update_field" + CodeDefaultVersionDelete = "default_version_delete" + CodeOnlyVersionDelete = "only_version_delete" + CodeMissingForceFlag = "missing_force_flag" + CodeUnsupportedConnectionCategory = "unsupported_connection_category" + CodeMissingIndex = "missing_index" + CodeUnsupportedIndexFlag = "unsupported_index_flag" + CodeMissingInstanceName = "missing_instance_name" + CodeUnsupportedInstanceNameFlag = "unsupported_instance_name_flag" + CodeInvalidSkillName = "invalid_skill_name" + CodeInvalidSkillSpec = "invalid_skill_spec" + CodeDuplicateSkill = "duplicate_skill" + CodeSkillNotInToolbox = "skill_not_in_toolbox" + CodeSkillAlreadyAttached = "skill_already_attached" + CodeDuplicateConnection = "duplicate_connection" + CodeDuplicateToolName = "duplicate_tool_name" + CodeMissingToolType = "missing_tool_type" + CodeConnectionNotFound = "connection_not_found" + CodeConnectionNotInToolbox = "connection_not_in_toolbox" + CodeConnectionMissingTarget = "connection_missing_target" + CodeLastToolRemoval = "last_tool_removal" + CodePendingToolboxStoreFailed = "pending_toolbox_store_failed" +) + +// Operation names for [ServiceFromAzure] errors. +// These are prefixed to the Azure error code (e.g., "get_toolbox.NotFound"). +const ( + OpCreateToolboxVersion = "create_toolbox_version" + OpGetToolbox = "get_toolbox" + OpDeleteToolbox = "delete_toolbox" + OpDeleteToolboxVersion = "delete_toolbox_version" + OpSetDefaultVersion = "set_default_version" + OpListToolboxes = "list_toolboxes" + OpGetToolboxVersion = "get_toolbox_version" + OpListToolboxVersions = "list_toolbox_versions" + OpResolveProjectConnection = "resolve_project_connection" +) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/exterrors/errors.go b/cli/azd/extensions/azure.ai.evaluations/internal/exterrors/errors.go new file mode 100644 index 00000000000..4e3c02b6731 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/exterrors/errors.go @@ -0,0 +1,155 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +// Package exterrors provides structured error helpers for the azure.ai.toolboxes +// extension. +// +// Use plain Go errors until the current code can confidently choose a final +// category, code, and suggestion. At that point, create a structured error with +// one of the helpers in this package or with [ServiceFromAzure] for Azure SDK +// failures. +// +// Once an error is structured, usually return it unchanged. Avoid wrapping a +// structured error with [fmt.Errorf] and %w for extra context: azd serializes +// the structured error's own message and metadata, not the outer wrapper text. +package exterrors + +import ( + "context" + "errors" + "fmt" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore" + "github.com/azure/azure-dev/cli/azd/pkg/azdext" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +// --------------------------------------------------------------------------- +// Structured error factories +// --------------------------------------------------------------------------- + +// Validation returns a validation [azdext.LocalError] for user input / flag errors. +func Validation(code, message, suggestion string) error { + return &azdext.LocalError{ + Message: message, + Code: code, + Category: azdext.LocalErrorCategoryValidation, + Suggestion: suggestion, + } +} + +// Dependency returns a dependency [azdext.LocalError] for missing resources or services. +func Dependency(code, message, suggestion string) error { + return &azdext.LocalError{ + Message: message, + Code: code, + Category: azdext.LocalErrorCategoryDependency, + Suggestion: suggestion, + } +} + +// Auth returns an auth [azdext.LocalError] for authentication/authorization failures. +func Auth(code, message, suggestion string) error { + return &azdext.LocalError{ + Message: message, + Code: code, + Category: azdext.LocalErrorCategoryAuth, + Suggestion: suggestion, + } +} + +// User returns a user-action [azdext.LocalError] (e.g. cancellation). No suggestion. +func User(code, message string) error { + return &azdext.LocalError{ + Message: message, + Code: code, + Category: azdext.LocalErrorCategoryUser, + } +} + +// Internal returns an internal [azdext.LocalError] for unexpected extension failures. +func Internal(code, message string) error { + return &azdext.LocalError{ + Message: message, + Code: code, + Category: azdext.LocalErrorCategoryInternal, + } +} + +// Cancelled returns a user cancellation error. +func Cancelled(message string) error { + return User(CodeCancelled, message) +} + +// --------------------------------------------------------------------------- +// Azure error converters +// --------------------------------------------------------------------------- + +// ServiceFromAzure wraps an [azcore.ResponseError] into an [azdext.ServiceError] +// with operation context. If the error is not an azcore.ResponseError, it +// returns a generic internal [azdext.LocalError]. +func ServiceFromAzure(err error, operation string) error { + var respErr *azcore.ResponseError + if errors.As(err, &respErr) { + serviceName := "" + if respErr.RawResponse != nil && respErr.RawResponse.Request != nil { + serviceName = respErr.RawResponse.Request.Host + } + code := respErr.ErrorCode + if code == "" { + code = fmt.Sprintf("%d", respErr.StatusCode) + } + return &azdext.ServiceError{ + Message: fmt.Sprintf("%s: %s", operation, respErr.Error()), + ErrorCode: fmt.Sprintf("%s.%s", operation, code), + StatusCode: respErr.StatusCode, + ServiceName: serviceName, + } + } + if IsCancellation(err) { + return Cancelled(fmt.Sprintf("%s was cancelled", operation)) + } + return Internal(operation, fmt.Sprintf("%s: %s", operation, err.Error())) +} + +// FromPrompt wraps a gRPC error from an azd host Prompt call into a structured +// error. Auth errors (Unauthenticated) are classified as Auth errors with a +// re-auth suggestion; cancellations as User cancellations; other errors are +// returned wrapped with the provided context message. +func FromPrompt(err error, contextMsg string) error { + if err == nil { + return nil + } + + if IsCancellation(err) { + return Cancelled(contextMsg) + } + + st, ok := status.FromError(err) + if ok && st.Code() == codes.Unauthenticated { + return Auth( + CodeAuthFailed, + fmt.Sprintf("%s: %s", contextMsg, st.Message()), + "run `azd auth login` to authenticate", + ) + } + + return fmt.Errorf("%s: %w", contextMsg, err) +} + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +// IsCancellation reports whether err represents user cancellation +// ([context.Canceled] or gRPC [codes.Canceled]). +func IsCancellation(err error) bool { + if errors.Is(err, context.Canceled) { + return true + } + if st, ok := status.FromError(err); ok && st.Code() == codes.Canceled { + return true + } + return false +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/foundry/projectctx/resolver.go b/cli/azd/extensions/azure.ai.evaluations/internal/foundry/projectctx/resolver.go new file mode 100644 index 00000000000..3c6c3129a50 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/foundry/projectctx/resolver.go @@ -0,0 +1,152 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package projectctx + +import ( + "context" + "errors" + "os" + + "github.com/azure/azure-dev/cli/azd/pkg/azdext" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +// ReadAzdHostedSourcesFunc is a package-level seam so tests can stub the +// daemon-backed lookup without spinning up a real azd gRPC server. +var ReadAzdHostedSourcesFunc = readAzdHostedSources + +// readAzdHostedSources dials the azd daemon (if reachable) and reads both the +// active environment's project endpoint and the global-config project context +// in a single client lifetime. The active-env read prefers +// FOUNDRY_PROJECT_ENDPOINT and falls back to AZURE_AI_PROJECT_ENDPOINT (the key +// `azd ai agent init` / `azd add` persist). Errors talking to the daemon are +// returned only for non-Unavailable cases on the config read — Unavailable is +// treated as "no daemon" and the caller falls through to subsequent levels. +func readAzdHostedSources(ctx context.Context) (AzdHostedSources, error) { + var out AzdHostedSources + + azdClient, err := azdext.NewAzdClient() + if err != nil { + // No azd client at all => no hosted sources, not an error. + return out, nil + } + defer azdClient.Close() + + if envResp, err := azdClient.Environment().GetCurrent( + ctx, &azdext.EmptyRequest{}, + ); err == nil { + for _, key := range []string{foundryEnvKey, azureAiEnvKey} { + envVal, valErr := azdClient.Environment().GetValue(ctx, &azdext.GetEnvRequest{ + EnvName: envResp.Environment.Name, + Key: key, + }) + if valErr == nil && envVal.Value != "" { + out.EnvValue = envVal.Value + out.EnvName = envResp.Environment.Name + break + } + } + } + + state, found, cfgErr := getProjectContext(ctx, azdClient) + if cfgErr != nil { + // A gRPC Unavailable code means the azd daemon is not reachable; + // treat it the same as azdClient creation failing and fall through. + // Any other error (e.g. parse failure) is a hard error. + if !containsGRPCCode(cfgErr, codes.Unavailable) { + return out, cfgErr + } + } else { + out.CfgState = state + out.CfgFound = found + } + + return out, nil +} + +// containsGRPCCode walks the error chain looking for a gRPC status with the +// specified code. fmt.Errorf("%w", ...) wraps errors without forwarding the +// GRPCStatus() method, so we must unwrap manually. +// +// Note: only follows errors.Unwrap chains; errors.Join multi-wraps are not traversed. +func containsGRPCCode(err error, code codes.Code) bool { + for ; err != nil; err = errors.Unwrap(err) { + if st, ok := status.FromError(err); ok && st.Code() == code { + return true + } + } + return false +} + +// Resolve resolves a Foundry project endpoint using the 5-level cascade: +// +// 1. --project-endpoint flag +// 2. Active azd env value (FOUNDRY_PROJECT_ENDPOINT, then AZURE_AI_PROJECT_ENDPOINT) +// 3. Global config: extensions.ai-agents.project.context.endpoint (read-only; +// owned by azure.ai.agents) +// 4. Host environment variable (FOUNDRY_PROJECT_ENDPOINT, then AZURE_AI_PROJECT_ENDPOINT) +// 5. Structured error with actionable suggestion +// +// Invalid values at any level produce a hard validation error (no silent fallback). +func Resolve(ctx context.Context, opts ResolveOpts) (*Resolved, error) { + // Level 1: explicit flag. + if opts.FlagValue != "" { + normalized, _, err := Validate(opts.FlagValue) + if err != nil { + return nil, err + } + return &Resolved{Endpoint: normalized, Source: SourceFlag}, nil + } + + // Levels 2 + 3: azd-hosted sources (active env, then global config). + sources, err := ReadAzdHostedSourcesFunc(ctx) + if err != nil { + return nil, err + } + + // Level 2: active azd environment's FOUNDRY_PROJECT_ENDPOINT (with the + // AZURE_AI_PROJECT_ENDPOINT fallback applied in readAzdHostedSources). + if sources.EnvValue != "" { + normalized, _, err := Validate(sources.EnvValue) + if err != nil { + return nil, err + } + return &Resolved{ + Endpoint: normalized, + Source: SourceAzdEnv, + AzdEnvName: sources.EnvName, + }, nil + } + + // Level 3: global config (~/.azd/config.json). + if sources.CfgFound && sources.CfgState.Endpoint != "" { + normalized, _, err := Validate(sources.CfgState.Endpoint) + if err != nil { + return nil, err + } + return &Resolved{ + Endpoint: normalized, + Source: SourceGlobalConfig, + SetAt: sources.CfgState.SetAt, + }, nil + } + + // Level 4: host environment variable (FOUNDRY_PROJECT_ENDPOINT, then the + // AZURE_AI_PROJECT_ENDPOINT fallback). + for _, key := range []string{foundryEnvKey, azureAiEnvKey} { + envVal := os.Getenv(key) + if envVal == "" { + continue + } + normalized, _, err := Validate(envVal) + if err != nil { + return nil, err + } + return &Resolved{Endpoint: normalized, Source: SourceFoundryEnv}, nil + } + + // Level 5: structured error. + return nil, NoEndpointError() +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/foundry/projectctx/resolver_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/foundry/projectctx/resolver_test.go new file mode 100644 index 00000000000..f5300db6fc3 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/foundry/projectctx/resolver_test.go @@ -0,0 +1,259 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package projectctx + +import ( + "context" + "errors" + "testing" + + "azureaieval/internal/exterrors" + + "github.com/azure/azure-dev/cli/azd/pkg/azdext" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// withHostedSources installs a stub for ReadAzdHostedSourcesFunc for the +// duration of the test and restores the production value on cleanup. Tests +// using this MUST NOT run in parallel because the seam is a package-level var. +func withHostedSources(t *testing.T, sources AzdHostedSources, err error) { + t.Helper() + orig := ReadAzdHostedSourcesFunc + ReadAzdHostedSourcesFunc = func(context.Context) (AzdHostedSources, error) { + return sources, err + } + t.Cleanup(func() { ReadAzdHostedSourcesFunc = orig }) +} + +// isolateFromAzdDaemon installs an empty hosted-sources stub and clears +// AZD_SERVER so any code path that bypasses the seam cannot reach a real +// daemon. After calling this, the resolver only sees the flag and the +// FOUNDRY_PROJECT_ENDPOINT / AZURE_AI_PROJECT_ENDPOINT host env vars. +func isolateFromAzdDaemon(t *testing.T) { + t.Helper() + t.Setenv("AZD_SERVER", "") + withHostedSources(t, AzdHostedSources{}, nil) +} + +func TestResolve_FlagWins(t *testing.T) { + // Even with FOUNDRY_PROJECT_ENDPOINT and azd-hosted sources set, the flag wins. + t.Setenv("FOUNDRY_PROJECT_ENDPOINT", "https://env.services.ai.azure.com/api/projects/env-proj") + withHostedSources(t, AzdHostedSources{ + EnvValue: "https://azdenv.services.ai.azure.com/api/projects/p", + EnvName: "dev", + }, nil) + + result, err := Resolve(t.Context(), ResolveOpts{ + FlagValue: "https://flag.services.ai.azure.com/api/projects/flag-proj", + }) + require.NoError(t, err) + assert.Equal(t, "https://flag.services.ai.azure.com/api/projects/flag-proj", result.Endpoint) + assert.Equal(t, SourceFlag, result.Source) +} + +func TestResolve_AzdEnvWinsOverConfigAndFoundryEnv(t *testing.T) { + // EnvValue here stands in for whichever active-env key readAzdHostedSources + // resolved (FOUNDRY_PROJECT_ENDPOINT, or the AZURE_AI_PROJECT_ENDPOINT + // fallback); either way level 2 wins over global config and the host env. + t.Setenv("FOUNDRY_PROJECT_ENDPOINT", "https://foundry.services.ai.azure.com/api/projects/p") + withHostedSources(t, AzdHostedSources{ + EnvValue: " HTTPS://Azdenv.Services.AI.Azure.com/api/projects/p/ ", + EnvName: "dev", + CfgState: State{ + Endpoint: "https://cfg.services.ai.azure.com/api/projects/p", + SetAt: "2025-01-01T00:00:00Z", + }, + CfgFound: true, + }, nil) + + result, err := Resolve(t.Context(), ResolveOpts{}) + require.NoError(t, err) + assert.Equal(t, "https://azdenv.services.ai.azure.com/api/projects/p", result.Endpoint) + assert.Equal(t, SourceAzdEnv, result.Source) + assert.Equal(t, "dev", result.AzdEnvName) +} + +func TestResolve_AzdEnvInvalidIsHardError(t *testing.T) { + // Level 2 invalid values are hard errors (no silent fallback to lower levels). + t.Setenv("FOUNDRY_PROJECT_ENDPOINT", "https://foundry.services.ai.azure.com/api/projects/p") + withHostedSources(t, AzdHostedSources{ + EnvValue: "http://not-https.services.ai.azure.com/api/projects/p", + EnvName: "dev", + }, nil) + + _, err := Resolve(t.Context(), ResolveOpts{}) + require.Error(t, err) + + var localErr *azdext.LocalError + require.ErrorAs(t, err, &localErr) + assert.Contains(t, localErr.Message, "https") +} + +func TestResolve_GlobalConfigWinsOverFoundryEnv(t *testing.T) { + t.Setenv("FOUNDRY_PROJECT_ENDPOINT", "https://foundry.services.ai.azure.com/api/projects/p") + withHostedSources(t, AzdHostedSources{ + CfgState: State{ + Endpoint: " HTTPS://Cfg.Services.AI.Azure.com/api/projects/p/ ", + SetAt: "2025-01-02T03:04:05Z", + }, + CfgFound: true, + }, nil) + + result, err := Resolve(t.Context(), ResolveOpts{}) + require.NoError(t, err) + assert.Equal(t, "https://cfg.services.ai.azure.com/api/projects/p", result.Endpoint) + assert.Equal(t, SourceGlobalConfig, result.Source) + assert.Equal(t, "2025-01-02T03:04:05Z", result.SetAt) +} + +func TestResolve_GlobalConfigInvalidIsHardError(t *testing.T) { + t.Setenv("FOUNDRY_PROJECT_ENDPOINT", "https://foundry.services.ai.azure.com/api/projects/p") + withHostedSources(t, AzdHostedSources{ + CfgState: State{ + Endpoint: "http://not-https.services.ai.azure.com/api/projects/p", + SetAt: "2025-01-02T03:04:05Z", + }, + CfgFound: true, + }, nil) + + _, err := Resolve(t.Context(), ResolveOpts{}) + require.Error(t, err) + + var localErr *azdext.LocalError + require.ErrorAs(t, err, &localErr) + assert.Contains(t, localErr.Message, "https") +} + +func TestResolve_HostedSourcesErrorPropagates(t *testing.T) { + // Non-recoverable errors from the hosted-source lookup must be surfaced + // and must not silently fall through to level 4. + t.Setenv("FOUNDRY_PROJECT_ENDPOINT", "https://foundry.services.ai.azure.com/api/projects/p") + sentinel := errors.New("boom") + withHostedSources(t, AzdHostedSources{}, sentinel) + + _, err := Resolve(t.Context(), ResolveOpts{}) + require.ErrorIs(t, err, sentinel) +} + +func TestResolve_FoundryEnvFallback(t *testing.T) { + isolateFromAzdDaemon(t) + t.Setenv("FOUNDRY_PROJECT_ENDPOINT", "https://env.services.ai.azure.com/api/projects/env-proj") + + result, err := Resolve(t.Context(), ResolveOpts{}) + require.NoError(t, err) + assert.Equal(t, "https://env.services.ai.azure.com/api/projects/env-proj", result.Endpoint) + assert.Equal(t, SourceFoundryEnv, result.Source) +} + +func TestResolve_AzureAiHostEnvFallback(t *testing.T) { + // When FOUNDRY_PROJECT_ENDPOINT is unset, the resolver falls back to the + // AZURE_AI_PROJECT_ENDPOINT host env var (the key azd ai agent init / azd + // add persist). See https://github.com/Azure/azure-dev/issues/8688. + isolateFromAzdDaemon(t) + t.Setenv("FOUNDRY_PROJECT_ENDPOINT", "") + t.Setenv("AZURE_AI_PROJECT_ENDPOINT", "https://azureai.services.ai.azure.com/api/projects/p") + + result, err := Resolve(t.Context(), ResolveOpts{}) + require.NoError(t, err) + assert.Equal(t, "https://azureai.services.ai.azure.com/api/projects/p", result.Endpoint) + assert.Equal(t, SourceFoundryEnv, result.Source) +} + +func TestResolve_FoundryHostEnvWinsOverAzureAi(t *testing.T) { + // With both host env vars set, FOUNDRY_PROJECT_ENDPOINT takes precedence. + isolateFromAzdDaemon(t) + t.Setenv("FOUNDRY_PROJECT_ENDPOINT", "https://foundry.services.ai.azure.com/api/projects/f") + t.Setenv("AZURE_AI_PROJECT_ENDPOINT", "https://azureai.services.ai.azure.com/api/projects/a") + + result, err := Resolve(t.Context(), ResolveOpts{}) + require.NoError(t, err) + assert.Equal(t, "https://foundry.services.ai.azure.com/api/projects/f", result.Endpoint) + assert.Equal(t, SourceFoundryEnv, result.Source) +} + +func TestResolve_FoundryEnvNormalized(t *testing.T) { + isolateFromAzdDaemon(t) + t.Setenv("FOUNDRY_PROJECT_ENDPOINT", " https://X.SERVICES.AI.AZURE.COM/api/projects/p/ ") + + result, err := Resolve(t.Context(), ResolveOpts{}) + require.NoError(t, err) + assert.Equal(t, "https://x.services.ai.azure.com/api/projects/p", result.Endpoint) + assert.Equal(t, SourceFoundryEnv, result.Source) +} + +func TestResolve_InvalidFlagRejected(t *testing.T) { + isolateFromAzdDaemon(t) + + _, err := Resolve(t.Context(), ResolveOpts{ + FlagValue: "http://not-https.services.ai.azure.com/api/projects/p", + }) + require.Error(t, err) + + var localErr *azdext.LocalError + require.ErrorAs(t, err, &localErr) + assert.Contains(t, localErr.Message, "https") +} + +func TestResolve_InvalidFoundryEnvRejected(t *testing.T) { + isolateFromAzdDaemon(t) + t.Setenv("FOUNDRY_PROJECT_ENDPOINT", "http://bad.services.ai.azure.com/api/projects/p") + + _, err := Resolve(t.Context(), ResolveOpts{}) + require.Error(t, err) + + var localErr *azdext.LocalError + require.ErrorAs(t, err, &localErr) + assert.Contains(t, localErr.Message, "https") +} + +func TestResolve_InvalidAzureAiHostEnvRejected(t *testing.T) { + // An invalid AZURE_AI_PROJECT_ENDPOINT fallback is a hard error, not a + // silent skip to level 5. + isolateFromAzdDaemon(t) + t.Setenv("FOUNDRY_PROJECT_ENDPOINT", "") + t.Setenv("AZURE_AI_PROJECT_ENDPOINT", "http://not-https.services.ai.azure.com/api/projects/p") + + _, err := Resolve(t.Context(), ResolveOpts{}) + require.Error(t, err) + + var localErr *azdext.LocalError + require.ErrorAs(t, err, &localErr) + assert.Contains(t, localErr.Message, "https") +} + +func TestResolve_NothingResolvable(t *testing.T) { + isolateFromAzdDaemon(t) + t.Setenv("FOUNDRY_PROJECT_ENDPOINT", "") + t.Setenv("AZURE_AI_PROJECT_ENDPOINT", "") + + _, err := Resolve(t.Context(), ResolveOpts{}) + require.Error(t, err) + + var localErr *azdext.LocalError + require.ErrorAs(t, err, &localErr) + assert.Equal(t, exterrors.CodeMissingProjectEndpoint, localErr.Code) + assert.Equal(t, azdext.LocalErrorCategoryDependency, localErr.Category) +} + +func TestResolve_CfgFoundButEndpointEmptyFallsThrough(t *testing.T) { + // CfgFound=true with Endpoint="" must not short-circuit; the resolver + // should continue to level 4 (FOUNDRY_PROJECT_ENDPOINT). + t.Setenv("FOUNDRY_PROJECT_ENDPOINT", "https://env.services.ai.azure.com/api/projects/p") + withHostedSources(t, AzdHostedSources{ + CfgState: State{Endpoint: "", SetAt: "2025-01-01T00:00:00Z"}, + CfgFound: true, + }, nil) + + result, err := Resolve(t.Context(), ResolveOpts{}) + require.NoError(t, err) + assert.Equal(t, "https://env.services.ai.azure.com/api/projects/p", result.Endpoint) + assert.Equal(t, SourceFoundryEnv, result.Source) +} + +func TestContainsGRPCCode_NonGRPCErrorReturnsFalse(t *testing.T) { + t.Parallel() + assert.False(t, containsGRPCCode(errors.New("plain"), 0)) + assert.False(t, containsGRPCCode(nil, 0)) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/foundry/projectctx/store.go b/cli/azd/extensions/azure.ai.evaluations/internal/foundry/projectctx/store.go new file mode 100644 index 00000000000..433e9ab8d41 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/foundry/projectctx/store.go @@ -0,0 +1,40 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package projectctx + +import ( + "context" + "fmt" + + "github.com/azure/azure-dev/cli/azd/pkg/azdext" +) + +// projectContextConfigPath is the read-only UserConfig path for the persisted +// project context owned by azure.ai.agents. The toolboxes extension reads this +// key but never writes it (§ 6 of the design spec). +const projectContextConfigPath = "extensions.ai-agents.project.context" + +// getProjectContext reads the persisted project context from global config. +// Returns (state, true, nil) when present, (zero, false, nil) when absent. +func getProjectContext( + ctx context.Context, azdClient *azdext.AzdClient, +) (State, bool, error) { + ch, err := azdext.NewConfigHelper(azdClient) + if err != nil { + return State{}, false, fmt.Errorf("getProjectContext: %w", err) + } + + var state State + found, err := ch.GetUserJSON(ctx, projectContextConfigPath, &state) + if err != nil { + return State{}, false, + fmt.Errorf("getProjectContext: failed to read config: %w", err) + } + + if !found || state.Endpoint == "" { + return State{}, false, nil + } + + return state, true, nil +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/foundry/projectctx/types.go b/cli/azd/extensions/azure.ai.evaluations/internal/foundry/projectctx/types.go new file mode 100644 index 00000000000..93bf43f3780 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/foundry/projectctx/types.go @@ -0,0 +1,81 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +// Package projectctx encapsulates the Foundry project endpoint cascade and +// validation shared by every Foundry-extension command tree. +// +// This is the toolboxes-extension copy of the agent_context.go / project_endpoint.go / +// project_context_store.go logic in azure.ai.agents (see § 3.2 of the toolbox +// design spec). Semantics match the agents original verbatim; identifiers are +// exported because they cross the package boundary in this layout. +package projectctx + +const ( + // foundryEnvKey is the canonical project-endpoint key. It is read both from + // the active azd environment (level 2) and as a host environment variable + // (level 4). + foundryEnvKey = "FOUNDRY_PROJECT_ENDPOINT" + // azureAiEnvKey is the legacy/sibling project-endpoint key written by + // `azd ai agent init` and `azd add` (Bicep output). It is read as a fallback + // after foundryEnvKey at both the active-azd-env and host-env levels so the + // hosted-agent + toolbox workflow resolves without an extra manual step. + // See https://github.com/Azure/azure-dev/issues/8688. + azureAiEnvKey = "AZURE_AI_PROJECT_ENDPOINT" +) + +// EndpointSource identifies where a resolved project endpoint came from. +type EndpointSource string + +const ( + // SourceFlag means the endpoint came from the --project-endpoint flag. + SourceFlag EndpointSource = "flag" + // SourceAzdEnv means the endpoint came from the active azd environment's + // FOUNDRY_PROJECT_ENDPOINT (or, as a fallback, AZURE_AI_PROJECT_ENDPOINT) value. + SourceAzdEnv EndpointSource = "azdEnv" + // SourceGlobalConfig means the endpoint came from ~/.azd/config.json + // (extensions.ai-agents.project.context.endpoint — owned by azure.ai.agents + // and shared read-only with sibling extensions). + SourceGlobalConfig EndpointSource = "globalConfig" + // SourceFoundryEnv means the endpoint came from the FOUNDRY_PROJECT_ENDPOINT + // (or, as a fallback, AZURE_AI_PROJECT_ENDPOINT) host environment variable. + SourceFoundryEnv EndpointSource = "foundryEnv" +) + +// ResolveOpts controls the 5-level endpoint resolution cascade. +type ResolveOpts struct { + // FlagValue is the value of the --project-endpoint flag (level 1). + // Empty means the flag was not provided. + FlagValue string +} + +// Resolved holds the result of Resolve. +type Resolved struct { + Endpoint string + Source EndpointSource + AzdEnvName string + SetAt string // RFC3339 timestamp; only meaningful when Source == SourceGlobalConfig +} + +// AzdHostedSources holds the values the resolver reads from azd-managed +// sources (active env + ~/.azd/config.json). Returned as a single struct so +// tests can stub the whole lookup via ReadAzdHostedSourcesFunc. +type AzdHostedSources struct { + // EnvValue is the active-azd-env project endpoint: FOUNDRY_PROJECT_ENDPOINT + // if set, otherwise AZURE_AI_PROJECT_ENDPOINT, otherwise "" (not set / no + // active env / no azd client available). + EnvValue string + // EnvName is the active azd env name. Only meaningful when EnvValue != "". + EnvName string + // CfgState is the project context persisted in global config. + CfgState State + // CfgFound indicates whether a non-empty endpoint was found in global config. + CfgFound bool +} + +// State is the JSON shape stored at extensions.ai-agents.project.context in +// ~/.azd/config.json. This key is owned by azure.ai.agents; the toolboxes +// extension reads it but never writes it. +type State struct { + Endpoint string `json:"endpoint"` + SetAt string `json:"setAt"` +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/foundry/projectctx/validator.go b/cli/azd/extensions/azure.ai.evaluations/internal/foundry/projectctx/validator.go new file mode 100644 index 00000000000..bf014dc99c4 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/foundry/projectctx/validator.go @@ -0,0 +1,113 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package projectctx + +import ( + "fmt" + "net/url" + "strings" + + "azureaieval/internal/exterrors" +) + +// foundryHostSuffixes is the authoritative list of accepted Foundry host suffixes. +var foundryHostSuffixes = []string{ + ".services.ai.azure.com", +} + +// projectEndpointPathPrefix is the expected path prefix for Foundry project endpoints. +const projectEndpointPathPrefix = "/api/projects/" + +// isFoundryHost reports whether the hostname ends with a recognized Foundry suffix. +func isFoundryHost(hostname string) bool { + h := strings.ToLower(hostname) + for _, suffix := range foundryHostSuffixes { + if strings.HasSuffix(h, suffix) { + return true + } + } + return false +} + +// Validate validates and normalizes a Foundry project endpoint URL. +// +// The URL must be an absolute https:// URL whose host ends with a recognized +// Foundry suffix. Whitespace is trimmed, trailing slashes are stripped, and +// the result is returned in normalized form. +// +// The second return value is true when the path does not look like +// /api/projects/ — callers may use this as a non-fatal warning. +func Validate(raw string) (normalized string, pathWarning bool, err error) { + raw = strings.TrimSpace(raw) + if raw == "" { + return "", false, exterrors.Validation( + exterrors.CodeInvalidParameter, + "project endpoint must not be empty", + "provide a Foundry project endpoint URL "+ + "(e.g. https://.services.ai.azure.com/api/projects/)", + ) + } + + u, parseErr := url.Parse(raw) + if parseErr != nil { + return "", false, exterrors.Validation( + exterrors.CodeInvalidParameter, + fmt.Sprintf("invalid project endpoint URL: %v", parseErr), + "provide a valid https:// Foundry project endpoint URL", + ) + } + + if !strings.EqualFold(u.Scheme, "https") { + return "", false, exterrors.Validation( + exterrors.CodeInvalidParameter, + "project endpoint must use https", + "provide an https:// URL", + ) + } + + host := u.Hostname() + if host == "" || !isFoundryHost(host) { + return "", false, exterrors.Validation( + exterrors.CodeInvalidParameter, + fmt.Sprintf( + "project endpoint host %q is not a recognized Foundry host (*%s)", + host, foundryHostSuffixes[0], + ), + "the host must end with "+foundryHostSuffixes[0], + ) + } + + if u.Port() != "" { + return "", false, exterrors.Validation( + exterrors.CodeInvalidParameter, + fmt.Sprintf("project endpoint host %q must not include a port", u.Host), + "remove the explicit port from the URL", + ) + } + + // Normalize: lowercase host, strip trailing slash. + path := strings.TrimRight(u.EscapedPath(), "/") + normalized = fmt.Sprintf("https://%s%s", strings.ToLower(host), path) + + // Warn when the path does not look like /api/projects/. + if !strings.HasPrefix(path, projectEndpointPathPrefix) || + strings.TrimPrefix(path, projectEndpointPathPrefix) == "" { + pathWarning = true + } + + return normalized, pathWarning, nil +} + +// NoEndpointError returns the structured dependency error used when no project +// endpoint could be resolved from any source. +func NoEndpointError() error { + return exterrors.Dependency( + exterrors.CodeMissingProjectEndpoint, + "no Foundry project endpoint resolved", + "persist a workspace default with `azd ai project set `, "+ + "or set FOUNDRY_PROJECT_ENDPOINT (or AZURE_AI_PROJECT_ENDPOINT) "+ + "in the active azd environment, "+ + "or export FOUNDRY_PROJECT_ENDPOINT (or AZURE_AI_PROJECT_ENDPOINT) in your shell", + ) +} From 45635ba54b5b061ccc5e56bded16dd7cdc1affbb Mon Sep 17 00:00:00 2001 From: mohessie Date: Thu, 6 Aug 2026 05:27:15 +0300 Subject: [PATCH 120/320] Test the wire, where a wrong URL costs a round trip to find out The clients were the largest untested surface in both extensions, and the part where being wrong compiles cleanly and fails only against a real service. A recording server pins what actually goes out. The quirks are the point. Cancel takes a colon, not a path segment -- `{id}/cancel` is a 404 while `{id}:cancel` reaches the action -- and it sends an empty JSON object because without a content type the route answers 415. Both were comments; now they fail if changed. Delete answers 204 with no body, which the pipeline has to accept or every successful delete reports as an error. The job routes answer with `data` where the dataset routes answer with `value`, so reading the wrong key turns a full response into an empty list. The OpenAI-compatible eval routes send no api-version at all, which is as easy to get wrong in the other direction. Also pinned: ids are escaped into the path, so a name cannot climb out of its route; a 404 stays recognizable, because commands branch on it to name the thing that is missing; and a 5xx stays transient, because the poller branches on that to keep waiting. The test pipelines disable SDK retries. A test answering 502 on purpose was spending ten seconds being retried before it could assert anything. eval_api 42.2% -> 48.0%, gen_api 21.9% -> 77.5%. --- .../internal/pkg/gen_api/operations_test.go | 176 ++++++++++++++++ .../internal/pkg/eval_api/operations_test.go | 194 ++++++++++++++++++ .../pkg/eval_api/publish_version_test.go | 9 +- 3 files changed, 377 insertions(+), 2 deletions(-) create mode 100644 cli/azd/extensions/azure.ai.dataset/internal/pkg/gen_api/operations_test.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/operations_test.go diff --git a/cli/azd/extensions/azure.ai.dataset/internal/pkg/gen_api/operations_test.go b/cli/azd/extensions/azure.ai.dataset/internal/pkg/gen_api/operations_test.go new file mode 100644 index 00000000000..765abb961da --- /dev/null +++ b/cli/azd/extensions/azure.ai.dataset/internal/pkg/gen_api/operations_test.go @@ -0,0 +1,176 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package gen_api + +import ( + "context" + "io" + "net/http" + "net/http/httptest" + "net/url" + "testing" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// call is what the client actually put on the wire, which is the part of these +// operations that can be wrong without anything failing to compile. +type call struct { + method string + path string + query url.Values + body string +} + +// recorder answers every request with status and body, remembering the last one. +func recorder(t *testing.T, status int, body string) (*Client, *call) { + t.Helper() + var last call + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + raw, _ := io.ReadAll(r.Body) + last = call{method: r.Method, path: r.URL.Path, query: r.URL.Query(), body: string(raw)} + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + if body != "" { + _, _ = w.Write([]byte(body)) + } + })) + t.Cleanup(server.Close) + + // MaxRetries -1 disables the SDK's retry policy. Without it a test that + // answers 5xx on purpose spends ten seconds being retried. + client := NewClientFromPipeline(server.URL, runtime.NewPipeline( + "test", "v1.0.0", runtime.PipelineOptions{}, + &policy.ClientOptions{Retry: policy.RetryOptions{MaxRetries: -1}})) + return client, &last +} + +// The cancel route takes a colon, not a path segment. `{id}/cancel` is a 404 +// while `{id}:cancel` reaches the action, and nothing but the URL says so. +func TestCancelDataGenerationJob_UsesTheColonForm(t *testing.T) { + client, last := recorder(t, http.StatusOK, `{"id":"dgj_1","status":"cancelled"}`) + + _, err := client.CancelDataGenerationJob(context.Background(), "dgj_1", "v1") + + require.NoError(t, err) + assert.Equal(t, http.MethodPost, last.method) + assert.Equal(t, "/data_generation_jobs/dgj_1:cancel", last.path) + assert.Equal(t, "{}", last.body, + "the empty object is what carries a content type; without it the route answers 415") +} + +// A delete that removed the record answers 204 with no body. Treating that as a +// failure would report every successful delete as an error. +func TestDeleteDataGenerationJob_AcceptsNoContent(t *testing.T) { + client, last := recorder(t, http.StatusNoContent, "") + + require.NoError(t, client.DeleteDataGenerationJob(context.Background(), "dgj_1", "v1")) + + assert.Equal(t, http.MethodDelete, last.method) + assert.Equal(t, "/data_generation_jobs/dgj_1", last.path) +} + +// The job routes answer with `data`, not the `value` the dataset routes use. +// Reading the wrong key returns an empty list from a full response. +func TestListDataGenerationJobs_ReadsTheDataEnvelope(t *testing.T) { + client, last := recorder(t, http.StatusOK, + `{"data":[{"id":"dgj_1","status":"completed"},{"id":"dgj_2","status":"running"}]}`) + + list, err := client.ListDataGenerationJobs(context.Background(), "v1") + + require.NoError(t, err) + assert.Equal(t, "/data_generation_jobs", last.path) + require.Len(t, list.Data, 2) + assert.Equal(t, "dgj_1", list.Data[0].ID) +} + +// The request body is what the service validates, so what reaches the wire is +// pinned rather than left to whatever the builder happened to set. +func TestCreateDataGenerationJob_SendsTheBuiltRequest(t *testing.T) { + client, last := recorder(t, http.StatusOK, `{"id":"dgj_1","status":"running"}`) + + req := NewDataGenerationJobRequest("support-regression", "gpt-4o", 15, + []GenerationSource{{Type: "prompt", Prompt: "be helpful"}}) + _, err := client.CreateDataGenerationJob(context.Background(), req, "v1") + + require.NoError(t, err) + assert.Equal(t, http.MethodPost, last.method) + assert.Equal(t, "/data_generation_jobs", last.path) + assert.Contains(t, last.body, `"name":"support-regression"`) + assert.Contains(t, last.body, `"max_samples":15`) + assert.Contains(t, last.body, `"scenario":"evaluation"`) +} + +// An id goes into the path, so one containing a separator has to be escaped or +// it silently addresses a different route. +func TestOperations_EscapeIdsInThePath(t *testing.T) { + client, last := recorder(t, http.StatusOK, `{"id":"x"}`) + + _, err := client.GetDataGenerationJob(context.Background(), "dgj/../evil", "v1") + + require.NoError(t, err) + assert.NotContains(t, last.path, "/../", + "an unescaped id would let a name climb out of its route") +} + +func TestOperations_SendTheApiVersion(t *testing.T) { + client, last := recorder(t, http.StatusOK, `{"id":"x"}`) + + _, err := client.GetDataGenerationJob(context.Background(), "dgj_1", "2025-11-15-preview") + + require.NoError(t, err) + assert.Equal(t, "2025-11-15-preview", last.query.Get("api-version")) +} + +// Only the newest agent version seeds generation: the point is to describe what +// the agent does now. +func TestGetAgent_ReadsTheCatalogEntry(t *testing.T) { + client, last := recorder(t, http.StatusOK, + `{"name":"support","versions":{"latest":{"version":"3",`+ + `"definition":{"instructions":"Be helpful."}}}}`) + + agent, err := client.GetAgent(context.Background(), "support", "v1") + + require.NoError(t, err) + assert.Equal(t, "/agents/support", last.path) + assert.Equal(t, "Be helpful.", agent.Instructions()) +} + +// A 404 has to arrive as one, because jobLookupError branches on it to point at +// the evaluator group rather than reporting a transport failure. +func TestOperations_NotFoundIsRecognizable(t *testing.T) { + client, _ := recorder(t, http.StatusNotFound, `{"error":{"code":"NotFound"}}`) + + _, err := client.GetDataGenerationJob(context.Background(), "dgj_missing", "v1") + + require.Error(t, err) + assert.True(t, IsNotFound(err)) +} + +// A server fault is worth retrying and has to be recognizable as such, or the +// poller gives up on a job the service is still working on. +func TestOperations_ServerFaultIsTransient(t *testing.T) { + client, _ := recorder(t, http.StatusBadGateway, `{"error":{"code":"BadGateway"}}`) + + _, err := client.GetDataGenerationJob(context.Background(), "dgj_1", "v1") + + require.Error(t, err) + assert.True(t, IsTransientError(err)) + assert.False(t, IsNotFound(err)) +} + +// An empty body on a success is not a parse failure: a 204 carries none, and +// the typed helper has to hand back a zero value rather than an error. +func TestOperations_EmptyBodyIsNotAnError(t *testing.T) { + client, _ := recorder(t, http.StatusOK, "") + + job, err := client.GetDataGenerationJob(context.Background(), "dgj_1", "v1") + + require.NoError(t, err) + require.NotNil(t, job) + assert.Empty(t, job.ID) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/operations_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/operations_test.go new file mode 100644 index 00000000000..0bbccfab675 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/operations_test.go @@ -0,0 +1,194 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package eval_api + +import ( + "context" + "encoding/json" + "io" + "net/http" + "net/url" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// call is what the client actually put on the wire, which is the part of these +// operations that can be wrong without anything failing to compile. +type call struct { + method string + path string + query url.Values + body string +} + +// recorder answers every request with status and body, remembering the last one. +func recorder(t *testing.T, status int, body string) (*EvalClient, *call) { + t.Helper() + var last call + client := newRecordingClient(t, func(w http.ResponseWriter, r *http.Request) { + raw, _ := io.ReadAll(r.Body) + last = call{method: r.Method, path: r.URL.Path, query: r.URL.Query(), body: string(raw)} + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + if body != "" { + _, _ = w.Write([]byte(body)) + } + }) + return client, &last +} + +// The cancel route takes a colon, not a path segment. `{id}/cancel` is a 404 +// while `{id}:cancel` reaches the action, and nothing but the URL says so. +func TestCancelGenerationJob_UsesTheColonForm(t *testing.T) { + tests := []struct { + name string + cancel func(*EvalClient) error + want string + }{ + { + name: "dataset", + cancel: func(c *EvalClient) error { + _, err := c.CancelDataGenerationJob(context.Background(), "dgj_1", "v1") + return err + }, + want: "/data_generation_jobs/dgj_1:cancel", + }, + { + name: "evaluator", + cancel: func(c *EvalClient) error { + _, err := c.CancelEvaluatorGenerationJob(context.Background(), "egj_1", "v1") + return err + }, + want: "/evaluator_generation_jobs/egj_1:cancel", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + client, last := recorder(t, http.StatusOK, `{"id":"j_1","status":"cancelled"}`) + + require.NoError(t, tt.cancel(client)) + + assert.Equal(t, http.MethodPost, last.method) + assert.Equal(t, tt.want, last.path) + assert.Equal(t, "{}", last.body, + "the empty object is what carries a content type; without it the route answers 415") + }) + } +} + +// A delete that removed the record answers 204 with no body. Treating that as a +// failure would report every successful delete as an error. +func TestDeleteGenerationJob_AcceptsNoContent(t *testing.T) { + client, last := recorder(t, http.StatusNoContent, "") + + require.NoError(t, client.DeleteDataGenerationJob(context.Background(), "dgj_1", "v1")) + + assert.Equal(t, http.MethodDelete, last.method) + assert.Equal(t, "/data_generation_jobs/dgj_1", last.path) +} + +// The job routes answer with `data`, not the `value` the dataset and evaluator +// routes use. Reading the wrong key returns an empty list from a full response. +func TestListGenerationJobs_ReadsTheDataEnvelope(t *testing.T) { + client, _ := recorder(t, http.StatusOK, + `{"data":[{"id":"dgj_1","status":"completed"},{"id":"dgj_2","status":"running"}]}`) + + list, err := client.ListDataGenerationJobs(context.Background(), "v1") + + require.NoError(t, err) + require.Len(t, list.Data, 2) + assert.Equal(t, "dgj_1", list.Data[0].ID) +} + +// An id goes into the path, so one containing a separator has to be escaped or +// it silently addresses a different route. +func TestOperations_EscapeIdsInThePath(t *testing.T) { + client, last := recorder(t, http.StatusOK, `{"id":"x"}`) + + _, err := client.GetDataGenerationJob(context.Background(), "dgj/../evil", "v1") + + require.NoError(t, err) + assert.NotContains(t, last.path, "/../", + "an unescaped id would let a name climb out of its route") +} + +// The api-version is what selects the contract; sending the wrong one, or none, +// is answered by a different shape than the client parses. +func TestOperations_SendTheApiVersion(t *testing.T) { + client, last := recorder(t, http.StatusOK, `{"id":"x"}`) + + _, err := client.GetDataGenerationJob(context.Background(), "dgj_1", "2025-11-15-preview") + + require.NoError(t, err) + assert.Equal(t, "2025-11-15-preview", last.query.Get("api-version")) +} + +// The OpenAI-compatible eval routes send no api-version at all, so adding one +// would be as wrong as omitting it elsewhere. +func TestOpenAIEvalRoutes_SendNoApiVersion(t *testing.T) { + client, last := recorder(t, http.StatusOK, `{"id":"eval_1"}`) + + _, err := client.GetOpenAIEval(context.Background(), "eval_1") + + require.NoError(t, err) + assert.Equal(t, "/openai/v1/evals/eval_1", last.path) + assert.Empty(t, last.query.Get("api-version")) +} + +// A rename is pushed in place so the eval keeps its id and its run history. +// The route is a POST to the eval itself, not a PATCH and not a new eval. +func TestUpdateOpenAIEval_PostsToTheEval(t *testing.T) { + client, last := recorder(t, http.StatusOK, `{"id":"eval_1","name":"renamed"}`) + + _, err := client.UpdateOpenAIEval(context.Background(), "eval_1", + &UpdateOpenAIEvalRequest{Name: "renamed"}) + + require.NoError(t, err) + assert.Equal(t, http.MethodPost, last.method) + assert.Equal(t, "/openai/v1/evals/eval_1", last.path) + + var sent map[string]any + require.NoError(t, json.Unmarshal([]byte(last.body), &sent)) + assert.Equal(t, "renamed", sent["name"]) +} + +// Only the newest agent version seeds generation: the point is to describe what +// the agent does now. +func TestGetAgent_ReadsTheCatalogEntry(t *testing.T) { + client, last := recorder(t, http.StatusOK, + `{"name":"support","versions":{"latest":{"version":"3",`+ + `"definition":{"instructions":"Be helpful."}}}}`) + + agent, err := client.GetAgent(context.Background(), "support", "v1") + + require.NoError(t, err) + assert.Equal(t, "/agents/support", last.path) + assert.Equal(t, "Be helpful.", agent.Instructions()) +} + +// A 404 has to arrive as one, because the commands branch on it to tell "no +// such thing" apart from "the call failed". +func TestOperations_NotFoundIsRecognizable(t *testing.T) { + client, _ := recorder(t, http.StatusNotFound, `{"error":{"code":"NotFound"}}`) + + _, err := client.GetOpenAIEval(context.Background(), "eval_missing") + + require.Error(t, err) + assert.True(t, IsNotFound(err), "the commands branch on this to name the thing that is missing") +} + +// An empty body on a success is not a parse failure: a 204 carries none, and +// the typed helper has to hand back a zero value rather than an error. +func TestOperations_EmptyBodyIsNotAnError(t *testing.T) { + client, _ := recorder(t, http.StatusOK, "") + + job, err := client.GetDataGenerationJob(context.Background(), "dgj_1", "v1") + + require.NoError(t, err) + require.NotNil(t, job) + assert.Empty(t, job.ID) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/publish_version_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/publish_version_test.go index e34e1460017..9f319fc586c 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/publish_version_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/publish_version_test.go @@ -13,6 +13,7 @@ import ( "testing" "time" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" "github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime" "github.com/stretchr/testify/assert" @@ -21,12 +22,16 @@ import ( // newRecordingClient points a client at a test server, with no credential // policy in the pipeline. +// +// MaxRetries -1 disables the SDK's retry policy, so a test that answers 5xx on +// purpose does not spend ten seconds being retried. func newRecordingClient(t *testing.T, handler http.HandlerFunc) *EvalClient { t.Helper() server := httptest.NewServer(handler) t.Cleanup(server.Close) - return NewEvalClientFromPipeline( - server.URL, runtime.NewPipeline("test", "v1.0.0", runtime.PipelineOptions{}, nil)) + return NewEvalClientFromPipeline(server.URL, runtime.NewPipeline( + "test", "v1.0.0", runtime.PipelineOptions{}, + &policy.ClientOptions{Retry: policy.RetryOptions{MaxRetries: -1}})) } // versionServer answers a version listing and a publish, assigning whatever From aeb4b57278743c22082ce95abf48e7ea7024e157 Mon Sep 17 00:00:00 2001 From: mohessie Date: Thu, 6 Aug 2026 05:31:59 +0300 Subject: [PATCH 121/320] Pin the exit-code contract against the spec's CI scenario The gating logic already matched the spec; nothing said so. Auditing it found the behaviour right and the evidence missing, which is the state where a refactor breaks a pipeline quietly. The breach message is extracted from applyGate so it can be asserted at all -- applyGate calls os.Exit, so its wording was unreachable from a test. The two lines are now compared whole against the block the spec's CI scenario prints, because a pipeline's log is where they are read and a marker change is what would make a gate look like a crash. Also pinned: exit 2 belongs to a breached threshold and nothing else; a completed run is exit 0 whatever it scored while an errored one is an operational failure, which is the distinction the separate code exists for; and pass-rate is passed/total with errored and skipped inside the total. That last one matters most -- were they outside it, a run with two passes and one error would report total=2 and score a perfect rate, which is precisely the broken evaluation a gate exists to catch. --- .../internal/cmd/gating.go | 11 ++- .../internal/cmd/gating_conformance_test.go | 94 +++++++++++++++++++ 2 files changed, 103 insertions(+), 2 deletions(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/gating_conformance_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/gating.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/gating.go index 0b6940a28d5..c98e9fd96a7 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/gating.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/gating.go @@ -95,6 +95,14 @@ func (g gate) breach(counts *eval_api.EvalRunResultCounts) string { return "" } +// gateBreachMessage is what a breached gate prints, kept separate from the +// exit so the wording can be tested: it is the block the spec's CI scenario +// shows, and a pipeline's logs are where it is read. +func gateBreachMessage(reason string) string { + return fmt.Sprintf("%s Evaluation gate: %s\n\nERROR: evaluation quality gate not met.\n", + failedMark, reason) +} + // applyGate ends the process with exit code 2 when the run missed its // threshold. // @@ -109,8 +117,7 @@ func applyGate(cmd *cobra.Command, g gate, run *eval_api.OpenAIEvalRun) { if reason == "" { return } - fmt.Fprintf(os.Stderr, "(x) Failed: Evaluation gate: %s\n\n", reason) - fmt.Fprintln(os.Stderr, "ERROR: evaluation quality gate not met.") + fmt.Fprint(os.Stderr, gateBreachMessage(reason)) os.Exit(exitCodeGateBreached) } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/gating_conformance_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/gating_conformance_test.go new file mode 100644 index 00000000000..e061314324c --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/gating_conformance_test.go @@ -0,0 +1,94 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "strings" + "testing" + + "azureaieval/internal/pkg/eval_api" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// The spec's CI scenario prints this block verbatim, and a pipeline's log is +// where it is read. Both lines are asserted whole: the marker is what tells a +// reader this is a gate rather than a crash, and the ERROR: line is what the +// azd style guide reserves for a terminal failure. +func TestGateBreachMessageMatchesTheScenario(t *testing.T) { + msg := gateBreachMessage("pass rate 76.0% is below the required 80.0%") + + assert.Equal(t, + "(x) Failed: Evaluation gate: pass rate 76.0% is below the required 80.0%\n\n"+ + "ERROR: evaluation quality gate not met.\n", + msg) +} + +// Exit 2 is the whole point of --fail-on: a pipeline has to tell "the +// evaluation regressed" apart from "the tool could not run", which is exit 1. +func TestGateBreachUsesItsOwnExitCode(t *testing.T) { + assert.Equal(t, 2, exitCodeGateBreached, + "the spec's exit table gives 2 to a breached threshold") +} + +// The spec's exit table: a completed run is 0 whatever it scored, and a run +// that errored rather than completed is an operational failure. +func TestRunCompletedSeparatesRegressionFromFailureToRun(t *testing.T) { + for _, status := range []string{"completed", "Completed", ""} { + assert.NoErrorf(t, runCompleted(&eval_api.OpenAIEvalRun{ID: "r", Status: status}), + "%q is a run that produced results, so its score decides the outcome", status) + } + + for _, status := range []string{"failed", "errored", "canceled"} { + err := runCompleted(&eval_api.OpenAIEvalRun{ID: "r1", Status: status}) + require.Errorf(t, err, "%q never produced results, so it did not regress", status) + assert.Contains(t, err.Error(), status) + assert.Contains(t, err.Error(), "r1") + } +} + +// pass-rate is passed/total, and the service puts errored and skipped rows +// inside total. Were they outside it, a run with two passes and one error +// would score a perfect rate -- exactly the broken evaluation a gate exists to +// catch. +func TestPassRateCountsErroredAndSkippedAgainstTheThreshold(t *testing.T) { + g, err := parseGate("pass-rate=0.8") + require.NoError(t, err) + + // 2 passed of 3 total, the third errored: 66.7%, below 80%. + breach := g.breach(&eval_api.EvalRunResultCounts{Total: 3, Passed: 2, Errored: 1}) + require.NotEmpty(t, breach, "an errored row is not a pass") + assert.Contains(t, breach, "66.7%") + + assert.Empty(t, g.breach(&eval_api.EvalRunResultCounts{Total: 3, Passed: 3})) +} + +// A run that scored nothing breaches every threshold rather than dividing by +// zero. "No rows passed" is the honest reading of an empty result. +func TestEmptyRunBreachesEveryThreshold(t *testing.T) { + g, err := parseGate("pass-rate=0.1") + require.NoError(t, err) + + breach := g.breach(&eval_api.EvalRunResultCounts{}) + + assert.NotEmpty(t, breach) + assert.NotContains(t, breach, "NaN", "dividing by zero must not reach the message") +} + +// --fail-on belongs to the commands that wait for a terminal state, so a +// pipeline that started a run asynchronously can still gate where it reattaches. +func TestFailOnSitsOnTheWaitingCommands(t *testing.T) { + for _, path := range []string{"run start", "run show"} { + assert.NotNilf(t, find(t, path).Flags().Lookup("fail-on"), + "%s waits for a terminal state, so it can gate on one", path) + } + + usage := find(t, "run start").Flags().Lookup("fail-on").Usage + for _, form := range []string{"any-failure", "pass-rate"} { + assert.Containsf(t, usage, form, "--fail-on accepts %q, so its help has to say so", form) + } + assert.Contains(t, strings.ToLower(usage), "2", + "the help has to name the exit code, which is the only reason to use the flag") +} From 86b0dd8627872f3eb8cd39f9e430c123a9fc1e5d Mon Sep 17 00:00:00 2001 From: mohessie Date: Thu, 6 Aug 2026 05:42:17 +0300 Subject: [PATCH 122/320] Give evaluator show a detail view, closing the last known deviation This was the one place I knew the code and the spec disagreed. The output conventions make `show` a two-column key/value view with Title Case keys, and `-o json` the machine-readable alternative. `evaluator show` emitted raw JSON whatever was asked for -- which also left it the only show command with nowhere to put the portal link the same section requires, since a line after JSON makes the output unparseable. It now prints name, version, type, description, categories and evaluation levels, then the portal link. The schemas stay in `-o json`: printed here they would bury the few lines a reader came for. `-o json` still answers with the service's document untouched, because a caller asking for JSON wants the evaluator rather than this view of it, and an evaluator shaped in a way the view cannot read still prints raw rather than failing. emitDetail is shared rather than local to this command, because the convention belongs to every show and the next one should not reinvent the spacing. It drops empty values: a column of blank keys says only that the writer did not know which fields that kind has. --- .../internal/cmd/detail_test.go | 122 ++++++++++++++++++ .../internal/cmd/evaluator.go | 49 ++++++- .../internal/cmd/output.go | 24 ++++ 3 files changed, 192 insertions(+), 3 deletions(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/detail_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/detail_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/detail_test.go new file mode 100644 index 00000000000..5df16c39a68 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/detail_test.go @@ -0,0 +1,122 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "bytes" + "context" + "strings" + "testing" + + "azureaieval/internal/pkg/eval_api" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// A detail view is two columns with Title Case keys, per the spec's output +// conventions. It is what `show` prints; `-o json` is the machine-readable +// alternative, not the only form. +func TestEmitDetail(t *testing.T) { + var buf bytes.Buffer + require.NoError(t, emitDetail(&buf, []field{ + {"Name", "support-quality"}, + {"Version", "3"}, + {"Type", "rubric"}, + })) + + lines := strings.Split(strings.TrimRight(buf.String(), "\n"), "\n") + require.Len(t, lines, 3) + for i, want := range []string{"Name", "Version", "Type"} { + assert.Truef(t, strings.HasPrefix(lines[i], want), + "line %d should start with the key %q, got %q", i, want, lines[i]) + } + assert.Contains(t, lines[0], "support-quality") +} + +// A blank value says only that the writer did not know which fields this kind +// has, so it is dropped rather than printed as an empty column. +func TestEmitDetail_DropsEmptyValues(t *testing.T) { + var buf bytes.Buffer + require.NoError(t, emitDetail(&buf, []field{ + {"Name", "support-quality"}, + {"Description", ""}, + {"Type", "rubric"}, + })) + + assert.NotContains(t, buf.String(), "Description") + assert.Len(t, strings.Split(strings.TrimRight(buf.String(), "\n"), "\n"), 2) +} + +// `evaluator show` used to emit raw JSON whatever was asked for, which left it +// the one `show` with no detail view and nowhere to put a portal link. +func TestRenderEvaluator(t *testing.T) { + var buf bytes.Buffer + ec := &evalContext{} + + require.NoError(t, ec.renderEvaluator(context.Background(), &buf, &eval_api.EvaluatorSummary{ + Name: "support-quality", + Version: "3", + EvaluatorType: "rubric", + Description: "Grades politeness and accuracy.", + Categories: []string{"quality", "custom"}, + SupportedEvaluationLevels: []string{"turn", "conversation"}, + })) + + out := buf.String() + for _, want := range []string{ + "Name", "support-quality", + "Version", "3", + "Type", "rubric", + "Description", "Grades politeness and accuracy.", + "Categories", "quality, custom", + "Evaluation Levels", "turn, conversation", + } { + assert.Contains(t, out, want) + } + + // The schemas live in -o json: printed here they would bury the few lines a + // reader came for. + assert.NotContains(t, out, "data_schema") + assert.NotContains(t, out, "init_parameters") +} + +// Both spellings of the type field are read, because the listing says +// evaluator_type and other payloads say type. +func TestRenderEvaluator_ReadsEitherTypeSpelling(t *testing.T) { + for _, e := range []*eval_api.EvaluatorSummary{ + {Name: "x", EvaluatorType: "rubric"}, + {Name: "x", TypeAlias: "rubric"}, + } { + var buf bytes.Buffer + ec := &evalContext{} + require.NoError(t, ec.renderEvaluator(context.Background(), &buf, e)) + assert.Contains(t, buf.String(), "rubric") + } +} + +// Without an azd environment there is no project to address, so the view ends +// at its last field rather than at an empty label. +func TestRenderEvaluator_NoPortalLinkWithoutAProject(t *testing.T) { + var buf bytes.Buffer + ec := &evalContext{} + + require.NoError(t, ec.renderEvaluator(context.Background(), &buf, + &eval_api.EvaluatorSummary{Name: "support-quality", Version: "3"})) + + assert.NotContains(t, buf.String(), "Portal:") +} + +// The spec gives evaluator URLs their own shape, distinct from datasets and +// runs, so a link built from the wrong one resolves to nothing. +func TestPortalEvaluatorURLShape(t *testing.T) { + prefix, err := eval_api.NewPortalPrefix( + "/subscriptions/00000000-1111-2222-3333-444444444444/resourceGroups/rg/" + + "providers/Microsoft.CognitiveServices/accounts/acct/projects/proj") + require.NoError(t, err) + + assert.True(t, strings.HasSuffix( + prefix.EvaluatorURL("support-quality", "3"), + "/build/evaluations/catalog/support-quality/3")) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator.go index 48aeb48a477..a196a773c24 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator.go @@ -4,9 +4,12 @@ package cmd import ( + "context" "encoding/json" "fmt" + "io" "os" + "strings" "azureaieval/internal/pkg/eval_api" @@ -327,12 +330,25 @@ func newEvaluatorShowCommand() *cobra.Command { return fmt.Errorf("reading evaluator %q: %w", name, err) } - var pretty any - if err := json.Unmarshal(raw, &pretty); err != nil { + // -o json answers with the service's document untouched, because a + // caller asking for JSON wants the evaluator, not this view of it. + if isJSON(cmd) { + var pretty any + if err := json.Unmarshal(raw, &pretty); err != nil { + fmt.Fprintln(cmd.OutOrStdout(), string(raw)) + return nil + } + return emitJSON(cmd.OutOrStdout(), pretty) + } + + var summary eval_api.EvaluatorSummary + if err := json.Unmarshal(raw, &summary); err != nil { + // An evaluator shaped in a way this view cannot read is still + // worth showing; falling back beats refusing to print it. fmt.Fprintln(cmd.OutOrStdout(), string(raw)) return nil } - return emitJSON(cmd.OutOrStdout(), pretty) + return ec.renderEvaluator(ctx, cmd.OutOrStdout(), &summary) }, } @@ -341,6 +357,33 @@ func newEvaluatorShowCommand() *cobra.Command { return cmd } +// renderEvaluator prints the detail view for one evaluator, closing with its +// portal link. +// +// The columns an evaluator is identified by, then what it grades and where it +// can run. The full definition is in `-o json`; a schema printed here would +// bury the four lines a reader came for. +func (ec *evalContext) renderEvaluator( + ctx context.Context, + out io.Writer, + e *eval_api.EvaluatorSummary, +) error { + if err := emitDetail(out, []field{ + {"Name", e.Name}, + {"Version", e.Version}, + {"Type", e.Type()}, + {"Description", e.Description}, + {"Categories", strings.Join(e.Categories, ", ")}, + {"Evaluation Levels", strings.Join(e.SupportedEvaluationLevels, ", ")}, + }); err != nil { + return err + } + if prefix := ec.portalPrefix(ctx); prefix != nil && e.Name != "" { + writePortalLink(out, prefix.EvaluatorURL(e.Name, e.Version)) + } + return nil +} + func newEvaluatorDeleteCommand() *cobra.Command { var ( version string diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/output.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/output.go index 07578515781..d043508268a 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/output.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/output.go @@ -92,6 +92,30 @@ func emitTable(w io.Writer, headers []string, rows [][]string) error { return tw.Flush() } +// field is one row of a detail view. +type field struct { + Key string // Title Case, per the azd style guide + Value string +} + +// emitDetail writes a two-column key/value view, the shape `show` uses. +// +// Empty values are dropped rather than printed blank: a detail view is read to +// learn what a thing is, and a column of empty keys says only that the writer +// did not know which fields this kind has. +func emitDetail(w io.Writer, fields []field) error { + tw := tabwriter.NewWriter(w, 0, 0, 3, ' ', 0) + for _, f := range fields { + if f.Value == "" { + continue + } + if _, err := fmt.Fprintf(tw, "%s\t%s\n", f.Key, f.Value); err != nil { + return err + } + } + return tw.Flush() +} + // requireFlag returns an error naming the missing flag, used when --no-prompt // prevents asking for a required value. func requireFlag(name string) error { From 4fb209121b9f59dd3d11cd0a22a03dd292d382ad Mon Sep 17 00:00:00 2001 From: mohessie Date: Thu, 6 Aug 2026 05:50:42 +0300 Subject: [PATCH 123/320] Make every show a detail view, and stop pointing at the other extension Finishing the evaluator change turned up the same deviation in two more places. `eval show` also emitted raw JSON whatever was asked for, and `dataset show` printed a one-row table -- the list-view shape -- for a single thing. The spec splits the two: uppercase headers over a rule for a list, Title Case key/value for a detail, and `-o json` as the machine-readable alternative to either. All three are detail views now, in both extensions, and a test asserts each `show` renders through emitDetail so the next one does not drift back. It reads the source rather than running the command, because these need a service and a shape regression should not wait for a live run. The dataset extension was also telling users to run `azd ai eval dataset list` when a dataset was missing -- the command it was copied from, in an extension they may not have installed, for something it serves itself as `azd ai dataset list`. The suggestion guard skipped it because `eval` is a legitimate sibling namespace for the one line that registers a generated file in eval.yaml. That exception is now listed by name and everything else under `azd ai eval dataset` fails. --- .../azure.ai.dataset/internal/cmd/dataset.go | 12 +++--- .../azure.ai.dataset/internal/cmd/output.go | 24 +++++++++++ .../internal/cmd/surface_test.go | 40 +++++++++++++++++++ .../internal/cmd/dataset.go | 10 +++-- .../internal/cmd/detail_test.go | 40 +++++++++++++++++++ .../internal/cmd/eval_group.go | 10 ++++- 6 files changed, 126 insertions(+), 10 deletions(-) diff --git a/cli/azd/extensions/azure.ai.dataset/internal/cmd/dataset.go b/cli/azd/extensions/azure.ai.dataset/internal/cmd/dataset.go index ad0b323e56f..9e2f700cea2 100644 --- a/cli/azd/extensions/azure.ai.dataset/internal/cmd/dataset.go +++ b/cli/azd/extensions/azure.ai.dataset/internal/cmd/dataset.go @@ -251,7 +251,7 @@ func newDatasetShowCommand() *cobra.Command { if gen_api.IsNotFound(err) { return fmt.Errorf( "no dataset %q at version %q in this project; "+ - "`azd ai eval dataset list` shows the ones there are", name, version) + "`azd ai dataset list` shows the ones there are", name, version) } return fmt.Errorf("reading dataset %q version %q: %w", name, version, err) } @@ -259,10 +259,12 @@ func newDatasetShowCommand() *cobra.Command { if isJSON(cmd) { return emitJSON(cmd.OutOrStdout(), ds) } - return emitTable(cmd.OutOrStdout(), - []string{"NAME", "VERSION", "FORMAT", "URI"}, - [][]string{{ds.Name, ds.Version, ds.Format, ds.ResolvedBlobURI()}}, - ) + return emitDetail(cmd.OutOrStdout(), []field{ + {"Name", ds.Name}, + {"Version", ds.Version}, + {"Format", ds.Format}, + {"URI", ds.ResolvedBlobURI()}, + }) }, } diff --git a/cli/azd/extensions/azure.ai.dataset/internal/cmd/output.go b/cli/azd/extensions/azure.ai.dataset/internal/cmd/output.go index 143b49e91ba..61a0242669b 100644 --- a/cli/azd/extensions/azure.ai.dataset/internal/cmd/output.go +++ b/cli/azd/extensions/azure.ai.dataset/internal/cmd/output.go @@ -79,6 +79,30 @@ func emitTable(w io.Writer, headers []string, rows [][]string) error { return tw.Flush() } +// field is one row of a detail view. +type field struct { + Key string // Title Case, per the azd style guide + Value string +} + +// emitDetail writes a two-column key/value view, the shape `show` uses. +// +// Empty values are dropped rather than printed blank: a detail view is read to +// learn what a thing is, and a column of empty keys says only that the writer +// did not know which fields this kind has. +func emitDetail(w io.Writer, fields []field) error { + tw := tabwriter.NewWriter(w, 0, 0, 3, ' ', 0) + for _, f := range fields { + if f.Value == "" { + continue + } + if _, err := fmt.Fprintf(tw, "%s\t%s\n", f.Key, f.Value); err != nil { + return err + } + } + return tw.Flush() +} + // requireFlag returns an error naming the missing flag, used when --no-prompt // prevents asking for a required value. func requireFlag(name string) error { diff --git a/cli/azd/extensions/azure.ai.dataset/internal/cmd/surface_test.go b/cli/azd/extensions/azure.ai.dataset/internal/cmd/surface_test.go index b625e22dede..001039e3ac6 100644 --- a/cli/azd/extensions/azure.ai.dataset/internal/cmd/surface_test.go +++ b/cli/azd/extensions/azure.ai.dataset/internal/cmd/surface_test.go @@ -187,6 +187,46 @@ func TestSuggestedCommandsExist(t *testing.T) { require.NoError(t, err) } +// A message pointing at `azd ai eval dataset ...` is almost always the copy +// these commands came from rather than a deliberate cross-extension pointer. +// `dataset` is this extension's own namespace, so telling a user to run the +// eval extension's version of a command it serves itself sends them somewhere +// they may not have installed. +// +// generate's "register this in an eval configuration" line is the one real +// exception, because eval.yaml genuinely belongs to the other extension. +func TestNoStaleEvalDatasetSuggestions(t *testing.T) { + allowed := map[string]bool{ + "azd ai eval dataset create": true, // registering a generated file in eval.yaml + } + + err := filepath.WalkDir("../..", func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() || !strings.HasSuffix(path, ".go") || strings.HasSuffix(path, "_test.go") { + return nil + } + body, err := os.ReadFile(path) + if err != nil { + return err + } + pattern := regexp.MustCompile(`azd ai eval dataset [a-z][a-z0-9-]*`) + for i, line := range strings.Split(string(body), "\n") { + if strings.HasPrefix(strings.TrimSpace(line), "//") { + continue + } + for _, m := range pattern.FindAllString(line, -1) { + assert.Truef(t, allowed[m], + "%s:%d suggests `%s`; this extension serves that command as "+ + "`azd ai dataset ...`", path, i+1, m) + } + } + return nil + }) + require.NoError(t, err) +} + // siblingNamespaces are the other Foundry extensions this one points users at. var siblingNamespaces = map[string]bool{ "eval": true, // registering a generated dataset in an eval configuration diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset.go index e9735a0b032..d131794be2c 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset.go @@ -277,10 +277,12 @@ func newDatasetShowCommand() *cobra.Command { if isJSON(cmd) { return emitJSON(cmd.OutOrStdout(), ds) } - if err := emitTable(cmd.OutOrStdout(), - []string{"NAME", "VERSION", "FORMAT", "URI"}, - [][]string{{ds.Name, ds.Version, ds.Format, ds.ResolvedBlobURI()}}, - ); err != nil { + if err := emitDetail(cmd.OutOrStdout(), []field{ + {"Name", ds.Name}, + {"Version", ds.Version}, + {"Format", ds.Format}, + {"URI", ds.ResolvedBlobURI()}, + }); err != nil { return err } if prefix := ec.portalPrefix(ctx); prefix != nil { diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/detail_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/detail_test.go index 5df16c39a68..1e5ebd2540d 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/detail_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/detail_test.go @@ -6,6 +6,8 @@ package cmd import ( "bytes" "context" + "os" + "path/filepath" "strings" "testing" @@ -15,6 +17,44 @@ import ( "github.com/stretchr/testify/require" ) +// The spec's output conventions split the two shapes: a list view is uppercase +// headers over a rule, a detail view is Title Case key/value. `show` returns +// one thing, so it is a detail view -- and all three used to disagree, two +// emitting raw JSON whatever was asked for and one printing a one-row table. +// +// Checked by reading the source rather than running the command, because these +// commands need a service; a shape regression should not wait for a live run. +func TestShowCommandsUseDetailViews(t *testing.T) { + // Command → the file and function that renders it. + renderers := map[string]string{ + "dataset show": "dataset.go", + "evaluator show": "evaluator.go", + "show": "eval_group.go", + } + + for path, file := range renderers { + t.Run(path, func(t *testing.T) { + require.NotNil(t, find(t, path), "the command has to exist to have a shape") + + body, err := os.ReadFile(filepath.Join(".", file)) + require.NoError(t, err) + assert.Containsf(t, string(body), "emitDetail", + "%s returns one thing, so %s renders it as a detail view", path, file) + }) + } +} + +// Every command returning data supports -o json, which is what makes the +// detail view a presentation choice rather than a loss of information. +func TestShowCommandsStillAnswerInJSON(t *testing.T) { + for _, path := range []string{"dataset show", "evaluator show", "show"} { + cmd := find(t, path) + // -o comes from the SDK root, so a command must not shadow it. + assert.Nilf(t, cmd.LocalFlags().Lookup("output"), + "%s must inherit -o rather than declaring its own", path) + } +} + // A detail view is two columns with Title Case keys, per the spec's output // conventions. It is what `show` prints; `-o json` is the machine-readable // alternative, not the only form. diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/eval_group.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/eval_group.go index 3082153d4b4..bda511e8fa6 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/eval_group.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/eval_group.go @@ -161,7 +161,15 @@ func newEvalShowCommand() *cobra.Command { } return fmt.Errorf("reading eval %q: %w", evalID, err) } - return emitJSON(cmd.OutOrStdout(), group) + if isJSON(cmd) { + return emitJSON(cmd.OutOrStdout(), group) + } + return emitDetail(cmd.OutOrStdout(), []field{ + {"Id", group.ID}, + {"Name", group.Name}, + {"Created", fmt.Sprint(group.CreatedAt)}, + {"Created By", group.CreatedBy}, + }) }, } From 9cf157ae52c2f15282ad909b78cad6dcd963e721 Mon Sep 17 00:00:00 2001 From: mohessie Date: Thu, 6 Aug 2026 05:58:53 +0300 Subject: [PATCH 124/320] Put the rule back under list headers The spec's output conventions say a list view is uppercase headers over a `----` rule, and cite azure.ai.skills, which prints one. Every scenario that shows a listing shows the rule. emitTable did not print it, so every `list` command in both extensions ran the header straight onto the first row. The run summary table had it all along -- renderRun builds its own header and rule -- which is how the convention stayed visible in the code while every other table quietly disagreed with it. The rule is dashes as wide as the header they sit under, so tabwriter pads them to the same column widths and they line up. A listing with no rows still prints both lines: a caller seeing nothing cannot tell an empty list from a command that failed to render. One of the new assertions was vacuous on the first pass -- it compared a string against ReplaceAll(s, " ", " "), which is a no-op and always true. Replaced with a real check that the rule holds nothing but dashes and padding. Removing the rule now fails these tests; I checked. --- .../azure.ai.dataset/internal/cmd/output.go | 13 +++- .../internal/cmd/table_test.go | 70 +++++++++++++++++++ .../internal/cmd/output.go | 13 +++- .../internal/cmd/table_test.go | 70 +++++++++++++++++++ 4 files changed, 164 insertions(+), 2 deletions(-) create mode 100644 cli/azd/extensions/azure.ai.dataset/internal/cmd/table_test.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/table_test.go diff --git a/cli/azd/extensions/azure.ai.dataset/internal/cmd/output.go b/cli/azd/extensions/azure.ai.dataset/internal/cmd/output.go index 61a0242669b..8cb9eb1f4ce 100644 --- a/cli/azd/extensions/azure.ai.dataset/internal/cmd/output.go +++ b/cli/azd/extensions/azure.ai.dataset/internal/cmd/output.go @@ -65,12 +65,23 @@ func emitJSONList[T any](w io.Writer, items []T) error { return emitJSON(w, items) } -// emitTable writes a simple aligned table. Rows must match the header width. +// emitTable writes a list view: uppercase headers over a rule, tab-aligned. +// +// The rule is what separates the header from the data at a glance, and it is +// what `azure.ai.skills` prints, so a reader moving between the Foundry +// extensions sees one table. func emitTable(w io.Writer, headers []string, rows [][]string) error { tw := tabwriter.NewWriter(w, 0, 0, 3, ' ', 0) if _, err := fmt.Fprintln(tw, strings.Join(headers, "\t")); err != nil { return err } + rule := make([]string, len(headers)) + for i, h := range headers { + rule[i] = strings.Repeat("-", len(h)) + } + if _, err := fmt.Fprintln(tw, strings.Join(rule, "\t")); err != nil { + return err + } for _, row := range rows { if _, err := fmt.Fprintln(tw, strings.Join(row, "\t")); err != nil { return err diff --git a/cli/azd/extensions/azure.ai.dataset/internal/cmd/table_test.go b/cli/azd/extensions/azure.ai.dataset/internal/cmd/table_test.go new file mode 100644 index 00000000000..92b18ee230a --- /dev/null +++ b/cli/azd/extensions/azure.ai.dataset/internal/cmd/table_test.go @@ -0,0 +1,70 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "bytes" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// A list view is uppercase headers over a rule, per the spec's output +// conventions and the sibling extension it cites. The rule is what separates +// the header from the data at a glance, and every `list` command was printing +// the header straight onto the first row. +func TestEmitTableWritesTheRule(t *testing.T) { + var buf bytes.Buffer + require.NoError(t, emitTable(&buf, + []string{"NAME", "VERSION"}, + [][]string{{"support-regression", "3"}, {"nightly", "1"}})) + + lines := strings.Split(strings.TrimRight(buf.String(), "\n"), "\n") + require.Len(t, lines, 4, "a header, its rule, and one line per row") + + assert.Contains(t, lines[0], "NAME") + assert.Contains(t, lines[0], "VERSION") + + // Dashes as wide as the header they sit under, which is what makes the + // rule line up once tabwriter has padded the columns. + assert.Contains(t, lines[1], strings.Repeat("-", len("NAME"))) + assert.Contains(t, lines[1], strings.Repeat("-", len("VERSION"))) + assert.Empty(t, strings.Trim(lines[1], "- "), + "the rule carries nothing but dashes and padding") + + assert.Contains(t, lines[2], "support-regression") + assert.Contains(t, lines[3], "nightly") +} + +// The columns line up: the rule is padded to the same widths as the header, so +// a wide value in the first row does not leave the rule short. +func TestEmitTableRuleAlignsWithTheHeader(t *testing.T) { + var buf bytes.Buffer + require.NoError(t, emitTable(&buf, + []string{"NAME", "STATUS"}, + [][]string{{"a-very-much-longer-value-than-the-header", "completed"}})) + + lines := strings.Split(strings.TrimRight(buf.String(), "\n"), "\n") + require.Len(t, lines, 3) + + // tabwriter pads every line in a column to the same width, so the header + // and its rule start their second column at the same offset. + assert.Equal(t, + strings.Index(lines[0], "STATUS"), + strings.Index(lines[1], "------"), + "the rule has to sit under the header it belongs to") +} + +// A listing with nothing in it still prints the header and rule: a caller +// seeing no output cannot tell an empty list from a command that failed to +// render. +func TestEmitTableWithNoRows(t *testing.T) { + var buf bytes.Buffer + require.NoError(t, emitTable(&buf, []string{"NAME", "VERSION"}, nil)) + + lines := strings.Split(strings.TrimRight(buf.String(), "\n"), "\n") + assert.Len(t, lines, 2) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/output.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/output.go index d043508268a..e89b19ae3b8 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/output.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/output.go @@ -78,12 +78,23 @@ func emitJSONList[T any](w io.Writer, items []T) error { return emitJSON(w, items) } -// emitTable writes a simple aligned table. Rows must match the header width. +// emitTable writes a list view: uppercase headers over a rule, tab-aligned. +// +// The rule is what separates the header from the data at a glance, and it is +// what `azure.ai.skills` prints, so a reader moving between the Foundry +// extensions sees one table. func emitTable(w io.Writer, headers []string, rows [][]string) error { tw := tabwriter.NewWriter(w, 0, 0, 3, ' ', 0) if _, err := fmt.Fprintln(tw, strings.Join(headers, "\t")); err != nil { return err } + rule := make([]string, len(headers)) + for i, h := range headers { + rule[i] = strings.Repeat("-", len(h)) + } + if _, err := fmt.Fprintln(tw, strings.Join(rule, "\t")); err != nil { + return err + } for _, row := range rows { if _, err := fmt.Fprintln(tw, strings.Join(row, "\t")); err != nil { return err diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/table_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/table_test.go new file mode 100644 index 00000000000..92b18ee230a --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/table_test.go @@ -0,0 +1,70 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "bytes" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// A list view is uppercase headers over a rule, per the spec's output +// conventions and the sibling extension it cites. The rule is what separates +// the header from the data at a glance, and every `list` command was printing +// the header straight onto the first row. +func TestEmitTableWritesTheRule(t *testing.T) { + var buf bytes.Buffer + require.NoError(t, emitTable(&buf, + []string{"NAME", "VERSION"}, + [][]string{{"support-regression", "3"}, {"nightly", "1"}})) + + lines := strings.Split(strings.TrimRight(buf.String(), "\n"), "\n") + require.Len(t, lines, 4, "a header, its rule, and one line per row") + + assert.Contains(t, lines[0], "NAME") + assert.Contains(t, lines[0], "VERSION") + + // Dashes as wide as the header they sit under, which is what makes the + // rule line up once tabwriter has padded the columns. + assert.Contains(t, lines[1], strings.Repeat("-", len("NAME"))) + assert.Contains(t, lines[1], strings.Repeat("-", len("VERSION"))) + assert.Empty(t, strings.Trim(lines[1], "- "), + "the rule carries nothing but dashes and padding") + + assert.Contains(t, lines[2], "support-regression") + assert.Contains(t, lines[3], "nightly") +} + +// The columns line up: the rule is padded to the same widths as the header, so +// a wide value in the first row does not leave the rule short. +func TestEmitTableRuleAlignsWithTheHeader(t *testing.T) { + var buf bytes.Buffer + require.NoError(t, emitTable(&buf, + []string{"NAME", "STATUS"}, + [][]string{{"a-very-much-longer-value-than-the-header", "completed"}})) + + lines := strings.Split(strings.TrimRight(buf.String(), "\n"), "\n") + require.Len(t, lines, 3) + + // tabwriter pads every line in a column to the same width, so the header + // and its rule start their second column at the same offset. + assert.Equal(t, + strings.Index(lines[0], "STATUS"), + strings.Index(lines[1], "------"), + "the rule has to sit under the header it belongs to") +} + +// A listing with nothing in it still prints the header and rule: a caller +// seeing no output cannot tell an empty list from a command that failed to +// render. +func TestEmitTableWithNoRows(t *testing.T) { + var buf bytes.Buffer + require.NoError(t, emitTable(&buf, []string{"NAME", "VERSION"}, nil)) + + lines := strings.Split(strings.TrimRight(buf.String(), "\n"), "\n") + assert.Len(t, lines, 2) +} From 68bb20c210461b768ccd113127027045e708ad80 Mon Sep 17 00:00:00 2001 From: mohessie Date: Thu, 6 Aug 2026 06:06:09 +0300 Subject: [PATCH 125/320] Give run list the columns the scenario reads Scenario 3 answers "did my change help?" by eyeballing two rows of run list, and Open Question 8 rests on that being an acceptable M1 answer. The columns were RUN ID / NAME / STATUS / RESULTS, where RESULTS was "14 passed, 1 failed, 0 errored" -- so the row carried neither when the run happened nor what it scored, and the question the scenario exists to answer could not be answered from it. They are RUN / STARTED / STATUS / SAMPLES / PASS RATE now. The rate is the gate's own arithmetic, passed over total with errored and skipped inside the total, so a row a reader gates on cannot disagree with the gate that acts on it. A run that has not scored yet shows an empty cell rather than 0.0%, which would read as a failure. Timestamps are RFC3339 UTC whichever shape the service sent, because it answers with epoch seconds on some routes and a string on others and a list mixing both would not sort. The spec also shows a DATASET column. It is not in this change because a run does not carry one: registered datasets are fetched and sent inline, so the name is not on the run to read. Recovering it would mean a lookup per row. --- .../internal/cmd/run_list_test.go | 58 +++++++++++++++++++ .../internal/cmd/run_ops.go | 30 +++++++++- 2 files changed, 86 insertions(+), 2 deletions(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_list_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_list_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_list_test.go new file mode 100644 index 00000000000..7e2aaf9f157 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_list_test.go @@ -0,0 +1,58 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "testing" + + "azureaieval/internal/pkg/eval_api" + + "github.com/stretchr/testify/assert" +) + +// Scenario 3 answers "did my change help?" by reading two rows of `run list`, +// which only works if a row carries when it ran and how it scored. The columns +// were RUN ID / NAME / STATUS / RESULTS, so the question the scenario exists to +// answer could not be. +func TestRunListColumnsMatchTheScenario(t *testing.T) { + counts := &eval_api.EvalRunResultCounts{Total: 15, Passed: 14, Failed: 1} + + assert.Equal(t, "15", sampleCount(counts), + "a rate over 15 samples and one over 200 are not the same claim") + assert.Equal(t, "93.3%", runPassRate(counts), + "the scenario compares 80.0% against 93.3%, so the row has to carry the rate") +} + +// The rate is the gate's arithmetic: passed over total, with errored and +// skipped inside the total. A row a reader gates on must not disagree with the +// gate that acts on it. +func TestRunListPassRateAgreesWithTheGate(t *testing.T) { + counts := &eval_api.EvalRunResultCounts{Total: 3, Passed: 2, Errored: 1} + + assert.Equal(t, "66.7%", runPassRate(counts), + "an errored row is not a pass, here or in the gate") + + g, err := parseGate("pass-rate=0.8") + assert.NoError(t, err) + assert.NotEmpty(t, g.breach(counts), + "the same counts that read 66.7% must breach an 80% threshold") +} + +// A run that has not scored yet has no rate to show. An empty cell says that; +// "0.0%" would say the run failed. +func TestRunListOmitsARateItCannotCompute(t *testing.T) { + assert.Empty(t, runPassRate(nil)) + assert.Empty(t, runPassRate(&eval_api.EvalRunResultCounts{})) + assert.Empty(t, sampleCount(nil)) +} + +// Timestamps are RFC3339 in UTC, whichever shape the service sent. The service +// answers with epoch seconds on some routes and a string on others, and a list +// that renders both would not sort. +func TestRunListTimestampsAreRFC3339UTC(t *testing.T) { + assert.Equal(t, "2026-08-01T09:15:22Z", timestampString(float64(1785575722))) + assert.Equal(t, "2026-08-01T09:15:22Z", timestampString(int64(1785575722))) + assert.Equal(t, "2026-08-01T09:15:22Z", timestampString("2026-08-01T09:15:22Z")) + assert.Empty(t, timestampString(nil)) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go index 2efd578269d..71180968fac 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go @@ -5,6 +5,7 @@ package cmd import ( "fmt" + "strconv" "azureaieval/internal/pkg/eval_api" @@ -73,10 +74,16 @@ func newRunListCommand() *cobra.Command { rows := make([][]string, 0, len(list.Data)) for _, run := range list.Data { - rows = append(rows, []string{run.ID, run.Name, run.Status, summarizeCounts(run.ResultCounts)}) + rows = append(rows, []string{ + run.ID, + timestampString(run.CreatedAt), + run.Status, + sampleCount(run.ResultCounts), + runPassRate(run.ResultCounts), + }) } return emitTable(cmd.OutOrStdout(), - []string{"RUN ID", "NAME", "STATUS", "RESULTS"}, rows) + []string{"RUN", "STARTED", "STATUS", "SAMPLES", "PASS RATE"}, rows) }, } addEvalFlag(cmd, &groupName) @@ -301,3 +308,22 @@ func summarizeCounts(counts *eval_api.EvalRunResultCounts) string { return fmt.Sprintf("%d passed, %d failed, %d errored", counts.Passed, counts.Failed, counts.Errored) } + +// sampleCount is how many rows the run scored, which is what makes two rows of +// `run list` comparable: a rate over 15 samples and one over 200 are not the +// same claim. +func sampleCount(counts *eval_api.EvalRunResultCounts) string { + if counts == nil { + return "" + } + return strconv.Itoa(counts.Total) +} + +// runPassRate is the same passed/total the gate uses, so a row a reader gates +// on cannot disagree with the gate. +func runPassRate(counts *eval_api.EvalRunResultCounts) string { + if counts == nil || counts.Total == 0 { + return "" + } + return formatRate(counts.Passed, counts.Total) +} From a6fa0ec5b828d4eb4c16be99bda689a988ab1027 Mon Sep 17 00:00:00 2001 From: mohessie Date: Thu, 6 Aug 2026 06:15:19 +0300 Subject: [PATCH 126/320] Pin eval.yaml's keys, which are the contract with whoever wrote one The configuration model was the last unaudited section, and its keys turned out to match the spec exactly. What was missing was anything saying so. The keys appeared in tests only as fixture text, so a renamed struct tag would have failed some parse somewhere rather than reporting that a published key changed under everyone who already wrote a file using it. The shapes are read off the yaml tags and compared to the spec's tables: the top level, an eval, the five keys of an evaluators: entry, source:, and the two catalogs. Renaming initialization_parameters fails them; I checked. Casing is asserted across every shape rather than per field, because the rule is that eval.yaml uses the API's snake_case throughout -- one camelCase key would be an exception a reader has to remember, and it is exactly the kind of thing that arrives one field at a time. My first pass had target: carrying a version, which it does not. The spec shows a type and a name, the code has a type and a name, and the assertion was the thing that was wrong. Corrected rather than "fixed" the code to match it. --- .../internal/project/config_keys_test.go | 121 ++++++++++++++++++ 1 file changed, 121 insertions(+) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/project/config_keys_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/config_keys_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/config_keys_test.go new file mode 100644 index 00000000000..ac2f91865f5 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/config_keys_test.go @@ -0,0 +1,121 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package project + +import ( + "reflect" + "strings" + "testing" + + "azureaieval/internal/pkg/evalcore" + + "github.com/stretchr/testify/assert" +) + +// eval.yaml is the file a user writes, so its keys are the contract. They are +// pinned whole rather than exercised through fixtures: a fixture that stops +// parsing says a test broke, not that a published key was renamed under +// everyone who already wrote one. +// +// The spec's configuration model is the source for every list here. Changing +// one means changing both. + +// yamlKeys reads the yaml tag names off a struct, in declaration order. +func yamlKeys(t *testing.T, v any) []string { + t.Helper() + typ := reflect.TypeOf(v) + for typ.Kind() == reflect.Ptr { + typ = typ.Elem() + } + + var keys []string + for i := range typ.NumField() { + tag := typ.Field(i).Tag.Get("yaml") + if tag == "" || tag == "-" { + continue + } + name := strings.Split(tag, ",")[0] + if name == "" { + continue + } + keys = append(keys, name) + } + return keys +} + +// The top level: catalogs first, then the evals defined over them. +func TestEvalConfigKeys(t *testing.T) { + assert.Equal(t, []string{"datasets", "evaluators", "evals"}, + yamlKeys(t, EvalConfig{}), + "the top-level shape is the spec's configuration model") +} + +// An eval names what it evaluates, what it reads, and how to grade it. +func TestEvalKeys(t *testing.T) { + assert.ElementsMatch(t, + []string{ + "name", "id", "description", "dataset", "source", + "evaluation_level", "max_samples", "evaluators", "target", + }, + yamlKeys(t, Eval{})) +} + +// Every entry in an eval's evaluators: list is a map keyed evaluator:, and the +// spec gives that map exactly five keys. +func TestEvaluatorRefKeys(t *testing.T) { + assert.ElementsMatch(t, + []string{"evaluator", "name", "version", "initialization_parameters", "data_mapping"}, + yamlKeys(t, evalcore.EvaluatorRef{}), + "the spec tabulates these five; a sixth is a promise it does not make") +} + +// source: says where rows come from when they are not a dataset. +func TestSourceDeclKeys(t *testing.T) { + assert.ElementsMatch(t, + []string{"type", "lookback_hours", "max_traces", "agent_name", "response_ids", "max_turns"}, + yamlKeys(t, SourceDecl{})) +} + +// The catalogs are named, reusable assets: a name and where it comes from. +func TestCatalogKeys(t *testing.T) { + assert.ElementsMatch(t, []string{"name", "source", "version"}, yamlKeys(t, DatasetDecl{})) + assert.ElementsMatch(t, []string{"name", "source", "version"}, yamlKeys(t, EvaluatorDecl{})) +} + +// The spec's casing table: eval.yaml uses the API's snake_case throughout, so +// a camelCase key would be the one place a reader has to remember an exception. +func TestEveryKeyIsSnakeCase(t *testing.T) { + shapes := map[string]any{ + "EvalConfig": EvalConfig{}, + "Eval": Eval{}, + "SourceDecl": SourceDecl{}, + "Target": Target{}, + "DatasetDecl": DatasetDecl{}, + "EvaluatorDecl": EvaluatorDecl{}, + "EvaluatorRef": evalcore.EvaluatorRef{}, + } + + for name, shape := range shapes { + for _, key := range yamlKeys(t, shape) { + assert.Equalf(t, strings.ToLower(key), key, + "%s.%s is not snake_case; eval.yaml uses the API's spelling throughout", name, key) + assert.NotContainsf(t, key, "-", + "%s.%s uses a dash; the API's convention is underscores", name, key) + } + } +} + +// `target:` always means invoke and `source:` always means where rows come +// from. A trace-backed eval has no target, which is what agent_name under +// source: exists to say. +func TestTargetAndSourceAreDistinct(t *testing.T) { + assert.ElementsMatch(t, []string{"type", "name"}, yamlKeys(t, Target{}), + "the spec's target: is a type and a name; a version there would pin the "+ + "agent an eval invokes, which nothing asks for") + + assert.Contains(t, yamlKeys(t, SourceDecl{}), "agent_name", + "a trace run filters by agent rather than invoking one") + assert.NotContains(t, yamlKeys(t, Target{}), "agent_name", + "the target already names what it invokes") +} From eb4f400b8fea1ccf28a1115961b63424a64d67d7 Mon Sep 17 00:00:00 2001 From: mohessie Date: Thu, 6 Aug 2026 06:32:06 +0300 Subject: [PATCH 127/320] Stop escaping the path twice, which renamed the resource Callers escape the name and version they put in a path, then doRequest assigned that to url.URL.Path -- the decoded field. Serializing re-escaped the percent signs, so a dataset named "my dataset" was fetched, deleted and downloaded as one named "my%20dataset", and the service answered 404 for a dataset the user could see in the list. Ordinary names escape to themselves, which is why this held up: only a name carrying a space or a separator ever went to the wrong place. The path is now set as the raw one, with the decoded form alongside it so the two agree. All four clients had the same line. The test that should have caught it asserted the decoded path held no "/..", which is true of the double-encoded form as well. It now reads the wire form, where the difference between %2F and %252F is visible. --- .../internal/pkg/dataset_api/operations.go | 11 +++++++++- .../internal/pkg/gen_api/operations.go | 11 +++++++++- .../internal/pkg/gen_api/operations_test.go | 22 ++++++++++++++----- .../internal/pkg/dataset_api/operations.go | 11 +++++++++- .../internal/pkg/eval_api/operations.go | 11 +++++++++- .../internal/pkg/eval_api/operations_test.go | 22 ++++++++++++++----- 6 files changed, 74 insertions(+), 14 deletions(-) diff --git a/cli/azd/extensions/azure.ai.dataset/internal/pkg/dataset_api/operations.go b/cli/azd/extensions/azure.ai.dataset/internal/pkg/dataset_api/operations.go index 562515d97b2..a37f97c79c0 100644 --- a/cli/azd/extensions/azure.ai.dataset/internal/pkg/dataset_api/operations.go +++ b/cli/azd/extensions/azure.ai.dataset/internal/pkg/dataset_api/operations.go @@ -542,7 +542,16 @@ func (c *DatasetClient) doRequest( return nil, fmt.Errorf("invalid endpoint URL: %w", err) } - u.Path += path + // Callers escape the name and version they interpolate, so the path is set + // as the raw one. Assigning it to u.Path re-escapes the percent signs, and + // a dataset named "my dataset" then addresses one named "my%20dataset". + escapedPath := u.EscapedPath() + path + decodedPath, err := url.PathUnescape(escapedPath) + if err != nil { + return nil, fmt.Errorf("invalid request path %q: %w", escapedPath, err) + } + u.Path, u.RawPath = decodedPath, escapedPath + q := u.Query() if apiVersion != "" { q.Set("api-version", apiVersion) diff --git a/cli/azd/extensions/azure.ai.dataset/internal/pkg/gen_api/operations.go b/cli/azd/extensions/azure.ai.dataset/internal/pkg/gen_api/operations.go index 5071b2e2e61..7598f290a38 100644 --- a/cli/azd/extensions/azure.ai.dataset/internal/pkg/gen_api/operations.go +++ b/cli/azd/extensions/azure.ai.dataset/internal/pkg/gen_api/operations.go @@ -149,7 +149,16 @@ func (c *Client) doRequest( return nil, fmt.Errorf("invalid endpoint URL: %w", err) } - u.Path += path + // Callers escape the ids they interpolate, so the path is set as the raw + // one. Assigning it to u.Path re-escapes the percent signs, and a job id + // carrying a separator then addresses a literally-named resource. + escapedPath := u.EscapedPath() + path + decodedPath, err := url.PathUnescape(escapedPath) + if err != nil { + return nil, fmt.Errorf("invalid request path %q: %w", escapedPath, err) + } + u.Path, u.RawPath = decodedPath, escapedPath + q := u.Query() if apiVersion != "" { q.Set("api-version", apiVersion) diff --git a/cli/azd/extensions/azure.ai.dataset/internal/pkg/gen_api/operations_test.go b/cli/azd/extensions/azure.ai.dataset/internal/pkg/gen_api/operations_test.go index 765abb961da..e7d02385414 100644 --- a/cli/azd/extensions/azure.ai.dataset/internal/pkg/gen_api/operations_test.go +++ b/cli/azd/extensions/azure.ai.dataset/internal/pkg/gen_api/operations_test.go @@ -22,8 +22,11 @@ import ( type call struct { method string path string - query url.Values - body string + // rawPath is the path as it went over the wire, where escaping is still + // visible. path has been decoded and cannot tell %2F from a separator. + rawPath string + query url.Values + body string } // recorder answers every request with status and body, remembering the last one. @@ -32,7 +35,13 @@ func recorder(t *testing.T, status int, body string) (*Client, *call) { var last call server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { raw, _ := io.ReadAll(r.Body) - last = call{method: r.Method, path: r.URL.Path, query: r.URL.Query(), body: string(raw)} + last = call{ + method: r.Method, + path: r.URL.Path, + rawPath: r.URL.EscapedPath(), + query: r.URL.Query(), + body: string(raw), + } w.Header().Set("Content-Type", "application/json") w.WriteHeader(status) if body != "" { @@ -107,14 +116,17 @@ func TestCreateDataGenerationJob_SendsTheBuiltRequest(t *testing.T) { // An id goes into the path, so one containing a separator has to be escaped or // it silently addresses a different route. +// +// The assertion is on the wire form: the decoded path shows the separators +// again, so it cannot tell a correctly escaped id from an unescaped one. func TestOperations_EscapeIdsInThePath(t *testing.T) { client, last := recorder(t, http.StatusOK, `{"id":"x"}`) _, err := client.GetDataGenerationJob(context.Background(), "dgj/../evil", "v1") require.NoError(t, err) - assert.NotContains(t, last.path, "/../", - "an unescaped id would let a name climb out of its route") + assert.Equal(t, "/data_generation_jobs/dgj%2F..%2Fevil", last.rawPath, + "the id stays one segment; escaping it twice would send %252F and address a differently named job") } func TestOperations_SendTheApiVersion(t *testing.T) { diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/operations.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/operations.go index 427f8c6af00..d6d5323dd1c 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/operations.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/operations.go @@ -542,7 +542,16 @@ func (c *DatasetClient) doRequest( return nil, fmt.Errorf("invalid endpoint URL: %w", err) } - u.Path += path + // Callers escape the name and version they interpolate, so the path is set + // as the raw one. Assigning it to u.Path re-escapes the percent signs, and + // a dataset named "my dataset" then addresses one named "my%20dataset". + escapedPath := u.EscapedPath() + path + decodedPath, err := url.PathUnescape(escapedPath) + if err != nil { + return nil, fmt.Errorf("invalid request path %q: %w", escapedPath, err) + } + u.Path, u.RawPath = decodedPath, escapedPath + q := u.Query() if apiVersion != "" { q.Set("api-version", apiVersion) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/operations.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/operations.go index 0b18f6556a2..100e9d87b6d 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/operations.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/operations.go @@ -517,7 +517,16 @@ func (c *EvalClient) doRequestWithHeaders( return nil, fmt.Errorf("invalid endpoint URL: %w", err) } - u.Path += path + // Callers escape the ids they interpolate, so the path is set as the raw + // one. Assigning it to u.Path re-escapes the percent signs, and an + // evaluator named "my evaluator" then addresses one named "my%20evaluator". + escapedPath := u.EscapedPath() + path + decodedPath, err := url.PathUnescape(escapedPath) + if err != nil { + return nil, fmt.Errorf("invalid request path %q: %w", escapedPath, err) + } + u.Path, u.RawPath = decodedPath, escapedPath + q := u.Query() if apiVersion != "" { q.Set("api-version", apiVersion) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/operations_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/operations_test.go index 0bbccfab675..da7ae585717 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/operations_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/operations_test.go @@ -20,8 +20,11 @@ import ( type call struct { method string path string - query url.Values - body string + // rawPath is the path as it went over the wire, where escaping is still + // visible. path has been decoded and cannot tell %2F from a separator. + rawPath string + query url.Values + body string } // recorder answers every request with status and body, remembering the last one. @@ -30,7 +33,13 @@ func recorder(t *testing.T, status int, body string) (*EvalClient, *call) { var last call client := newRecordingClient(t, func(w http.ResponseWriter, r *http.Request) { raw, _ := io.ReadAll(r.Body) - last = call{method: r.Method, path: r.URL.Path, query: r.URL.Query(), body: string(raw)} + last = call{ + method: r.Method, + path: r.URL.Path, + rawPath: r.URL.EscapedPath(), + query: r.URL.Query(), + body: string(raw), + } w.Header().Set("Content-Type", "application/json") w.WriteHeader(status) if body != "" { @@ -106,14 +115,17 @@ func TestListGenerationJobs_ReadsTheDataEnvelope(t *testing.T) { // An id goes into the path, so one containing a separator has to be escaped or // it silently addresses a different route. +// +// The assertion is on the wire form: the decoded path shows the separators +// again, so it cannot tell a correctly escaped id from an unescaped one. func TestOperations_EscapeIdsInThePath(t *testing.T) { client, last := recorder(t, http.StatusOK, `{"id":"x"}`) _, err := client.GetDataGenerationJob(context.Background(), "dgj/../evil", "v1") require.NoError(t, err) - assert.NotContains(t, last.path, "/../", - "an unescaped id would let a name climb out of its route") + assert.Equal(t, "/data_generation_jobs/dgj%2F..%2Fevil", last.rawPath, + "the id stays one segment; escaping it twice would send %252F and address a differently named job") } // The api-version is what selects the contract; sending the wrong one, or none, From 2c87804f059af555688369c291be73d3bb3eadec Mon Sep 17 00:00:00 2001 From: mohessie Date: Thu, 6 Aug 2026 06:32:06 +0300 Subject: [PATCH 128/320] Cover the dataset client's download, which guesses at what it was given A dataset URI names either the blob or the container holding it, and nothing in the payload says which: isSingleFile is true either way. Uploaded ones end in the file name, generated ones in the container, and reading a container directly answers 409. That guess, its fallback, and the JSONL preference had no tests, so all three were free to rot. Also covers the operation paths, the SAS token surviving the container listing, and the 204 a delete answers with. dataset_api goes from 52.4% to 82.8% in both extensions. --- .../pkg/dataset_api/download_wire_test.go | 255 ++++++++++++++++++ .../pkg/dataset_api/operations_wire_test.go | 205 ++++++++++++++ .../pkg/dataset_api/download_wire_test.go | 255 ++++++++++++++++++ .../pkg/dataset_api/operations_wire_test.go | 205 ++++++++++++++ 4 files changed, 920 insertions(+) create mode 100644 cli/azd/extensions/azure.ai.dataset/internal/pkg/dataset_api/download_wire_test.go create mode 100644 cli/azd/extensions/azure.ai.dataset/internal/pkg/dataset_api/operations_wire_test.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/download_wire_test.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/operations_wire_test.go diff --git a/cli/azd/extensions/azure.ai.dataset/internal/pkg/dataset_api/download_wire_test.go b/cli/azd/extensions/azure.ai.dataset/internal/pkg/dataset_api/download_wire_test.go new file mode 100644 index 00000000000..a8b23386e34 --- /dev/null +++ b/cli/azd/extensions/azure.ai.dataset/internal/pkg/dataset_api/download_wire_test.go @@ -0,0 +1,255 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package dataset_api + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "net/url" + "strings" + "sync" + "testing" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const testAPIVersion = "2025-11-15-preview" + +// blobListing is the shape Azure Blob Storage answers a container list with. +func blobListing(names ...string) string { + var b strings.Builder + b.WriteString(``) + for _, n := range names { + b.WriteString("" + n + "") + } + b.WriteString(``) + return b.String() +} + +// storageServer stands in for both the dataset API and blob storage, recording +// what each leg of a download was asked for. +type storageServer struct { + mu sync.Mutex + + // credential is the sasUri handed back for a download, relative to the + // server's own address. + credentialPath string + // blobs maps a container-relative blob name to its content. + blobs map[string]string + // directBlobStatus is the status a direct GET of credentialPath answers. + directBlobStatus int + + gotListQuery url.Values + gotBlobPaths []string + gotAPIVer []string +} + +func (s *storageServer) start(t *testing.T) (*DatasetClient, *httptest.Server) { + t.Helper() + var srv *httptest.Server + srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + s.mu.Lock() + defer s.mu.Unlock() + + if v := r.URL.Query().Get("api-version"); v != "" { + s.gotAPIVer = append(s.gotAPIVer, v) + } + + switch { + case strings.HasSuffix(r.URL.Path, "/credentials"): + w.Header().Set("Content-Type", "application/json") + require.NoError(t, json.NewEncoder(w).Encode(map[string]any{ + "blobReferenceForConsumption": map[string]any{ + "credential": map[string]any{"sasUri": srv.URL + s.credentialPath + "?sig=secret"}, + }, + })) + + case r.URL.Query().Get("comp") == "list": + s.gotListQuery = r.URL.Query() + names := make([]string, 0, len(s.blobs)) + for n := range s.blobs { + names = append(names, n) + } + w.Header().Set("Content-Type", "application/xml") + _, _ = w.Write([]byte(blobListing(names...))) + + // A direct read of the credential URI itself, keyed under "". + case r.URL.Path == s.credentialPath: + s.gotBlobPaths = append(s.gotBlobPaths, r.URL.Path) + if s.directBlobStatus != 0 { + w.WriteHeader(s.directBlobStatus) + return + } + _, _ = w.Write([]byte(s.blobs[""])) + + default: + s.gotBlobPaths = append(s.gotBlobPaths, r.URL.Path) + body, ok := s.blobs[strings.TrimPrefix(r.URL.Path, s.credentialPath+"/")] + if !ok { + w.WriteHeader(http.StatusNotFound) + return + } + _, _ = w.Write([]byte(body)) + } + })) + t.Cleanup(srv.Close) + + client := NewDatasetClientFromPipeline( + srv.URL, runtime.NewPipeline("test", "v1", runtime.PipelineOptions{}, nil)) + return client, srv +} + +// A dataset that was uploaded names its own file, so it reads in one hop and +// the container must never be listed. +func TestDownloadDatasetContentReadsABlobURIDirectly(t *testing.T) { + server := &storageServer{ + credentialPath: "/c/rows.jsonl", + blobs: map[string]string{"": `{"query":"direct"}`}, + } + client, _ := server.start(t) + + data, err := client.DownloadDatasetContent(context.Background(), "ds", "1.0", testAPIVersion) + require.NoError(t, err) + assert.Equal(t, `{"query":"direct"}`, string(data)) + assert.Nil(t, server.gotListQuery, "a blob URI needs no container listing") +} + +// A generated dataset names the container it was written into, and nothing in +// the payload says so: isSingleFile is true either way. Reading the container +// directly returns a 409, so the blob inside has to be found first. +func TestDownloadDatasetContentListsAContainerURI(t *testing.T) { + server := &storageServer{ + credentialPath: "/generated-container", + blobs: map[string]string{ + "_meta.json": `{"ignored":true}`, + "data.jsonl": `{"query":"from the container"}`, + }, + } + client, _ := server.start(t) + + data, err := client.DownloadDatasetContent(context.Background(), "ds", "1.0", testAPIVersion) + require.NoError(t, err) + assert.Equal(t, `{"query":"from the container"}`, string(data), + "the JSONL is chosen over the metadata sitting beside it") + require.NotNil(t, server.gotListQuery) + assert.Equal(t, "container", server.gotListQuery.Get("restype")) + assert.Equal(t, "secret", server.gotListQuery.Get("sig"), + "the listing must keep the SAS token, or storage answers 403") +} + +// A URI can name a file and still be a container — the extension is a guess, +// not a fact. When the direct read fails the listing is the fallback, so the +// download succeeds rather than surfacing the first status. +func TestDownloadDatasetContentFallsBackWhenTheBlobReadFails(t *testing.T) { + server := &storageServer{ + credentialPath: "/c/looks.jsonl", + directBlobStatus: http.StatusConflict, + blobs: map[string]string{"real.jsonl": `{"query":"found by listing"}`}, + } + client, _ := server.start(t) + + data, err := client.DownloadDatasetContent(context.Background(), "ds", "1.0", testAPIVersion) + require.NoError(t, err, "a 409 on the direct read is the container case, not a failure") + assert.Equal(t, `{"query":"found by listing"}`, string(data)) + assert.NotNil(t, server.gotListQuery) +} + +// An empty container is a dataset with nothing to read, and saying so beats +// returning empty content that looks like a dataset with no rows. +func TestDownloadDatasetContentReportsAnEmptyContainer(t *testing.T) { + server := &storageServer{credentialPath: "/empty", blobs: map[string]string{}} + client, _ := server.start(t) + + _, err := client.DownloadDatasetContent(context.Background(), "ds", "1.0", testAPIVersion) + require.Error(t, err) + assert.Contains(t, err.Error(), "no downloadable file") +} + +// The URI carries no SAS of its own, so a credential that resolves to nothing +// has to be reported here rather than as an unauthorized read later. +func TestDownloadDatasetContentRequiresADownloadURI(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{}`)) + })) + t.Cleanup(srv.Close) + + client := NewDatasetClientFromPipeline( + srv.URL, runtime.NewPipeline("test", "v1", runtime.PipelineOptions{}, nil)) + + _, err := client.DownloadDatasetContent(context.Background(), "ds", "1.0", testAPIVersion) + require.Error(t, err) + assert.Contains(t, err.Error(), "no download URI") +} + +// The blob name is appended to the container path, and the SAS token stays on +// the query where storage expects it. +func TestDownloadBlobKeepsTheSASToken(t *testing.T) { + var gotPath, gotSig string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotPath, gotSig = r.URL.Path, r.URL.Query().Get("sig") + _, _ = w.Write([]byte("rows")) + })) + t.Cleanup(srv.Close) + + client := NewDatasetClientFromPipeline( + srv.URL, runtime.NewPipeline("test", "v1", runtime.PipelineOptions{}, nil)) + + data, err := client.DownloadBlob(context.Background(), srv.URL+"/container?sig=secret", "data.jsonl") + require.NoError(t, err) + assert.Equal(t, "rows", string(data)) + assert.Equal(t, "/container/data.jsonl", gotPath) + assert.Equal(t, "secret", gotSig) +} + +// A storage failure has to name the blob, since the container holds several +// and the status alone does not say which one was refused. +func TestDownloadBlobReportsTheStatusAndName(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusForbidden) + })) + t.Cleanup(srv.Close) + + client := NewDatasetClientFromPipeline( + srv.URL, runtime.NewPipeline("test", "v1", runtime.PipelineOptions{}, nil)) + + _, err := client.DownloadBlob(context.Background(), srv.URL+"/c", "data.jsonl") + require.Error(t, err) + assert.Contains(t, err.Error(), "403") + assert.Contains(t, err.Error(), "data.jsonl") +} + +// A malformed URI is the caller's mistake, and it is worth catching before a +// request goes out against a half-parsed address. +func TestBlobOperationsRejectAnUnparseableURI(t *testing.T) { + client := NewDatasetClientFromPipeline( + "https://example", runtime.NewPipeline("test", "v1", runtime.PipelineOptions{}, nil)) + + _, err := client.DownloadBlob(context.Background(), "://nope", "x.jsonl") + require.Error(t, err) + + _, err = client.ListContainerBlobs(context.Background(), "://nope") + require.Error(t, err) + + err = client.UploadBlob(context.Background(), "://nope", "x.jsonl", []byte("{}")) + require.Error(t, err) +} + +// Storage answers a listing in XML, and a shape that does not parse yields no +// names rather than a panic. +func TestParseBlobNames(t *testing.T) { + assert.Equal(t, []string{"a.jsonl", "b.json"}, + parseBlobNames(blobListing("a.jsonl", "b.json"))) + assert.Empty(t, parseBlobNames(blobListing())) + assert.Empty(t, parseBlobNames("not xml at all"), + "an unreadable listing is an empty one, not a crash") + assert.Empty(t, parseBlobNames( + ``), + "a nameless blob cannot be downloaded, so it is not offered") +} diff --git a/cli/azd/extensions/azure.ai.dataset/internal/pkg/dataset_api/operations_wire_test.go b/cli/azd/extensions/azure.ai.dataset/internal/pkg/dataset_api/operations_wire_test.go new file mode 100644 index 00000000000..00f4bb8ff19 --- /dev/null +++ b/cli/azd/extensions/azure.ai.dataset/internal/pkg/dataset_api/operations_wire_test.go @@ -0,0 +1,205 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package dataset_api + +import ( + "context" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// fakeCredential satisfies the constructor without reaching for a real token. +type fakeCredential struct{} + +func (fakeCredential) GetToken(context.Context, policy.TokenRequestOptions) (azcore.AccessToken, error) { + return azcore.AccessToken{Token: "fake", ExpiresOn: time.Now().Add(time.Hour)}, nil +} + +// recordedCall is one request the client made, as the service saw it. +type recordedCall struct { + method string + path string + rawPath string + apiVersion string +} + +// recordingDatasetClient answers every request with body and status, recording +// what was asked. Retries are off so a deliberate failure is one call. +func recordingDatasetClient(t *testing.T, status int, body string) (*DatasetClient, *[]recordedCall) { + t.Helper() + calls := &[]recordedCall{} + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + *calls = append(*calls, recordedCall{ + method: r.Method, + path: r.URL.Path, + rawPath: r.URL.EscapedPath(), + apiVersion: r.URL.Query().Get("api-version"), + }) + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + if body != "" { + _, _ = w.Write([]byte(body)) + } + })) + t.Cleanup(srv.Close) + + pipeline := runtime.NewPipeline("test", "v1", runtime.PipelineOptions{}, + &policy.ClientOptions{Retry: policy.RetryOptions{MaxRetries: -1}}) + return NewDatasetClientFromPipeline(srv.URL, pipeline), calls +} + +// The paths are the service contract, and a wrong one costs a round trip to +// find out. Each is pinned against the shape the API documents. +func TestDatasetOperationPaths(t *testing.T) { + cases := []struct { + name string + call func(c *DatasetClient) error + wantMethod string + wantPath string + }{ + { + name: "list", + call: func(c *DatasetClient) error { _, err := c.ListDatasets(t.Context(), testAPIVersion); return err }, + wantMethod: http.MethodGet, + wantPath: "/datasets", + }, + { + name: "list versions", + call: func(c *DatasetClient) error { + _, err := c.ListDatasetVersions(t.Context(), "ds", testAPIVersion) + return err + }, + wantMethod: http.MethodGet, + wantPath: "/datasets/ds/versions", + }, + { + name: "get", + call: func(c *DatasetClient) error { + _, err := c.GetDataset(t.Context(), "ds", "1.0", testAPIVersion) + return err + }, + wantMethod: http.MethodGet, + wantPath: "/datasets/ds/versions/1.0", + }, + { + name: "credential", + call: func(c *DatasetClient) error { + _, err := c.GetDatasetCredential(t.Context(), "ds", "1.0", testAPIVersion) + return err + }, + wantMethod: http.MethodPost, + wantPath: "/datasets/ds/versions/1.0/credentials", + }, + { + name: "start pending upload", + call: func(c *DatasetClient) error { + _, err := c.StartPendingUpload(t.Context(), "ds", "1.0", testAPIVersion) + return err + }, + wantMethod: http.MethodPost, + wantPath: "/datasets/ds/versions/1.0/startPendingUpload", + }, + { + name: "finalize", + call: func(c *DatasetClient) error { + _, err := c.FinalizeDatasetVersion(t.Context(), "ds", "1.0", "https://x/y.jsonl", testAPIVersion) + return err + }, + wantMethod: http.MethodPut, + wantPath: "/datasets/ds/versions/1.0", + }, + { + name: "create", + call: func(c *DatasetClient) error { + _, err := c.CreateDataset(t.Context(), &CreateDatasetRequest{Name: "ds"}, testAPIVersion) + return err + }, + wantMethod: http.MethodPost, + wantPath: "/datasets", + }, + { + name: "delete", + call: func(c *DatasetClient) error { return c.DeleteDatasetVersion(t.Context(), "ds", "1.0", testAPIVersion) }, + wantMethod: http.MethodDelete, + wantPath: "/datasets/ds/versions/1.0", + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + client, calls := recordingDatasetClient(t, http.StatusOK, `{"name":"ds","version":"1.0","value":[]}`) + require.NoError(t, tc.call(client)) + require.Len(t, *calls, 1) + assert.Equal(t, tc.wantMethod, (*calls)[0].method) + assert.Equal(t, tc.wantPath, (*calls)[0].path) + assert.Equal(t, testAPIVersion, (*calls)[0].apiVersion, + "the service rejects a request that names no api-version") + }) + } +} + +// A name is caller-supplied and a version can be anything the author wrote, so +// both are escaped rather than pasted into the path. +func TestDatasetPathsEscapeNameAndVersion(t *testing.T) { + client, calls := recordingDatasetClient(t, http.StatusOK, `{}`) + _, err := client.GetDataset(t.Context(), "my dataset/v", "1.0 beta", testAPIVersion) + require.NoError(t, err) + + require.Len(t, *calls, 1) + assert.Equal(t, "/datasets/my%20dataset%2Fv/versions/1.0%20beta", (*calls)[0].rawPath, + "an unescaped slash would address a different resource entirely") +} + +// A delete answers 204 with nothing in it, which must not read as a failure to +// parse a body that was never promised. +func TestDeleteDatasetVersionAcceptsNoContent(t *testing.T) { + client, calls := recordingDatasetClient(t, http.StatusNoContent, "") + require.NoError(t, client.DeleteDatasetVersion(t.Context(), "ds", "1.0", testAPIVersion)) + assert.Len(t, *calls, 1) +} + +// The listing arrives wrapped in a value envelope; reading it flat yields an +// empty list rather than an error, which looks like a project with no datasets. +func TestListDatasetsReadsTheValueEnvelope(t *testing.T) { + client, _ := recordingDatasetClient(t, http.StatusOK, + `{"value":[{"name":"a","version":"1.0"},{"name":"b","version":"2.0"}]}`) + + list, err := client.ListDatasets(t.Context(), testAPIVersion) + require.NoError(t, err) + require.Len(t, list.Value, 2) + assert.Equal(t, "a", list.Value[0].Name) + assert.Equal(t, "2.0", list.Value[1].Version) +} + +// A failure has to surface as one, since the caller otherwise proceeds with a +// zero-valued dataset and fails somewhere further away. +func TestDatasetOperationsSurfaceServiceFailures(t *testing.T) { + client, _ := recordingDatasetClient(t, http.StatusNotFound, `{"error":{"code":"NotFound"}}`) + + _, err := client.GetDataset(t.Context(), "missing", "1.0", testAPIVersion) + require.Error(t, err) + + err = client.DeleteDatasetVersion(t.Context(), "missing", "1.0", testAPIVersion) + require.Error(t, err) + + _, err = client.ListDatasets(t.Context(), testAPIVersion) + require.Error(t, err) +} + +// The constructor has to build a usable client — it wires the auth policies +// the live service needs, and nothing else exercises that path. +func TestNewDatasetClient(t *testing.T) { + client := NewDatasetClient("https://example.services.ai.azure.com/api/projects/p", fakeCredential{}) + require.NotNil(t, client) + assert.Equal(t, "https://example.services.ai.azure.com/api/projects/p", client.endpoint) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/download_wire_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/download_wire_test.go new file mode 100644 index 00000000000..a8b23386e34 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/download_wire_test.go @@ -0,0 +1,255 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package dataset_api + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "net/url" + "strings" + "sync" + "testing" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const testAPIVersion = "2025-11-15-preview" + +// blobListing is the shape Azure Blob Storage answers a container list with. +func blobListing(names ...string) string { + var b strings.Builder + b.WriteString(``) + for _, n := range names { + b.WriteString("" + n + "") + } + b.WriteString(``) + return b.String() +} + +// storageServer stands in for both the dataset API and blob storage, recording +// what each leg of a download was asked for. +type storageServer struct { + mu sync.Mutex + + // credential is the sasUri handed back for a download, relative to the + // server's own address. + credentialPath string + // blobs maps a container-relative blob name to its content. + blobs map[string]string + // directBlobStatus is the status a direct GET of credentialPath answers. + directBlobStatus int + + gotListQuery url.Values + gotBlobPaths []string + gotAPIVer []string +} + +func (s *storageServer) start(t *testing.T) (*DatasetClient, *httptest.Server) { + t.Helper() + var srv *httptest.Server + srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + s.mu.Lock() + defer s.mu.Unlock() + + if v := r.URL.Query().Get("api-version"); v != "" { + s.gotAPIVer = append(s.gotAPIVer, v) + } + + switch { + case strings.HasSuffix(r.URL.Path, "/credentials"): + w.Header().Set("Content-Type", "application/json") + require.NoError(t, json.NewEncoder(w).Encode(map[string]any{ + "blobReferenceForConsumption": map[string]any{ + "credential": map[string]any{"sasUri": srv.URL + s.credentialPath + "?sig=secret"}, + }, + })) + + case r.URL.Query().Get("comp") == "list": + s.gotListQuery = r.URL.Query() + names := make([]string, 0, len(s.blobs)) + for n := range s.blobs { + names = append(names, n) + } + w.Header().Set("Content-Type", "application/xml") + _, _ = w.Write([]byte(blobListing(names...))) + + // A direct read of the credential URI itself, keyed under "". + case r.URL.Path == s.credentialPath: + s.gotBlobPaths = append(s.gotBlobPaths, r.URL.Path) + if s.directBlobStatus != 0 { + w.WriteHeader(s.directBlobStatus) + return + } + _, _ = w.Write([]byte(s.blobs[""])) + + default: + s.gotBlobPaths = append(s.gotBlobPaths, r.URL.Path) + body, ok := s.blobs[strings.TrimPrefix(r.URL.Path, s.credentialPath+"/")] + if !ok { + w.WriteHeader(http.StatusNotFound) + return + } + _, _ = w.Write([]byte(body)) + } + })) + t.Cleanup(srv.Close) + + client := NewDatasetClientFromPipeline( + srv.URL, runtime.NewPipeline("test", "v1", runtime.PipelineOptions{}, nil)) + return client, srv +} + +// A dataset that was uploaded names its own file, so it reads in one hop and +// the container must never be listed. +func TestDownloadDatasetContentReadsABlobURIDirectly(t *testing.T) { + server := &storageServer{ + credentialPath: "/c/rows.jsonl", + blobs: map[string]string{"": `{"query":"direct"}`}, + } + client, _ := server.start(t) + + data, err := client.DownloadDatasetContent(context.Background(), "ds", "1.0", testAPIVersion) + require.NoError(t, err) + assert.Equal(t, `{"query":"direct"}`, string(data)) + assert.Nil(t, server.gotListQuery, "a blob URI needs no container listing") +} + +// A generated dataset names the container it was written into, and nothing in +// the payload says so: isSingleFile is true either way. Reading the container +// directly returns a 409, so the blob inside has to be found first. +func TestDownloadDatasetContentListsAContainerURI(t *testing.T) { + server := &storageServer{ + credentialPath: "/generated-container", + blobs: map[string]string{ + "_meta.json": `{"ignored":true}`, + "data.jsonl": `{"query":"from the container"}`, + }, + } + client, _ := server.start(t) + + data, err := client.DownloadDatasetContent(context.Background(), "ds", "1.0", testAPIVersion) + require.NoError(t, err) + assert.Equal(t, `{"query":"from the container"}`, string(data), + "the JSONL is chosen over the metadata sitting beside it") + require.NotNil(t, server.gotListQuery) + assert.Equal(t, "container", server.gotListQuery.Get("restype")) + assert.Equal(t, "secret", server.gotListQuery.Get("sig"), + "the listing must keep the SAS token, or storage answers 403") +} + +// A URI can name a file and still be a container — the extension is a guess, +// not a fact. When the direct read fails the listing is the fallback, so the +// download succeeds rather than surfacing the first status. +func TestDownloadDatasetContentFallsBackWhenTheBlobReadFails(t *testing.T) { + server := &storageServer{ + credentialPath: "/c/looks.jsonl", + directBlobStatus: http.StatusConflict, + blobs: map[string]string{"real.jsonl": `{"query":"found by listing"}`}, + } + client, _ := server.start(t) + + data, err := client.DownloadDatasetContent(context.Background(), "ds", "1.0", testAPIVersion) + require.NoError(t, err, "a 409 on the direct read is the container case, not a failure") + assert.Equal(t, `{"query":"found by listing"}`, string(data)) + assert.NotNil(t, server.gotListQuery) +} + +// An empty container is a dataset with nothing to read, and saying so beats +// returning empty content that looks like a dataset with no rows. +func TestDownloadDatasetContentReportsAnEmptyContainer(t *testing.T) { + server := &storageServer{credentialPath: "/empty", blobs: map[string]string{}} + client, _ := server.start(t) + + _, err := client.DownloadDatasetContent(context.Background(), "ds", "1.0", testAPIVersion) + require.Error(t, err) + assert.Contains(t, err.Error(), "no downloadable file") +} + +// The URI carries no SAS of its own, so a credential that resolves to nothing +// has to be reported here rather than as an unauthorized read later. +func TestDownloadDatasetContentRequiresADownloadURI(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{}`)) + })) + t.Cleanup(srv.Close) + + client := NewDatasetClientFromPipeline( + srv.URL, runtime.NewPipeline("test", "v1", runtime.PipelineOptions{}, nil)) + + _, err := client.DownloadDatasetContent(context.Background(), "ds", "1.0", testAPIVersion) + require.Error(t, err) + assert.Contains(t, err.Error(), "no download URI") +} + +// The blob name is appended to the container path, and the SAS token stays on +// the query where storage expects it. +func TestDownloadBlobKeepsTheSASToken(t *testing.T) { + var gotPath, gotSig string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotPath, gotSig = r.URL.Path, r.URL.Query().Get("sig") + _, _ = w.Write([]byte("rows")) + })) + t.Cleanup(srv.Close) + + client := NewDatasetClientFromPipeline( + srv.URL, runtime.NewPipeline("test", "v1", runtime.PipelineOptions{}, nil)) + + data, err := client.DownloadBlob(context.Background(), srv.URL+"/container?sig=secret", "data.jsonl") + require.NoError(t, err) + assert.Equal(t, "rows", string(data)) + assert.Equal(t, "/container/data.jsonl", gotPath) + assert.Equal(t, "secret", gotSig) +} + +// A storage failure has to name the blob, since the container holds several +// and the status alone does not say which one was refused. +func TestDownloadBlobReportsTheStatusAndName(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusForbidden) + })) + t.Cleanup(srv.Close) + + client := NewDatasetClientFromPipeline( + srv.URL, runtime.NewPipeline("test", "v1", runtime.PipelineOptions{}, nil)) + + _, err := client.DownloadBlob(context.Background(), srv.URL+"/c", "data.jsonl") + require.Error(t, err) + assert.Contains(t, err.Error(), "403") + assert.Contains(t, err.Error(), "data.jsonl") +} + +// A malformed URI is the caller's mistake, and it is worth catching before a +// request goes out against a half-parsed address. +func TestBlobOperationsRejectAnUnparseableURI(t *testing.T) { + client := NewDatasetClientFromPipeline( + "https://example", runtime.NewPipeline("test", "v1", runtime.PipelineOptions{}, nil)) + + _, err := client.DownloadBlob(context.Background(), "://nope", "x.jsonl") + require.Error(t, err) + + _, err = client.ListContainerBlobs(context.Background(), "://nope") + require.Error(t, err) + + err = client.UploadBlob(context.Background(), "://nope", "x.jsonl", []byte("{}")) + require.Error(t, err) +} + +// Storage answers a listing in XML, and a shape that does not parse yields no +// names rather than a panic. +func TestParseBlobNames(t *testing.T) { + assert.Equal(t, []string{"a.jsonl", "b.json"}, + parseBlobNames(blobListing("a.jsonl", "b.json"))) + assert.Empty(t, parseBlobNames(blobListing())) + assert.Empty(t, parseBlobNames("not xml at all"), + "an unreadable listing is an empty one, not a crash") + assert.Empty(t, parseBlobNames( + ``), + "a nameless blob cannot be downloaded, so it is not offered") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/operations_wire_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/operations_wire_test.go new file mode 100644 index 00000000000..00f4bb8ff19 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/operations_wire_test.go @@ -0,0 +1,205 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package dataset_api + +import ( + "context" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// fakeCredential satisfies the constructor without reaching for a real token. +type fakeCredential struct{} + +func (fakeCredential) GetToken(context.Context, policy.TokenRequestOptions) (azcore.AccessToken, error) { + return azcore.AccessToken{Token: "fake", ExpiresOn: time.Now().Add(time.Hour)}, nil +} + +// recordedCall is one request the client made, as the service saw it. +type recordedCall struct { + method string + path string + rawPath string + apiVersion string +} + +// recordingDatasetClient answers every request with body and status, recording +// what was asked. Retries are off so a deliberate failure is one call. +func recordingDatasetClient(t *testing.T, status int, body string) (*DatasetClient, *[]recordedCall) { + t.Helper() + calls := &[]recordedCall{} + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + *calls = append(*calls, recordedCall{ + method: r.Method, + path: r.URL.Path, + rawPath: r.URL.EscapedPath(), + apiVersion: r.URL.Query().Get("api-version"), + }) + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + if body != "" { + _, _ = w.Write([]byte(body)) + } + })) + t.Cleanup(srv.Close) + + pipeline := runtime.NewPipeline("test", "v1", runtime.PipelineOptions{}, + &policy.ClientOptions{Retry: policy.RetryOptions{MaxRetries: -1}}) + return NewDatasetClientFromPipeline(srv.URL, pipeline), calls +} + +// The paths are the service contract, and a wrong one costs a round trip to +// find out. Each is pinned against the shape the API documents. +func TestDatasetOperationPaths(t *testing.T) { + cases := []struct { + name string + call func(c *DatasetClient) error + wantMethod string + wantPath string + }{ + { + name: "list", + call: func(c *DatasetClient) error { _, err := c.ListDatasets(t.Context(), testAPIVersion); return err }, + wantMethod: http.MethodGet, + wantPath: "/datasets", + }, + { + name: "list versions", + call: func(c *DatasetClient) error { + _, err := c.ListDatasetVersions(t.Context(), "ds", testAPIVersion) + return err + }, + wantMethod: http.MethodGet, + wantPath: "/datasets/ds/versions", + }, + { + name: "get", + call: func(c *DatasetClient) error { + _, err := c.GetDataset(t.Context(), "ds", "1.0", testAPIVersion) + return err + }, + wantMethod: http.MethodGet, + wantPath: "/datasets/ds/versions/1.0", + }, + { + name: "credential", + call: func(c *DatasetClient) error { + _, err := c.GetDatasetCredential(t.Context(), "ds", "1.0", testAPIVersion) + return err + }, + wantMethod: http.MethodPost, + wantPath: "/datasets/ds/versions/1.0/credentials", + }, + { + name: "start pending upload", + call: func(c *DatasetClient) error { + _, err := c.StartPendingUpload(t.Context(), "ds", "1.0", testAPIVersion) + return err + }, + wantMethod: http.MethodPost, + wantPath: "/datasets/ds/versions/1.0/startPendingUpload", + }, + { + name: "finalize", + call: func(c *DatasetClient) error { + _, err := c.FinalizeDatasetVersion(t.Context(), "ds", "1.0", "https://x/y.jsonl", testAPIVersion) + return err + }, + wantMethod: http.MethodPut, + wantPath: "/datasets/ds/versions/1.0", + }, + { + name: "create", + call: func(c *DatasetClient) error { + _, err := c.CreateDataset(t.Context(), &CreateDatasetRequest{Name: "ds"}, testAPIVersion) + return err + }, + wantMethod: http.MethodPost, + wantPath: "/datasets", + }, + { + name: "delete", + call: func(c *DatasetClient) error { return c.DeleteDatasetVersion(t.Context(), "ds", "1.0", testAPIVersion) }, + wantMethod: http.MethodDelete, + wantPath: "/datasets/ds/versions/1.0", + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + client, calls := recordingDatasetClient(t, http.StatusOK, `{"name":"ds","version":"1.0","value":[]}`) + require.NoError(t, tc.call(client)) + require.Len(t, *calls, 1) + assert.Equal(t, tc.wantMethod, (*calls)[0].method) + assert.Equal(t, tc.wantPath, (*calls)[0].path) + assert.Equal(t, testAPIVersion, (*calls)[0].apiVersion, + "the service rejects a request that names no api-version") + }) + } +} + +// A name is caller-supplied and a version can be anything the author wrote, so +// both are escaped rather than pasted into the path. +func TestDatasetPathsEscapeNameAndVersion(t *testing.T) { + client, calls := recordingDatasetClient(t, http.StatusOK, `{}`) + _, err := client.GetDataset(t.Context(), "my dataset/v", "1.0 beta", testAPIVersion) + require.NoError(t, err) + + require.Len(t, *calls, 1) + assert.Equal(t, "/datasets/my%20dataset%2Fv/versions/1.0%20beta", (*calls)[0].rawPath, + "an unescaped slash would address a different resource entirely") +} + +// A delete answers 204 with nothing in it, which must not read as a failure to +// parse a body that was never promised. +func TestDeleteDatasetVersionAcceptsNoContent(t *testing.T) { + client, calls := recordingDatasetClient(t, http.StatusNoContent, "") + require.NoError(t, client.DeleteDatasetVersion(t.Context(), "ds", "1.0", testAPIVersion)) + assert.Len(t, *calls, 1) +} + +// The listing arrives wrapped in a value envelope; reading it flat yields an +// empty list rather than an error, which looks like a project with no datasets. +func TestListDatasetsReadsTheValueEnvelope(t *testing.T) { + client, _ := recordingDatasetClient(t, http.StatusOK, + `{"value":[{"name":"a","version":"1.0"},{"name":"b","version":"2.0"}]}`) + + list, err := client.ListDatasets(t.Context(), testAPIVersion) + require.NoError(t, err) + require.Len(t, list.Value, 2) + assert.Equal(t, "a", list.Value[0].Name) + assert.Equal(t, "2.0", list.Value[1].Version) +} + +// A failure has to surface as one, since the caller otherwise proceeds with a +// zero-valued dataset and fails somewhere further away. +func TestDatasetOperationsSurfaceServiceFailures(t *testing.T) { + client, _ := recordingDatasetClient(t, http.StatusNotFound, `{"error":{"code":"NotFound"}}`) + + _, err := client.GetDataset(t.Context(), "missing", "1.0", testAPIVersion) + require.Error(t, err) + + err = client.DeleteDatasetVersion(t.Context(), "missing", "1.0", testAPIVersion) + require.Error(t, err) + + _, err = client.ListDatasets(t.Context(), testAPIVersion) + require.Error(t, err) +} + +// The constructor has to build a usable client — it wires the auth policies +// the live service needs, and nothing else exercises that path. +func TestNewDatasetClient(t *testing.T) { + client := NewDatasetClient("https://example.services.ai.azure.com/api/projects/p", fakeCredential{}) + require.NotNil(t, client) + assert.Equal(t, "https://example.services.ai.azure.com/api/projects/p", client.endpoint) +} From 59984737e719a0163cbf75043766518b6a21442e Mon Sep 17 00:00:00 2001 From: mohessie Date: Thu, 6 Aug 2026 06:40:39 +0300 Subject: [PATCH 129/320] Cover the poller and the evaluator contract, where waiting is the feature The poller is what a run spends its time in, and every decision it makes was untested: which statuses end it, that "cancelled" and "canceled" both do, that a 429 is worth retrying and a 404 is not, and that a cancelled job is finished without having succeeded. Getting any of those wrong either hangs the CLI for ten minutes or reports a job that failed as one that worked. Also pins the two cancel routes apart. Cancelling a run works only when the body is empty -- a body makes it an update that rewrites the run's status and counters -- while the generation-job cancel next door requires an empty object. Nothing said so, and they sit close enough to be unified. eval_api goes from 48.3% to 70.0%. --- .../internal/pkg/eval_api/evaluators_test.go | 195 +++++++++++++++ .../internal/pkg/eval_api/poller_test.go | 228 ++++++++++++++++++ 2 files changed, 423 insertions(+) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/evaluators_test.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/poller_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/evaluators_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/evaluators_test.go new file mode 100644 index 00000000000..8c84fb95e4e --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/evaluators_test.go @@ -0,0 +1,195 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package eval_api + +import ( + "context" + "encoding/json" + "net/http" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// The listing carries each evaluator's contract so a request can be shaped to +// match it. Reading that contract wrong means shaping the request wrong and +// taking a service-side rejection instead. +func TestEvaluatorSummaryReadsTheContract(t *testing.T) { + var summary EvaluatorSummary + require.NoError(t, json.Unmarshal([]byte(`{ + "name": "relevance", + "version": "3", + "evaluator_type": "Builtin", + "supported_evaluation_levels": ["Run", "Turn"], + "definition": { + "data_schema": {"type":"object","required":["query"], + "properties":{"query":{},"response":{},"context":{}}}, + "init_parameters": {"type":"object","properties":{"model_config":{},"threshold":{}}} + } + }`), &summary)) + + assert.Equal(t, "Builtin", summary.Type()) + assert.Equal(t, []string{"context", "query", "response"}, summary.DataSchema().PropertyNames(), + "sorted, so a listing does not reorder itself between runs") + assert.True(t, summary.DataSchema().Accepts("response")) + assert.False(t, summary.DataSchema().Accepts("ground_truth")) + assert.Equal(t, []string{"model_config", "threshold"}, summary.InitSchema().PropertyNames()) +} + +// The kind arrives under two names depending on the route. Reading only one +// leaves the type blank, which then reads as a custom evaluator. +func TestEvaluatorTypeAcceptsEitherSpelling(t *testing.T) { + spelled := EvaluatorSummary{EvaluatorType: "Builtin"} + aliased := EvaluatorSummary{TypeAlias: "Builtin"} + + assert.Equal(t, "Builtin", spelled.Type()) + assert.Equal(t, "Builtin", aliased.Type()) + assert.Empty(t, (&EvaluatorSummary{}).Type()) +} + +// An evaluator that declares no levels runs at any of them. Treating an empty +// list as "supports nothing" would reject every evaluator the listing does not +// describe fully. +func TestSupportsLevel(t *testing.T) { + constrained := EvaluatorSummary{SupportedEvaluationLevels: []string{"Run", "Turn"}} + assert.True(t, constrained.SupportsLevel("Run")) + assert.True(t, constrained.SupportsLevel("run"), "the level is matched without regard to case") + assert.False(t, constrained.SupportsLevel("Conversation")) + assert.True(t, constrained.SupportsLevel(""), "asking about no level is not a constraint") + + unconstrained := EvaluatorSummary{} + assert.True(t, unconstrained.SupportsLevel("Conversation"), + "an evaluator that declares no levels is unconstrained, not unusable") +} + +// A missing definition has to read as "not described", not crash the caller +// that asked what an evaluator accepts. +func TestEvaluatorSchemasTolerateAnAbsentDefinition(t *testing.T) { + var absent *EvaluatorSummary + assert.Nil(t, absent.DataSchema()) + assert.Nil(t, absent.InitSchema()) + + bare := EvaluatorSummary{Name: "custom"} + assert.Nil(t, bare.DataSchema()) + assert.Nil(t, bare.InitSchema()) + + var noSchema *JSONSchema + assert.Nil(t, noSchema.PropertyNames()) + assert.False(t, noSchema.Accepts("query")) + assert.False(t, (&JSONSchema{}).Accepts("query")) +} + +// The index is how an eval.yaml entry is matched to what the project offers. +func TestByName(t *testing.T) { + list := &EvaluatorListResponse{Value: []EvaluatorSummary{ + {Name: "relevance", Version: "3"}, + {Name: "coherence", Version: "1"}, + }} + + index := list.ByName() + require.Len(t, index, 2) + assert.Equal(t, "3", index["relevance"].Version) + assert.Nil(t, index["missing"]) + + var absent *EvaluatorListResponse + assert.Nil(t, absent.ByName()) +} + +// Listing built-ins is a filter on the same route, and dropping the parameter +// returns the project's custom evaluators mixed in. +func TestListEvaluatorsFiltersByType(t *testing.T) { + client, last := recorder(t, http.StatusOK, `{"value":[{"name":"relevance"}]}`) + + _, err := client.ListEvaluators(context.Background(), EvaluatorTypeBuiltin, "v1") + require.NoError(t, err) + assert.Equal(t, "/evaluators", last.path) + assert.Equal(t, "Builtin", last.query.Get("type")) + + _, err = client.ListEvaluators(context.Background(), "", "v1") + require.NoError(t, err) + assert.Empty(t, last.query.Get("type"), "no filter asks for everything") +} + +// This route cancels only when the body is empty, and updates the run's status +// and counters when it is not. The generation-job cancel next door requires an +// empty object, so the two are easy to unify into a bug that silently rewrites +// a run instead of stopping it. +func TestCancelOpenAIEvalRunSendsNoBody(t *testing.T) { + client, last := recorder(t, http.StatusOK, `{"id":"run_1","status":"canceled"}`) + + _, err := client.CancelOpenAIEvalRun(context.Background(), "eval_1", "run_1") + + require.NoError(t, err) + assert.Equal(t, http.MethodPost, last.method) + assert.Equal(t, "/openai/v1/evals/eval_1/runs/run_1", last.path) + assert.Empty(t, last.body, "a body turns this cancel into an update") +} + +// The run routes are OpenAI-compatible and carry no api-version; sending one +// is answered by a different contract than the client parses. +func TestRunRoutesAndTheirPaths(t *testing.T) { + cases := []struct { + name string + call func(c *EvalClient) error + wantMethod string + wantPath string + }{ + { + name: "delete run", + call: func(c *EvalClient) error { return c.DeleteOpenAIEvalRun(context.Background(), "eval_1", "run_1") }, + wantMethod: http.MethodDelete, + wantPath: "/openai/v1/evals/eval_1/runs/run_1", + }, + { + name: "list output items", + call: func(c *EvalClient) error { + _, err := c.ListOutputItems(context.Background(), "eval_1", "run_1", 0) + return err + }, + wantMethod: http.MethodGet, + wantPath: "/openai/v1/evals/eval_1/runs/run_1/output_items", + }, + { + name: "get output item", + call: func(c *EvalClient) error { + _, err := c.GetOutputItem(context.Background(), "eval_1", "run_1", "item_1") + return err + }, + wantMethod: http.MethodGet, + wantPath: "/openai/v1/evals/eval_1/runs/run_1/output_items/item_1", + }, + { + name: "delete evaluator version", + call: func(c *EvalClient) error { + return c.DeleteEvaluatorVersion(context.Background(), "custom", "2", "v1") + }, + wantMethod: http.MethodDelete, + wantPath: "/evaluators/custom/versions/2", + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + client, last := recorder(t, http.StatusOK, `{"data":[]}`) + require.NoError(t, tc.call(client)) + assert.Equal(t, tc.wantMethod, last.method) + assert.Equal(t, tc.wantPath, last.path) + }) + } +} + +// A limit is only sent when asked for: sending limit=0 asks the service for +// nothing rather than for everything. +func TestListOutputItemsSendsTheLimitOnlyWhenSet(t *testing.T) { + client, last := recorder(t, http.StatusOK, `{"data":[]}`) + + _, err := client.ListOutputItems(context.Background(), "eval_1", "run_1", 50) + require.NoError(t, err) + assert.Equal(t, "50", last.query.Get("limit")) + + _, err = client.ListOutputItems(context.Background(), "eval_1", "run_1", 0) + require.NoError(t, err) + assert.Empty(t, last.query.Get("limit"), "no limit means the service's default, not zero rows") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/poller_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/poller_test.go new file mode 100644 index 00000000000..55d02d27cd6 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/poller_test.go @@ -0,0 +1,228 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package eval_api + +import ( + "context" + "errors" + "net/http" + "sync/atomic" + "testing" + "time" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// fastPoller polls without the two-second wait a real one takes, so a test +// covering an exhausted attempt budget finishes in milliseconds. +func fastPoller(t *testing.T, get GetJobFunc, maxAttempts int) *Poller { + t.Helper() + p := NewPoller("op_1", "v1", get) + p.Options = PollerOptions{Interval: time.Millisecond, MaxAttempts: maxAttempts} + return p +} + +// jobs answers each poll with the next status in turn, holding the last one. +func jobs(statuses ...string) (GetJobFunc, *atomic.Int32) { + var calls atomic.Int32 + return func(context.Context, string, string) (*GenerationJob, error) { + i := int(calls.Add(1)) - 1 + if i >= len(statuses) { + i = len(statuses) - 1 + } + return &GenerationJob{ID: "op_1", Status: statuses[i]}, nil + }, &calls +} + +// The service spells terminal states several ways and does not agree with +// itself on case. Missing one leaves the CLI polling a job that will never +// change again, until the attempt budget runs out minutes later. +func TestJobStatusTerminalAndFailed(t *testing.T) { + cases := []struct { + raw string + terminal bool + failed bool + }{ + {"running", false, false}, + {"queued", false, false}, + {"", false, false}, // an absent status is a job that has not started + {"completed", true, false}, + {"succeeded", true, false}, + {"Succeeded", true, false}, // the service does not agree with itself on case + {"COMPLETED", true, false}, + {"failed", true, true}, + {"cancelled", true, true}, // both spellings are in use + {"canceled", true, true}, + {"Cancelled", true, true}, + } + + for _, tc := range cases { + status := ParseJobStatus(tc.raw) + assert.Equal(t, tc.terminal, status.IsTerminal(), "IsTerminal(%q)", tc.raw) + assert.Equal(t, tc.failed, status.IsFailed(), "IsFailed(%q)", tc.raw) + } + + assert.Equal(t, JobStatusRunning, ParseJobStatus(""), + "a job with no status yet is running, not finished") + assert.Equal(t, "succeeded", ParseJobStatus("Succeeded").String()) +} + +// A job that reaches a success state is returned, and polling stops there +// rather than continuing to spend attempts. +func TestPollerReturnsOnSuccess(t *testing.T) { + get, calls := jobs("running", "running", "completed") + job, err := fastPoller(t, get, 10).Poll(context.Background()) + + require.NoError(t, err) + require.NotNil(t, job) + assert.Equal(t, "completed", job.Status) + assert.Equal(t, int32(3), calls.Load(), "polling stops at the terminal state") +} + +// A failure has to carry the service's own message, since "job failed" alone +// leaves the user with nothing to act on. +func TestPollerReportsTheFailureMessage(t *testing.T) { + get := func(context.Context, string, string) (*GenerationJob, error) { + return &GenerationJob{ + ID: "op_1", + Status: "failed", + Error: &JobError{Message: "the model deployment was not found"}, + }, nil + } + + _, err := fastPoller(t, get, 5).Poll(context.Background()) + + var failed *JobFailedError + require.ErrorAs(t, err, &failed, "the caller inspects the job, so the type has to survive") + assert.Equal(t, JobStatusFailed, failed.Status) + assert.Contains(t, err.Error(), "the model deployment was not found") +} + +// A cancelled job is finished but not successful, so it must not be returned +// as a job whose output can be read. +func TestPollerTreatsCancellationAsFailure(t *testing.T) { + get, _ := jobs("cancelled") + job, err := fastPoller(t, get, 5).Poll(context.Background()) + + require.Error(t, err) + assert.Nil(t, job) + var failed *JobFailedError + require.ErrorAs(t, err, &failed) + assert.Equal(t, JobStatusCancelled, failed.Status) +} + +// A job with no error object still has to say something, or the failure +// surfaces as an empty line. +func TestJobFailedErrorWithoutAMessage(t *testing.T) { + err := &JobFailedError{Status: JobStatusFailed} + assert.Contains(t, err.Error(), "failed") +} + +// Throttling and server faults are the service being busy, not the job being +// broken: giving up on one would fail a run that was going to succeed. +func TestPollerRetriesTransientErrors(t *testing.T) { + var calls atomic.Int32 + get := func(context.Context, string, string) (*GenerationJob, error) { + switch calls.Add(1) { + case 1: + return nil, &azcore.ResponseError{StatusCode: http.StatusTooManyRequests} + case 2: + return nil, &azcore.ResponseError{StatusCode: http.StatusBadGateway} + case 3: + return nil, errors.New("read tcp: connection reset by peer") + default: + return &GenerationJob{ID: "op_1", Status: "succeeded"}, nil + } + } + + job, err := fastPoller(t, get, 10).Poll(context.Background()) + + require.NoError(t, err) + assert.Equal(t, "succeeded", job.Status) + assert.Equal(t, int32(4), calls.Load()) +} + +// A real failure — a deleted job, a bad token — is not worth retrying for the +// rest of the budget, so it is returned at once. +func TestPollerStopsOnAPermanentError(t *testing.T) { + var calls atomic.Int32 + get := func(context.Context, string, string) (*GenerationJob, error) { + calls.Add(1) + return nil, &azcore.ResponseError{StatusCode: http.StatusNotFound} + } + + _, err := fastPoller(t, get, 10).Poll(context.Background()) + + require.Error(t, err) + assert.Equal(t, int32(1), calls.Load(), "a 404 will not become a 200") +} + +// A job that never finishes has to end as a timeout naming the operation, so +// the user can go look it up rather than being told nothing. +func TestPollerTimesOut(t *testing.T) { + get, calls := jobs("running") + _, err := fastPoller(t, get, 3).Poll(context.Background()) + + var timeout *PollerTimeoutError + require.ErrorAs(t, err, &timeout) + assert.Equal(t, "op_1", timeout.OperationID) + assert.Equal(t, 3, timeout.Attempts) + assert.Contains(t, err.Error(), "op_1") + assert.Equal(t, int32(3), calls.Load(), "the budget is attempts, not retries after the first") +} + +// Ctrl-C during a wait has to return promptly rather than after the interval, +// and the context's own error is what says why. +func TestPollerHonoursContextCancellation(t *testing.T) { + get, _ := jobs("running") + p := NewPoller("op_1", "v1", get) + p.Options = PollerOptions{Interval: time.Hour, MaxAttempts: 10} + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + done := make(chan error, 1) + go func() { _, err := p.Poll(ctx); done <- err }() + + select { + case err := <-done: + require.ErrorIs(t, err, context.Canceled) + case <-time.After(5 * time.Second): + t.Fatal("Poll ignored a cancelled context and waited out the interval") + } +} + +// Progress reporting is what the user sees during a long run, so the callback +// fires on every poll and not only the last. +func TestPollerReportsEachStatus(t *testing.T) { + get, _ := jobs("queued", "running", "succeeded") + p := fastPoller(t, get, 10) + + var seen []JobStatus + p.OnPoll = func(s JobStatus) { seen = append(seen, s) } + + _, err := p.Poll(context.Background()) + require.NoError(t, err) + assert.Equal(t, []JobStatus{"queued", "running", "succeeded"}, seen) +} + +// Polling an empty id would ask the service about nothing, forever. It is the +// caller's bug and is worth naming before the first request. +func TestPollerRejectsAnEmptyOperationID(t *testing.T) { + _, err := NewPoller("", "v1", nil).Poll(context.Background()) + require.Error(t, err) + assert.Contains(t, err.Error(), "operation ID") +} + +// The defaults bound a run at roughly ten minutes. A shorter budget would +// abandon generation jobs that legitimately take that long. +func TestDefaultPollerOptions(t *testing.T) { + opts := DefaultPollerOptions() + assert.Equal(t, 2*time.Second, opts.Interval) + assert.Equal(t, 300, opts.MaxAttempts) + assert.GreaterOrEqual(t, opts.Interval*time.Duration(opts.MaxAttempts), 10*time.Minute) +} From 3d9dfffeeec02fe3a691d3e3b73332b4c41c9c0a Mon Sep 17 00:00:00 2001 From: mohessie Date: Thu, 6 Aug 2026 06:48:15 +0300 Subject: [PATCH 130/320] Say the endpoint order the code actually follows Both contexts resolve through projectctx, which reads five places. The comment above each still described the three it used to read, omitting global config and the AZURE_AI_PROJECT_ENDPOINT spelling entirely -- so anyone debugging why an endpoint was picked would have looked in the wrong places. --- .../extensions/azure.ai.dataset/internal/cmd/context.go | 9 ++++++--- .../azure.ai.evaluations/internal/cmd/context.go | 9 ++++++--- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/cli/azd/extensions/azure.ai.dataset/internal/cmd/context.go b/cli/azd/extensions/azure.ai.dataset/internal/cmd/context.go index 0ad178f0f67..39e120e351f 100644 --- a/cli/azd/extensions/azure.ai.dataset/internal/cmd/context.go +++ b/cli/azd/extensions/azure.ai.dataset/internal/cmd/context.go @@ -42,11 +42,14 @@ type datasetContext struct { } // newDatasetContext resolves the project endpoint and builds the data-plane -// clients. Endpoint resolution order: +// clients. The resolution order is projectctx's, so that every Foundry +// extension answers the same question the same way: // // 1. --project-endpoint -// 2. the active azd environment's FOUNDRY_PROJECT_ENDPOINT -// 3. the host environment variable of the same name +// 2. the active azd environment (FOUNDRY_PROJECT_ENDPOINT, then AZURE_AI_PROJECT_ENDPOINT) +// 3. global config: extensions.ai-agents.project.context.endpoint +// 4. the host environment variables of the same two names +// 5. otherwise an error naming how to set one func newDatasetContext(ctx context.Context, endpointFlag string) (*datasetContext, error) { azdClient, err := azdext.NewAzdClient() if err != nil { diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/context.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/context.go index e5412a72bd7..21eb8b0bee8 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/context.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/context.go @@ -36,11 +36,14 @@ type evalContext struct { } // newEvalContext resolves the project endpoint and builds the data-plane -// clients. Endpoint resolution order: +// clients. The resolution order is projectctx's, so that every Foundry +// extension answers the same question the same way: // // 1. --project-endpoint -// 2. the active azd environment's FOUNDRY_PROJECT_ENDPOINT -// 3. the host environment variable of the same name +// 2. the active azd environment (FOUNDRY_PROJECT_ENDPOINT, then AZURE_AI_PROJECT_ENDPOINT) +// 3. global config: extensions.ai-agents.project.context.endpoint +// 4. the host environment variables of the same two names +// 5. otherwise an error naming how to set one func newEvalContext(ctx context.Context, endpointFlag string) (*evalContext, error) { azdClient, err := azdext.NewAzdClient() if err != nil { From eb66497a798e391aeaee5a21db87ba6ce74e0a0d Mon Sep 17 00:00:00 2001 From: mohessie Date: Thu, 6 Aug 2026 06:48:15 +0300 Subject: [PATCH 131/320] Cover the dataset extension's output, which was copied without its tests The output helpers came over from azure.ai.evaluations, the tests did not, and extensions share no code -- so the conventions the spec sets out were being kept here by nothing at all. The rule under a list header, the bare array a list answers with, the empty values a detail view drops: each is a line of code that reads as decoration and deletes cleanly. Also covers what generation seeds from. The service accepts an agent source meant to pull an agent's instructions but fails for every agent, so they are read here instead; losing that read makes generate quietly ignore the agent it was pointed at, and produce a plausible dataset about nothing in particular. internal/cmd goes from 25.9% to 36.2%. --- .../internal/cmd/instruction_test.go | 132 +++++++++++++ .../internal/cmd/output_test.go | 177 ++++++++++++++++++ 2 files changed, 309 insertions(+) create mode 100644 cli/azd/extensions/azure.ai.dataset/internal/cmd/instruction_test.go create mode 100644 cli/azd/extensions/azure.ai.dataset/internal/cmd/output_test.go diff --git a/cli/azd/extensions/azure.ai.dataset/internal/cmd/instruction_test.go b/cli/azd/extensions/azure.ai.dataset/internal/cmd/instruction_test.go new file mode 100644 index 00000000000..086c9be62e3 --- /dev/null +++ b/cli/azd/extensions/azure.ai.dataset/internal/cmd/instruction_test.go @@ -0,0 +1,132 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "bytes" + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + + "azureaidataset/internal/pkg/gen_api" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// contextServingAgent builds a datasetContext whose generation client answers +// the agent read with status and body. +func contextServingAgent(t *testing.T, status int, body string) *datasetContext { + t.Helper() + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + if body != "" { + _, _ = w.Write([]byte(body)) + } + })) + t.Cleanup(srv.Close) + + pipeline := runtime.NewPipeline("test", "v1", runtime.PipelineOptions{}, + &policy.ClientOptions{Retry: policy.RetryOptions{MaxRetries: -1}}) + return &datasetContext{genClient: gen_api.NewClientFromPipeline(srv.URL, pipeline)} +} + +// agentBody is a catalog agent as the service returns one: versions inlined, +// with only `latest` populated on a plain read. +func agentBody(t *testing.T, instructions string) string { + t.Helper() + raw, err := json.Marshal(map[string]any{ + "name": "support-bot", + "versions": map[string]any{ + "latest": map[string]any{ + "version": "1", + "definition": map[string]any{"instructions": instructions}, + }, + }, + }) + require.NoError(t, err) + return string(raw) +} + +// What the caller passed wins: --instruction is the explicit answer to the +// question the agent is only a fallback for. +func TestResolveGenerationInstructionPrefersTheExplicitValue(t *testing.T) { + dc := contextServingAgent(t, http.StatusOK, agentBody(t, "from the agent")) + var out bytes.Buffer + + got, err := dc.resolveGenerationInstruction( + context.Background(), "from the flag", "support-bot", &out, false) + + require.NoError(t, err) + assert.Equal(t, "from the flag", got) + assert.Empty(t, out.String(), "nothing was looked up, so nothing is announced") +} + +// With no agent to read there is nothing to fall back to, and an empty +// instruction is a valid request rather than an error. +func TestResolveGenerationInstructionWithoutAnAgent(t *testing.T) { + dc := contextServingAgent(t, http.StatusOK, agentBody(t, "unused")) + var out bytes.Buffer + + got, err := dc.resolveGenerationInstruction(context.Background(), "", "", &out, false) + + require.NoError(t, err) + assert.Empty(t, got) +} + +// The service accepts an agent source meant to pull these instructions, but it +// fails for every agent, so they are read here instead. Losing this read makes +// generation silently ignore the agent it was pointed at. +func TestResolveGenerationInstructionReadsTheAgent(t *testing.T) { + dc := contextServingAgent(t, http.StatusOK, agentBody(t, "Answer support questions politely.")) + var out bytes.Buffer + + got, err := dc.resolveGenerationInstruction( + context.Background(), "", "support-bot", &out, false) + + require.NoError(t, err) + assert.Equal(t, "Answer support questions politely.", got) + assert.Contains(t, out.String(), "support-bot", + "the user is told what the generation was seeded from") +} + +// Generation can still proceed from the agent source alone, so a failure to +// read the agent is reported and stepped over rather than ending the command. +func TestResolveGenerationInstructionWarnsButContinues(t *testing.T) { + dc := contextServingAgent(t, http.StatusNotFound, `{"error":{"code":"NotFound"}}`) + var out bytes.Buffer + + got, err := dc.resolveGenerationInstruction( + context.Background(), "", "missing-bot", &out, false) + + require.NoError(t, err, "an unreadable agent must not fail the generation") + assert.Empty(t, got) + assert.Contains(t, out.String(), "warning") + assert.Contains(t, out.String(), "missing-bot") +} + +// --quiet is for scripts, where the progress lines are noise on stdout that a +// caller may be parsing. +func TestResolveGenerationInstructionStaysQuiet(t *testing.T) { + var out bytes.Buffer + + found := contextServingAgent(t, http.StatusOK, agentBody(t, "Answer politely.")) + got, err := found.resolveGenerationInstruction( + context.Background(), "", "support-bot", &out, true) + require.NoError(t, err) + assert.Equal(t, "Answer politely.", got, "quiet changes what is printed, not what is resolved") + assert.Empty(t, out.String()) + + missing := contextServingAgent(t, http.StatusNotFound, `{}`) + _, err = missing.resolveGenerationInstruction( + context.Background(), "", "missing-bot", &out, true) + require.NoError(t, err) + assert.Empty(t, out.String(), "even the warning is suppressed") +} diff --git a/cli/azd/extensions/azure.ai.dataset/internal/cmd/output_test.go b/cli/azd/extensions/azure.ai.dataset/internal/cmd/output_test.go new file mode 100644 index 00000000000..3e7fec6ab20 --- /dev/null +++ b/cli/azd/extensions/azure.ai.dataset/internal/cmd/output_test.go @@ -0,0 +1,177 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "bytes" + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + + "azureaidataset/internal/pkg/dataset_api" + + "github.com/spf13/cobra" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// commandWithOutput builds a command carrying the -o flag the azd SDK root +// supplies at runtime, on the same flag set the production code reads. +func commandWithOutput(t *testing.T, value string) *cobra.Command { + t.Helper() + cmd := &cobra.Command{Use: "x"} + cmd.Flags().StringP("output", "o", "", "") + require.NoError(t, cmd.Flags().Set("output", value)) + cmd.SetOut(&bytes.Buffer{}) + return cmd +} + +// The flag selects machine-readable output, and a caller who types JSON in +// caps means the same thing as one who does not. +func TestOutputFormatAndIsJSON(t *testing.T) { + assert.True(t, isJSON(commandWithOutput(t, "json"))) + assert.True(t, isJSON(commandWithOutput(t, "JSON")), "the format is matched without regard to case") + assert.False(t, isJSON(commandWithOutput(t, "table"))) + assert.False(t, isJSON(commandWithOutput(t, ""))) + + assert.False(t, isJSON(nil), "a command with no flags is not JSON output") + assert.Empty(t, outputFormat(nil)) + + // A command that never declared -o must not panic on being asked. + assert.Empty(t, outputFormat(&cobra.Command{Use: "bare"})) +} + +// A nil slice encodes as null, which a caller iterating the result reads as a +// type error rather than as an empty list. +func TestEmitJSONListNormalizesNil(t *testing.T) { + var buf bytes.Buffer + var none []dataset_api.Dataset + require.NoError(t, emitJSONList(&buf, none)) + assert.Equal(t, "[]", strings.TrimSpace(buf.String())) + + buf.Reset() + require.NoError(t, emitJSONList(&buf, []dataset_api.Dataset{{Name: "a", Version: "1.0"}})) + var round []map[string]any + require.NoError(t, json.Unmarshal(buf.Bytes(), &round)) + require.Len(t, round, 1) + assert.Equal(t, "a", round[0]["name"]) +} + +// The list emits a bare array, not the envelope the service replied with: the +// envelopes disagree with each other and carry paging this extension does not +// follow. +func TestEmitJSONListDropsTheEnvelope(t *testing.T) { + cmd := commandWithOutput(t, "json") + var buf bytes.Buffer + cmd.SetOut(&buf) + + require.NoError(t, renderDatasets(cmd, &dataset_api.DatasetList{ + Value: []dataset_api.Dataset{{Name: "a", Version: "1.0"}}, + NextLink: "https://example/page2", + })) + + assert.True(t, strings.HasPrefix(strings.TrimSpace(buf.String()), "["), + "a list answers with an array") + assert.NotContains(t, buf.String(), "nextLink", + "paging this extension does not follow must not suggest there is more to fetch") +} + +// A list view is uppercase headers over a rule. The rule is what separates the +// header from the data at a glance, and it is what the sibling extensions print. +func TestRenderDatasetsTable(t *testing.T) { + cmd := commandWithOutput(t, "") + var buf bytes.Buffer + cmd.SetOut(&buf) + + require.NoError(t, renderDatasets(cmd, &dataset_api.DatasetList{Value: []dataset_api.Dataset{ + {Name: "golden", Version: "2.0", Format: "jsonl"}, + {Name: "smoke", Version: "1.0", Format: "jsonl"}, + }})) + + lines := strings.Split(strings.TrimRight(buf.String(), "\n"), "\n") + require.Len(t, lines, 4, "a header, its rule, and one line per dataset") + assert.Contains(t, lines[0], "NAME") + assert.Contains(t, lines[0], "VERSION") + assert.Contains(t, lines[0], "FORMAT") + assert.True(t, strings.HasPrefix(strings.TrimSpace(lines[1]), "----"), + "the rule under the header is the convention, got %q", lines[1]) + assert.Contains(t, lines[2], "golden") + assert.Contains(t, lines[3], "smoke") +} + +// An empty project has to say so. A bare header over nothing reads as output +// that got cut off. +func TestRenderDatasetsSaysWhenThereAreNone(t *testing.T) { + cmd := commandWithOutput(t, "") + var buf bytes.Buffer + cmd.SetOut(&buf) + + require.NoError(t, renderDatasets(cmd, &dataset_api.DatasetList{})) + assert.Contains(t, buf.String(), "No datasets found.") + assert.NotContains(t, buf.String(), "NAME") +} + +// A detail view is Title Case key/value, the shape `show` uses, and a blank +// value is dropped rather than printed as an empty column. +func TestEmitDetail(t *testing.T) { + var buf bytes.Buffer + require.NoError(t, emitDetail(&buf, []field{ + {"Name", "golden"}, + {"Version", "2.0"}, + {"Description", ""}, + {"Format", "jsonl"}, + })) + + lines := strings.Split(strings.TrimRight(buf.String(), "\n"), "\n") + require.Len(t, lines, 3, "the empty Description is dropped") + assert.True(t, strings.HasPrefix(lines[0], "Name")) + assert.Contains(t, lines[0], "golden") + assert.NotContains(t, buf.String(), "Description") +} + +// `show` returns one thing, so the spec's output conventions make it a detail +// view rather than the raw JSON it would otherwise be easiest to print. +func TestShowUsesADetailView(t *testing.T) { + body, err := os.ReadFile(filepath.Join(".", "dataset.go")) + require.NoError(t, err) + assert.Contains(t, string(body), "emitDetail", + "dataset show returns one thing, so dataset.go renders it as a detail view") +} + +// The message has to name the flag that would have supplied the value, since +// --no-prompt means nobody is there to be asked. +func TestRequireFlag(t *testing.T) { + err := requireFlag("name") + require.Error(t, err) + assert.Contains(t, err.Error(), "--name") + assert.Contains(t, err.Error(), "--no-prompt") +} + +// --from-file takes either the file or the directory holding it, because both +// are what a caller has to hand. Anything else is worth refusing by name. +func TestDatasetUploadDir(t *testing.T) { + dir := t.TempDir() + rows := filepath.Join(dir, "rows.jsonl") + require.NoError(t, os.WriteFile(rows, []byte("{\"query\":\"q\"}\n"), 0o600)) + + got, err := datasetUploadDir(rows) + require.NoError(t, err) + assert.Equal(t, dir, got, "a file resolves to the directory the upload scans") + + got, err = datasetUploadDir(dir) + require.NoError(t, err) + assert.Equal(t, dir, got) + + notJSONL := filepath.Join(dir, "rows.csv") + require.NoError(t, os.WriteFile(notJSONL, []byte("a,b\n"), 0o600)) + _, err = datasetUploadDir(notJSONL) + require.Error(t, err) + assert.Contains(t, err.Error(), ".jsonl") + + _, err = datasetUploadDir(filepath.Join(dir, "missing.jsonl")) + require.Error(t, err) + assert.Contains(t, err.Error(), "--from-file") +} From 20776f0d3a40c59c02c769e13d76328128f2643c Mon Sep 17 00:00:00 2001 From: mohessie Date: Thu, 6 Aug 2026 06:55:59 +0300 Subject: [PATCH 132/320] Guard the dataset half of drift, and how --eval finds an eval Evaluator drift was tested and dataset drift was not, though it is the same risk against content nobody has a local copy of: a version published outside the repo, silently overwritten by the next deploy. The check is also narrower than it looks -- only a *newer* version is somebody else's work, and an empty listing means the eventually-consistent listing has not caught up rather than that the project is behind. Also covers what --eval accepts. One flag takes a name or an id, and the useful case is the one in between: a name that is declared but not deployed, which has no id to send and would come back as a service 404 rather than as "run azd up first". --- .../internal/cmd/evalref_drift_test.go | 198 ++++++++++++++++++ 1 file changed, 198 insertions(+) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/evalref_drift_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evalref_drift_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evalref_drift_test.go new file mode 100644 index 00000000000..f52e5b1654d --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evalref_drift_test.go @@ -0,0 +1,198 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "testing" + + "azureaieval/internal/pkg/dataset_api" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// reconcilerListingVersions builds a reconciler whose dataset client answers a +// version listing with these versions. +func reconcilerListingVersions(t *testing.T, versions ...string) *evalReconciler { + t.Helper() + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + values := make([]map[string]any, 0, len(versions)) + for _, v := range versions { + values = append(values, map[string]any{"name": "golden", "version": v}) + } + w.Header().Set("Content-Type", "application/json") + require.NoError(t, json.NewEncoder(w).Encode(map[string]any{"value": values})) + })) + t.Cleanup(srv.Close) + + pipeline := runtime.NewPipeline("test", "v1", runtime.PipelineOptions{}, + &policy.ClientOptions{Retry: policy.RetryOptions{MaxRetries: -1}}) + return &evalReconciler{ec: &evalContext{ + datasetClient: dataset_api.NewDatasetClientFromPipeline(srv.URL, pipeline), + }} +} + +// A version published outside the repo would otherwise be overwritten by the +// next deploy. The evaluator side of this is guarded; the dataset side is the +// same risk against content nobody has a copy of. +func TestCheckDatasetDriftRefusesANewerPublishedVersion(t *testing.T) { + r := reconcilerListingVersions(t, "1.0", "2.0", "3.0") + + err := r.checkDatasetDrift(context.Background(), "golden", "2.0") + + require.Error(t, err) + assert.Contains(t, err.Error(), "3.0", "the version that is actually there") + assert.Contains(t, err.Error(), "2.0", "and the one this repo last deployed") + assert.Contains(t, err.Error(), "outside this repo") + assert.Contains(t, err.Error(), "version: 3.0", "the fix is a pin the user can paste") +} + +// Matching versions are the ordinary case and must stay silent, or every +// deploy would report drift. +func TestCheckDatasetDriftAcceptsAMatch(t *testing.T) { + r := reconcilerListingVersions(t, "1.0", "2.0") + require.NoError(t, r.checkDatasetDrift(context.Background(), "golden", "2.0")) +} + +// The listing is eventually consistent and answers with nothing for a second +// or two after a publish. Reading that as "the project is behind" would report +// drift on a dataset this repo had just deployed. +func TestCheckDatasetDriftIgnoresAnEmptyListing(t *testing.T) { + r := reconcilerListingVersions(t) + require.NoError(t, r.checkDatasetDrift(context.Background(), "golden", "2.0")) + assert.Empty(t, r.latestDatasetVersion(context.Background(), "golden")) +} + +// Only a newer version is someone else's work. An older one means this repo is +// ahead, which the deploy is about to fix anyway. +func TestCheckDatasetDriftIgnoresAnOlderVersion(t *testing.T) { + r := reconcilerListingVersions(t, "1.0") + require.NoError(t, r.checkDatasetDrift(context.Background(), "golden", "2.0"), + "a project behind this repo is not drift") +} + +// An unreachable project must not fail the deploy on drift it could not check. +func TestLatestDatasetVersionToleratesAFailedListing(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusInternalServerError) + })) + t.Cleanup(srv.Close) + + pipeline := runtime.NewPipeline("test", "v1", runtime.PipelineOptions{}, + &policy.ClientOptions{Retry: policy.RetryOptions{MaxRetries: -1}}) + r := &evalReconciler{ec: &evalContext{ + datasetClient: dataset_api.NewDatasetClientFromPipeline(srv.URL, pipeline), + }} + + assert.Empty(t, r.latestDatasetVersion(context.Background(), "golden")) + require.NoError(t, r.checkDatasetDrift(context.Background(), "golden", "2.0")) +} + +// writeEvalYAML puts a configuration in a temp dir and returns the dir. +func writeEvalYAML(t *testing.T, body string) string { + t.Helper() + dir := t.TempDir() + evals := filepath.Join(dir, "evals") + require.NoError(t, os.MkdirAll(evals, 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(evals, "eval.yaml"), []byte(body), 0o600)) + return evals +} + +// A declared eval that was never deployed has no id to address, and the +// service would answer 404 for a name it never saw. Naming `azd up` is the +// difference between a dead end and a next step. +func TestResolveEvalRefFailsFastOnAnUndeployedDeclaration(t *testing.T) { + dir := writeEvalYAML(t, ` +datasets: + - name: golden +evals: + - name: support-quality + dataset: golden + evaluators: + - evaluator: builtin.relevance +`) + ec := &evalContext{} + + _, err := ec.resolveEvalRef(context.Background(), dir, "support-quality") + + require.Error(t, err) + assert.Contains(t, err.Error(), "support-quality") + assert.Contains(t, err.Error(), "azd up", "the error has to say what would fix it") +} + +// An eval made by `azd ai eval create` has no evals: entry, so anything that +// is not a declared name is sent on as an id rather than refused. +func TestResolveEvalRefTreatsAnUnknownNameAsAnID(t *testing.T) { + dir := writeEvalYAML(t, ` +datasets: + - name: golden +evals: + - name: support-quality + dataset: golden + evaluators: + - evaluator: builtin.relevance +`) + ec := &evalContext{} + + ref, err := ec.resolveEvalRef(context.Background(), dir, "eval_68a1f2c3") + + require.NoError(t, err) + assert.Equal(t, "eval_68a1f2c3", ref.ID) + assert.False(t, ref.Declared(), "an id carries no declaration to run from") + assert.Nil(t, ref.Eval) +} + +// With no configuration and no name there is nothing to resolve, and the +// message has to say where a declaration would have been looked for. +func TestResolveEvalRefWithoutAConfigurationOrAName(t *testing.T) { + ec := &evalContext{} + + _, err := ec.resolveEvalRef(context.Background(), t.TempDir(), "") + + require.Error(t, err) + assert.Contains(t, err.Error(), "--eval") + assert.Contains(t, err.Error(), "eval.yaml") +} + +// Outside a project an id is still enough to address every route under it. +func TestResolveEvalRefAcceptsAnIDWithoutAConfiguration(t *testing.T) { + ec := &evalContext{} + + ref, err := ec.resolveEvalRef(context.Background(), t.TempDir(), "eval_68a1f2c3") + + require.NoError(t, err) + assert.Equal(t, "eval_68a1f2c3", ref.ID) + assert.False(t, ref.Declared()) +} + +// Naming nothing where several evals are declared is ambiguous, and the +// configuration's own complaint is the useful one. +func TestResolveEvalRefReportsAmbiguity(t *testing.T) { + dir := writeEvalYAML(t, ` +datasets: + - name: golden +evals: + - name: first + dataset: golden + evaluators: + - evaluator: builtin.relevance + - name: second + dataset: golden + evaluators: + - evaluator: builtin.coherence +`) + ec := &evalContext{} + + _, err := ec.resolveEvalRef(context.Background(), dir, "") + require.Error(t, err) +} From 553e75134c5b4282d1edd2ef2f40589df3e1a864 Mon Sep 17 00:00:00 2001 From: mohessie Date: Thu, 6 Aug 2026 18:23:19 +0300 Subject: [PATCH 133/320] Leave generation with the extension that owns eval.yaml dataset generate writes the datasets: entry into evals/eval.yaml. That file belongs to azure.ai.evaluations, and azd extensions share no code, so a copy here could only reach it by duplicating that extension's schema. The copy shipped without doing so, and printed zd ai eval dataset create as the way to register what it produced. That command uploads to the service and records a version; it does not touch eval.yaml. Following it left the dataset registered and absent from the configuration, so zd up would not reconcile it and no eval could name it -- the artifact existed and nothing could use it. So generation stays where the file is. What moves here is the CRUD that needs no configuration at all: create, update, list, show, delete, versions. The job group goes with generate, having nothing left to reattach to, and gen_api with it -- this extension now speaks to one API. A guard fails the build if a --from flag reappears, since the failure it protects against is silent. --- .../extensions/azure.ai.dataset/CHANGELOG.md | 10 +- cli/azd/extensions/azure.ai.dataset/README.md | 23 +- .../azure.ai.dataset/extension.yaml | 8 +- .../internal/cmd/artifacts.go | 75 +--- .../azure.ai.dataset/internal/cmd/context.go | 10 - .../azure.ai.dataset/internal/cmd/dataset.go | 5 +- .../azure.ai.dataset/internal/cmd/generate.go | 366 ------------------ .../internal/cmd/helpers_test.go | 151 -------- .../internal/cmd/instruction_test.go | 132 ------- .../azure.ai.dataset/internal/cmd/job.go | 197 ---------- .../azure.ai.dataset/internal/cmd/root.go | 6 +- .../internal/cmd/surface_test.go | 44 +-- .../internal/pkg/dataset_api/operations.go | 9 + .../internal/pkg/gen_api/errors.go | 49 --- .../internal/pkg/gen_api/generation.go | 180 --------- .../internal/pkg/gen_api/generation_test.go | 232 ----------- .../internal/pkg/gen_api/models.go | 144 ------- .../internal/pkg/gen_api/models_test.go | 179 --------- .../internal/pkg/gen_api/operations.go | 233 ----------- .../internal/pkg/gen_api/operations_test.go | 188 --------- .../internal/pkg/gen_api/poller.go | 193 --------- 21 files changed, 48 insertions(+), 2386 deletions(-) delete mode 100644 cli/azd/extensions/azure.ai.dataset/internal/cmd/generate.go delete mode 100644 cli/azd/extensions/azure.ai.dataset/internal/cmd/instruction_test.go delete mode 100644 cli/azd/extensions/azure.ai.dataset/internal/cmd/job.go delete mode 100644 cli/azd/extensions/azure.ai.dataset/internal/pkg/gen_api/errors.go delete mode 100644 cli/azd/extensions/azure.ai.dataset/internal/pkg/gen_api/generation.go delete mode 100644 cli/azd/extensions/azure.ai.dataset/internal/pkg/gen_api/generation_test.go delete mode 100644 cli/azd/extensions/azure.ai.dataset/internal/pkg/gen_api/models.go delete mode 100644 cli/azd/extensions/azure.ai.dataset/internal/pkg/gen_api/models_test.go delete mode 100644 cli/azd/extensions/azure.ai.dataset/internal/pkg/gen_api/operations.go delete mode 100644 cli/azd/extensions/azure.ai.dataset/internal/pkg/gen_api/operations_test.go delete mode 100644 cli/azd/extensions/azure.ai.dataset/internal/pkg/gen_api/poller.go diff --git a/cli/azd/extensions/azure.ai.dataset/CHANGELOG.md b/cli/azd/extensions/azure.ai.dataset/CHANGELOG.md index d4808aec590..17c3d3ee366 100644 --- a/cli/azd/extensions/azure.ai.dataset/CHANGELOG.md +++ b/cli/azd/extensions/azure.ai.dataset/CHANGELOG.md @@ -4,8 +4,8 @@ ### Features Added -- Initial release. `create`, `update`, `generate`, `list`, `show`, `delete`, - `versions list`, and the `job` group for reattaching to generation jobs. -- The command groups moved here from `azure.ai.evaluations`, where they shipped - first because M1's hero scenarios needed `dataset generate` to exist before a - second extension could be published. +- Initial release. `create`, `update`, `list`, `show`, `delete`, and + `versions list`. +- The CRUD groups moved here from `azure.ai.evaluations`. Generation stayed + there: `dataset generate` writes the `datasets:` entry in `evals/eval.yaml`, + and that file belongs to the evaluation extension. diff --git a/cli/azd/extensions/azure.ai.dataset/README.md b/cli/azd/extensions/azure.ai.dataset/README.md index 2059b58fcc7..544310c1047 100644 --- a/cli/azd/extensions/azure.ai.dataset/README.md +++ b/cli/azd/extensions/azure.ai.dataset/README.md @@ -1,6 +1,6 @@ # Foundry datasets (Beta) -Create, generate and version Foundry datasets from your terminal. +Register and version Foundry datasets from your terminal. ```console $ azd extension install azure.ai.dataset @@ -17,27 +17,20 @@ than inside `azure.ai.evaluations`. |---|---| | `azd ai dataset create --from-file ` | Register a dataset, publishing its first version | | `azd ai dataset update --from-file ` | Publish a further version | -| `azd ai dataset generate ` | Generate a dataset and download it | | `azd ai dataset list` | List the project's datasets | | `azd ai dataset show ` | Show one dataset | | `azd ai dataset delete ` | Delete a dataset version | | `azd ai dataset versions list ` | List a dataset's versions | -| `azd ai dataset job {list,show,cancel,delete}` | Reattach to a generation job | -## Generation +## Generating a dataset -`generate` submits a job to the evaluation service and downloads what it -produced, so this extension speaks to two APIs: datasets are registered through -the dataset API but generated by the evaluation service. +Generation is `azd ai eval dataset generate`, in `azure.ai.evaluations`, and +stays there: it writes the `datasets:` entry in `evals/eval.yaml`, which is that +extension's file. Splitting the two would leave a generated dataset registered +with the service but absent from the configuration, so `azd up` would not +reconcile it and no eval could name it. -`--from` selects one or more of the sources the service accepts, and is -repeatable. It defaults to `traces` when the project has Application Insights -connected — real conversations make a better dataset than synthesized ones — -and to `agent` otherwise. - -Registering a generated file in an evaluation configuration is a separate step: -`evals/eval.yaml` belongs to `azure.ai.evaluations`, so `generate` prints the -command rather than writing that file itself. +Once a file exists, `create` registers it here. ## Project endpoint diff --git a/cli/azd/extensions/azure.ai.dataset/extension.yaml b/cli/azd/extensions/azure.ai.dataset/extension.yaml index 7616c3ee65d..39e5b02d77c 100644 --- a/cli/azd/extensions/azure.ai.dataset/extension.yaml +++ b/cli/azd/extensions/azure.ai.dataset/extension.yaml @@ -2,7 +2,7 @@ id: azure.ai.dataset namespace: ai.dataset displayName: Foundry datasets (Beta) -description: Create, generate and version Foundry datasets from your terminal. (Beta) +description: Register and version Foundry datasets from your terminal. (Beta) usage: azd ai dataset [options] # NOTE: Make sure version.txt is in sync with this version. version: 1.0.0-beta.1 @@ -15,9 +15,9 @@ examples: - name: create description: Register a dataset from a local file. usage: azd ai dataset create support-regression --from-file ./data/golden.jsonl - - name: generate - description: Generate a dataset from an agent's traces. - usage: azd ai dataset generate support-regression --from traces + - name: versions list + description: List the versions of a dataset. + usage: azd ai dataset versions list support-regression tags: - ai - foundry diff --git a/cli/azd/extensions/azure.ai.dataset/internal/cmd/artifacts.go b/cli/azd/extensions/azure.ai.dataset/internal/cmd/artifacts.go index 7d05564e271..c8a101b5a36 100644 --- a/cli/azd/extensions/azure.ai.dataset/internal/cmd/artifacts.go +++ b/cli/azd/extensions/azure.ai.dataset/internal/cmd/artifacts.go @@ -3,64 +3,7 @@ package cmd -import ( - "fmt" - "path/filepath" - "strings" -) - -// The sample count the service accepts, and the default the spec documents. -const ( - minSampleSize = 15 - maxSampleSize = 1000 - defaultSampleSize = 15 -) - -// defaultOutputDir is where a generated dataset lands, matching the layout -// `azd ai eval init` scaffolds so a generated file is already where the eval -// configuration expects to find it. -const defaultOutputDir = "evals/datasets" - -// Sources a dataset can be generated from. -const ( - generateFromTraces = "traces" - generateFromAgent = "agent" - generateFromPrompt = "prompt" - generateFromFile = "file" -) - -// generateSources is what --from accepts, in help order. -var generateSources = []string{ - generateFromTraces, generateFromAgent, generateFromPrompt, generateFromFile, -} - -// validateGenerateSource rejects a --from value the service has no path for. -func validateGenerateSource(from string) error { - switch from { - case "", generateFromTraces, generateFromAgent, generateFromPrompt, generateFromFile: - return nil - default: - return fmt.Errorf( - "--from %q is not a source; use one of %s", - from, strings.Join(generateSources, ", ")) - } -} - -// validateSampleSize rejects a row count the service would reject, before a -// generation job is submitted and billed. -func validateSampleSize(n int) error { - if n != 0 && (n < minSampleSize || n > maxSampleSize) { - return fmt.Errorf( - "--max-samples must be between %d and %d, got %d", - minSampleSize, maxSampleSize, n) - } - return nil -} - -// artifactPath is where a generated dataset is written. -func artifactPath(outputDir, name string) string { - return filepath.Join(outputDir, name+".jsonl") -} +import "fmt" // envKeyDatasetVersion caches the version resolved at the last publish, so a // later read does not have to list every version to find the newest. @@ -78,19 +21,3 @@ func checkAssetExistence(verb, kind, name string, exists bool) error { } return nil } - -// defaultGenerationSource picks what `dataset generate` sends when --from was -// not given, from the Application Insights connection string the project has -// (or has not) been given. -// -// Traces are the better dataset when they exist, being real conversations -// rather than synthesized ones, so they win whenever the project is wired to -// collect them. Outside a project, or in one with no Application Insights, -// there are no traces to ask for and the agent's own definition is all that is -// left. -func defaultGenerationSource(appInsightsConnection string) []string { - if appInsightsConnection != "" { - return []string{generateFromTraces} - } - return []string{generateFromAgent} -} diff --git a/cli/azd/extensions/azure.ai.dataset/internal/cmd/context.go b/cli/azd/extensions/azure.ai.dataset/internal/cmd/context.go index 39e120e351f..2e6b6380ebb 100644 --- a/cli/azd/extensions/azure.ai.dataset/internal/cmd/context.go +++ b/cli/azd/extensions/azure.ai.dataset/internal/cmd/context.go @@ -12,7 +12,6 @@ import ( "azureaidataset/internal/foundry/projectctx" "azureaidataset/internal/pkg/dataset_api" - "azureaidataset/internal/pkg/gen_api" "github.com/Azure/azure-sdk-for-go/sdk/azcore" "github.com/Azure/azure-sdk-for-go/sdk/azidentity" @@ -23,14 +22,7 @@ import ( // endpoint the data-plane clients target. const projectEndpointEnvKey = "FOUNDRY_PROJECT_ENDPOINT" -// appInsightsEnvKey is where a connected Application Insights resource lands in -// the azd environment, and so how a project says it collects traces at all. -const appInsightsEnvKey = "APPLICATIONINSIGHTS_CONNECTION_STRING" - // datasetContext carries everything the commands need to reach the data plane. -// -// Two clients, because a dataset is registered through the dataset API but -// generated by the evaluation service. type datasetContext struct { azdClient *azdext.AzdClient endpoint string @@ -38,7 +30,6 @@ type datasetContext struct { cred azcore.TokenCredential datasetClient *dataset_api.DatasetClient - genClient *gen_api.Client } // newDatasetContext resolves the project endpoint and builds the data-plane @@ -79,7 +70,6 @@ func newDatasetContext(ctx context.Context, endpointFlag string) (*datasetContex dc.cred = cred dc.datasetClient = dataset_api.NewDatasetClient(dc.endpoint, cred) - dc.genClient = gen_api.NewClient(dc.endpoint, cred) return dc, nil } diff --git a/cli/azd/extensions/azure.ai.dataset/internal/cmd/dataset.go b/cli/azd/extensions/azure.ai.dataset/internal/cmd/dataset.go index 9e2f700cea2..c96f52e3f45 100644 --- a/cli/azd/extensions/azure.ai.dataset/internal/cmd/dataset.go +++ b/cli/azd/extensions/azure.ai.dataset/internal/cmd/dataset.go @@ -11,7 +11,6 @@ import ( "strings" "azureaidataset/internal/pkg/dataset_api" - "azureaidataset/internal/pkg/gen_api" "github.com/spf13/cobra" ) @@ -248,7 +247,7 @@ func newDatasetShowCommand() *cobra.Command { ds, err := ec.datasetClient.GetDataset(ctx, name, version, ProjectEndpointAPIVersion) if err != nil { - if gen_api.IsNotFound(err) { + if dataset_api.IsNotFound(err) { return fmt.Errorf( "no dataset %q at version %q in this project; "+ "`azd ai dataset list` shows the ones there are", name, version) @@ -299,7 +298,7 @@ func newDatasetDeleteCommand() *cobra.Command { if err := ec.datasetClient.DeleteDatasetVersion( ctx, name, version, ProjectEndpointAPIVersion, ); err != nil { - if gen_api.IsNotFound(err) { + if dataset_api.IsNotFound(err) { return fmt.Errorf( "no dataset %q at version %q in this project", name, version) } diff --git a/cli/azd/extensions/azure.ai.dataset/internal/cmd/generate.go b/cli/azd/extensions/azure.ai.dataset/internal/cmd/generate.go deleted file mode 100644 index 75478e9a546..00000000000 --- a/cli/azd/extensions/azure.ai.dataset/internal/cmd/generate.go +++ /dev/null @@ -1,366 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package cmd - -import ( - "context" - "errors" - "fmt" - "io" - "os" - "path/filepath" - "strings" - "time" - - "azureaidataset/internal/pkg/gen_api" - - "github.com/spf13/cobra" -) - -// generatePollBudget replaces the inherited 2s x 300 (10 minute) client budget. -// The generation job is not gateway-capped; the old limit simply gave up while -// the service was still working, forcing a second command. -var generatePollBudget = gen_api.PollerOptions{ - Interval: 5 * time.Second, - MaxAttempts: 720, // one hour -} - -// generationPlan is everything one generation job needs, after the flags have -// been reconciled. -type generationPlan struct { - Name string - Agent string - Instruction string - Model string - OutputDir string - SampleSize int - // From is what --from named: which of the service's sources to send. Empty - // sends whatever the plan has to offer. - From []string - // TraceDays seeds generation from that many days of recent traces. - TraceDays int -} - -func (p generationPlan) traceOptions() *gen_api.TraceOptions { - if p.TraceDays <= 0 { - return nil - } - return &gen_api.TraceOptions{Days: p.TraceDays} -} - -func newDatasetGenerateCommand() *cobra.Command { - var ( - name string - target string - instruction string - instructionFile string - model string - outputDir string - endpoint string - from []string - maxSamples int - traceDays int - noWait bool - force bool - ) - - cmd := &cobra.Command{ - Use: "generate ", - Short: "Generate a dataset and download it.", - Long: "Generate a dataset and download it.\n\n" + - "--from selects one or more of the sources the service accepts, and " + - "is repeatable. Generating from the agent's own definition is a " + - "preference rather than a fallback: it covers cases no user has hit " + - "yet, and it can supply reference answers, which a transcript cannot.", - Args: cobra.ExactArgs(1), - RunE: func(cmd *cobra.Command, args []string) error { - name = args[0] - - for _, src := range from { - if err := validateGenerateSource(src); err != nil { - return err - } - } - if err := validateSampleSize(maxSamples); err != nil { - return err - } - if instruction == "" && instructionFile != "" { - body, err := os.ReadFile(instructionFile) - if err != nil { - return fmt.Errorf("reading --agent-instruction-file %q: %w", instructionFile, err) - } - instruction = strings.TrimSpace(string(body)) - } - if model == "" { - return errors.New("--generation-model is required") - } - - plan := generationPlan{ - Name: name, - Agent: target, - Instruction: instruction, - Model: model, - OutputDir: outputDir, - SampleSize: maxSamples, - From: from, - TraceDays: traceDays, - } - if plan.OutputDir == "" { - plan.OutputDir = defaultOutputDir - } - if plan.SampleSize == 0 { - plan.SampleSize = defaultSampleSize - } - if err := refuseExistingArtifact(artifactPath(plan.OutputDir, plan.Name), force); err != nil { - return err - } - - ctx := cmd.Context() - dc, err := newDatasetContext(ctx, endpoint) - if err != nil { - return err - } - defer dc.Close() - - if len(plan.From) == 0 { - plan.From = defaultGenerationSource(dc.getEnvValue(ctx, appInsightsEnvKey)) - } - plan.Instruction, err = dc.resolveGenerationInstruction( - ctx, plan.Instruction, plan.Agent, cmd.OutOrStdout(), isJSON(cmd)) - if err != nil { - return err - } - - return dc.generateDataset(ctx, plan, cmd.OutOrStdout(), noWait, isJSON(cmd)) - }, - } - - cmd.Flags().StringVar(&target, "target", "", "Agent whose context seeds generation.") - cmd.Flags().StringVar(&instruction, "agent-instruction", "", - "What the agent does and what to test.") - cmd.Flags().StringVar(&instructionFile, "agent-instruction-file", "", - "Read the agent instruction from this file. Mutually exclusive with --agent-instruction.") - cmd.MarkFlagsMutuallyExclusive("agent-instruction", "agent-instruction-file") - cmd.Flags().StringVar(&model, "generation-model", "", - "Model deployment that generates the dataset.") - cmd.Flags().StringVar(&outputDir, "output-dir", "", - fmt.Sprintf("Directory the generated dataset is written to. Defaults to %s.", defaultOutputDir)) - cmd.Flags().IntVar(&maxSamples, "max-samples", 0, - fmt.Sprintf("Rows to synthesize (%d-%d). Defaults to %d.", - minSampleSize, maxSampleSize, defaultSampleSize)) - cmd.Flags().StringSliceVar(&from, "from", nil, - fmt.Sprintf("Where rows come from: %s. Repeatable, and the service accepts "+ - "more than one. Defaults to %s when the project has Application Insights "+ - "connected, otherwise %s.", - strings.Join(generateSources, ", "), generateFromTraces, generateFromAgent)) - cmd.Flags().IntVar(&traceDays, "trace-days", 0, - "Narrow a traces source to this many days of recent activity.") - cmd.Flags().BoolVar(&noWait, "no-wait", false, - "Submit the job and return its id without polling.") - cmd.Flags().BoolVar(&force, "force", false, - "Overwrite a dataset file that already exists.") - cmd.Flags().StringVar(&endpoint, "project-endpoint", "", "Foundry project endpoint.") - return cmd -} - -// refuseExistingArtifact stops a generation that would overwrite a checked-in -// file, because the job is billed and the diff is what the author reviews. -func refuseExistingArtifact(path string, force bool) error { - if force { - return nil - } - if _, err := os.Stat(path); err == nil { - return fmt.Errorf( - "%s already exists; pass --force to overwrite it, or --output-dir to write elsewhere", - filepath.ToSlash(path)) - } - return nil -} - -// resolveGenerationInstruction decides what generation is seeded from: what the -// caller passed, then the agent's published instructions. -// -// The service accepts an agent source that is meant to pull those instructions, -// but it fails for every agent, so they are read here instead. -func (dc *datasetContext) resolveGenerationInstruction( - ctx context.Context, - explicit, agentName string, - out io.Writer, - quiet bool, -) (string, error) { - if explicit != "" || agentName == "" { - return explicit, nil - } - - agent, err := dc.genClient.GetAgent(ctx, agentName, ProjectEndpointAPIVersion) - if err != nil { - // Generation can still proceed from the agent source alone, so a - // failure to read the agent is reported without stopping. - if !quiet { - fmt.Fprintf(out, " warning: could not read agent %q for generation context: %v\n", - agentName, err) - } - return "", nil - } - instructions := agent.Instructions() - if instructions != "" && !quiet { - fmt.Fprintf(out, " Seeding generation from the instructions of agent %q.\n", agentName) - } - return instructions, nil -} - -// generateDataset submits the generation job and downloads what it produced. -func (dc *datasetContext) generateDataset( - ctx context.Context, - plan generationPlan, - out io.Writer, - noWait, quiet bool, -) error { - if !quiet { - fmt.Fprintf(out, "Generating dataset %s (%d samples)...\n", plan.Name, plan.SampleSize) - } - - sources, unbuildable := gen_api.BuildGenerationSources( - plan.From, plan.Agent, "", plan.Instruction, plan.traceOptions()) - if err := refuseUnbuildableSources(unbuildable); err != nil { - return err - } - req := gen_api.NewDataGenerationJobRequest(plan.Name, plan.Model, plan.SampleSize, sources) - - job, err := dc.genClient.CreateDataGenerationJob(ctx, req, DataGenerationAPIVersion) - if err != nil { - return fmt.Errorf("submitting the data generation job: %w", err) - } - if noWait { - fmt.Fprintf(out, " submitted job %s\n", job.ID) - fmt.Fprintf(out, "\nReattach with: azd ai dataset job show %s\n", job.ID) - return nil - } - - completed, err := dc.pollGeneration(ctx, job.ID, DataGenerationAPIVersion, - dc.genClient.GetDataGenerationJob) - if err != nil && isAgentSeededGenerationFailure(err) { - // Agent-seeded generation fails server-side for every agent, while the - // same request carrying only the prompt succeeds. Failing the whole - // command would block the documented flow on a defect the user cannot - // do anything about, so retry without the agent and say so. - promptOnly := gen_api.WithoutAgentSource(sources) - if gen_api.HasPromptSource(promptOnly) { - fmt.Fprintf(out, - " warning: generating from agent %q failed in the service; "+ - "retrying from the instruction alone.\n", plan.Agent) - - req = gen_api.NewDataGenerationJobRequest( - plan.Name, plan.Model, plan.SampleSize, promptOnly) - job, err = dc.genClient.CreateDataGenerationJob(ctx, req, DataGenerationAPIVersion) - if err != nil { - return fmt.Errorf("submitting the data generation job: %w", err) - } - completed, err = dc.pollGeneration(ctx, job.ID, DataGenerationAPIVersion, - dc.genClient.GetDataGenerationJob) - } - } - if err != nil { - return fmt.Errorf("data generation: %w", explainDataGenerationFailure(err, plan.Agent)) - } - - name, version := completed.ResolvedNameVersion() - if name == "" { - return fmt.Errorf("the data generation job returned no dataset reference") - } - - // Confirm the version exists before reading it, so a missing dataset is - // reported as such rather than as a download failure. - if _, err := dc.datasetClient.GetDataset(ctx, name, version, ProjectEndpointAPIVersion); err != nil { - return fmt.Errorf("reading the generated dataset %q: %w", name, err) - } - content, err := dc.datasetClient.DownloadDatasetContent( - ctx, name, version, ProjectEndpointAPIVersion) - if err != nil { - return fmt.Errorf("downloading the generated dataset %q: %w", name, err) - } - - path := artifactPath(plan.OutputDir, plan.Name) - if err := os.MkdirAll(filepath.Dir(path), 0o750); err != nil { - return fmt.Errorf("creating %q: %w", filepath.Dir(path), err) - } - if err := os.WriteFile(path, content, 0o600); err != nil { - return fmt.Errorf("writing %q: %w", path, err) - } - - if quiet { - return emitJSON(out, map[string]any{ - "dataset": name, "version": version, "path": filepath.ToSlash(path), - }) - } - fmt.Fprintf(out, " wrote %s\n", filepath.ToSlash(path)) - - // The datasets: catalog belongs to `azure.ai.eval`, which owns eval.yaml, - // so registering the file there is a separate step rather than a silent - // cross-extension write. - fmt.Fprintf(out, "\nNext: azd ai eval dataset create %s --from-file %s\n", - plan.Name, filepath.ToSlash(path)) - return nil -} - -// pollGeneration waits for a generation job using the raised budget. -func (dc *datasetContext) pollGeneration( - ctx context.Context, - operationID, apiVersion string, - get gen_api.GetJobFunc, -) (*gen_api.GenerationJob, error) { - poller := gen_api.NewPoller(operationID, apiVersion, get) - poller.Options = generatePollBudget - return poller.Poll(ctx) -} - -// refuseUnbuildableSources reports a --from the plan could not honour. -// -// Submitting anyway would run a billed job seeded from less than was asked for -// and return a plausible-looking artifact, which is the worst outcome: the -// caller has no way to tell it apart from one built the way they intended. -func refuseUnbuildableSources(kinds []string) error { - if len(kinds) == 0 { - return nil - } - reasons := map[string]string{ - generateFromPrompt: "--from prompt needs --agent-instruction or --agent-instruction-file", - generateFromAgent: "--from agent needs a target agent; pass --target", - generateFromFile: "--from file is not a generation source; " + - "register the file with `azd ai dataset create` instead", - } - messages := make([]string, 0, len(kinds)) - for _, k := range kinds { - if reason, ok := reasons[k]; ok { - messages = append(messages, reason) - continue - } - messages = append(messages, fmt.Sprintf("--from %s cannot be built from this plan", k)) - } - return errors.New(strings.Join(messages, "; ")) -} - -// isAgentSeededGenerationFailure recognizes the service-side failure that hits -// every agent, so it can be retried without the agent rather than surfaced. -func isAgentSeededGenerationFailure(err error) bool { - if err == nil { - return false - } - text := err.Error() - return strings.Contains(text, "DataGenerationJobSystemError") || - strings.Contains(text, "Something went wrong during data generation") -} - -// explainDataGenerationFailure adds context to the service's opaque system -// error, which says only that something went wrong and to try again — sending -// users into a retry loop against a deterministic failure. -func explainDataGenerationFailure(err error, agentName string) error { - if err == nil || agentName == "" || !isAgentSeededGenerationFailure(err) { - return err - } - return fmt.Errorf( - "%w\n\nSeeding generation from agent %q fails server-side for every agent. "+ - "Pass --agent-instruction to generate from the instruction alone.", - err, agentName) -} diff --git a/cli/azd/extensions/azure.ai.dataset/internal/cmd/helpers_test.go b/cli/azd/extensions/azure.ai.dataset/internal/cmd/helpers_test.go index ae4a253b342..8c226d8b5e4 100644 --- a/cli/azd/extensions/azure.ai.dataset/internal/cmd/helpers_test.go +++ b/cli/azd/extensions/azure.ai.dataset/internal/cmd/helpers_test.go @@ -4,68 +4,12 @@ package cmd import ( - "errors" - "os" - "path/filepath" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) -// A sample count the service would reject is refused here, before a job is -// submitted and billed. -func TestValidateSampleSize(t *testing.T) { - assert.NoError(t, validateSampleSize(0), "unset means take the default") - assert.NoError(t, validateSampleSize(minSampleSize)) - assert.NoError(t, validateSampleSize(maxSampleSize)) - assert.NoError(t, validateSampleSize(100)) - - for _, n := range []int{1, minSampleSize - 1, maxSampleSize + 1, -5} { - err := validateSampleSize(n) - require.Errorf(t, err, "%d is out of range", n) - assert.Contains(t, err.Error(), "15") - assert.Contains(t, err.Error(), "1000", "the message has to name the range it enforces") - } -} - -// --from names a source the service has a path for. A typo caught here costs -// nothing; the same typo reaching the service costs a job. -func TestValidateGenerateSource(t *testing.T) { - assert.NoError(t, validateGenerateSource(""), "unset means take the default") - for _, s := range generateSources { - assert.NoErrorf(t, validateGenerateSource(s), "%q is a documented source", s) - } - - err := validateGenerateSource("tracez") - require.Error(t, err) - assert.Contains(t, err.Error(), "tracez") - for _, s := range generateSources { - assert.Containsf(t, err.Error(), s, "the refusal has to list %q as an option", s) - } -} - -// The generated file lands where `azd ai eval init` scaffolds, so a generated -// dataset is already where an evaluation configuration expects it. -func TestArtifactPath(t *testing.T) { - assert.Equal(t, - filepath.Join("evals", "datasets", "support-regression.jsonl"), - artifactPath(defaultOutputDir, "support-regression")) - assert.Equal(t, - filepath.Join("out", "x.jsonl"), - artifactPath("out", "x")) -} - -// The spec's default: traces when the project has Application Insights -// connected, otherwise the agent. The connection string is how a project says -// it collects traces at all, so asking for traces without one would submit a -// billed job against nothing. -func TestDefaultGenerationSource(t *testing.T) { - assert.Equal(t, []string{generateFromTraces}, - defaultGenerationSource("InstrumentationKey=00000000-0000-0000-0000-000000000000")) - assert.Equal(t, []string{generateFromAgent}, defaultGenerationSource("")) -} - // The one difference between create and update, and the only thing stopping a // create from silently publishing version 2 of someone else's dataset. func TestCheckAssetExistence(t *testing.T) { @@ -80,98 +24,3 @@ func TestCheckAssetExistence(t *testing.T) { require.Error(t, err) assert.Contains(t, err.Error(), "create") } - -// Generation is billed and the file is checked in, so overwriting one is a -// decision the caller makes rather than a side effect. -func TestRefuseExistingArtifact(t *testing.T) { - dir := t.TempDir() - path := filepath.Join(dir, "support.jsonl") - - assert.NoError(t, refuseExistingArtifact(path, false), "nothing there yet") - - require.NoError(t, os.WriteFile(path, []byte("{}"), 0o600)) - - err := refuseExistingArtifact(path, false) - require.Error(t, err) - assert.Contains(t, err.Error(), "--force") - assert.Contains(t, err.Error(), "--output-dir", "both ways out have to be named") - - assert.NoError(t, refuseExistingArtifact(path, true), "--force is the way through") -} - -// --from is a request, and one the plan cannot honour has to stop the command -// rather than quietly submit a job seeded from less than was asked for. -func TestRefuseUnbuildableSources(t *testing.T) { - assert.NoError(t, refuseUnbuildableSources(nil)) - assert.NoError(t, refuseUnbuildableSources([]string{})) - - tests := []struct{ kind, says string }{ - {generateFromPrompt, "--agent-instruction"}, - {generateFromAgent, "--target"}, - {generateFromFile, "azd ai dataset create"}, - } - for _, tt := range tests { - t.Run(tt.kind, func(t *testing.T) { - err := refuseUnbuildableSources([]string{tt.kind}) - require.Error(t, err) - assert.Contains(t, err.Error(), tt.says, - "the error has to name the way out, not just the problem") - }) - } - - both := refuseUnbuildableSources([]string{generateFromPrompt, generateFromAgent}) - require.Error(t, both) - assert.Contains(t, both.Error(), "--agent-instruction") - assert.Contains(t, both.Error(), "--target", - "two unhonoured sources are two things to fix, so both are reported at once") -} - -// The service's system error says only that something went wrong and to try -// again, which sends users into a retry loop against a deterministic failure. -func TestExplainDataGenerationFailure(t *testing.T) { - systemErr := errors.New("DataGenerationJobSystemError: Something went wrong during data generation") - - explained := explainDataGenerationFailure(systemErr, "support-agent") - require.Error(t, explained) - assert.Contains(t, explained.Error(), "support-agent") - assert.Contains(t, explained.Error(), "--agent-instruction", - "the explanation has to name the way around it") - assert.ErrorIs(t, explained, systemErr, "the original must still be reachable") - - assert.NoError(t, explainDataGenerationFailure(nil, "support-agent")) - assert.Equal(t, systemErr, explainDataGenerationFailure(systemErr, ""), - "with no agent named there is nothing to explain") - - other := errors.New("connection reset") - assert.Equal(t, other, explainDataGenerationFailure(other, "support-agent"), - "an unrelated failure must not be blamed on the agent") -} - -func TestIsAgentSeededGenerationFailure(t *testing.T) { - assert.True(t, isAgentSeededGenerationFailure(errors.New("DataGenerationJobSystemError"))) - assert.True(t, isAgentSeededGenerationFailure( - errors.New("Something went wrong during data generation"))) - assert.False(t, isAgentSeededGenerationFailure(errors.New("429 Too Many Requests"))) - assert.False(t, isAgentSeededGenerationFailure(nil)) -} - -// The two job types share an id shape, so reaching for the wrong group is the -// likely mistake and the error has to say where the other one is. -func TestJobLookupErrorPointsAtTheEvaluatorGroup(t *testing.T) { - err := jobLookupError("dgj_01", errors.New("boom")) - require.Error(t, err) - assert.Contains(t, err.Error(), "dgj_01") - - // A non-404 is reported as itself rather than as a wrong-group guess. - assert.NotContains(t, err.Error(), "azd ai eval evaluator job") -} - -// A window narrows a traces request; its absence is not a request for none. -func TestGenerationPlanTraceOptions(t *testing.T) { - assert.Nil(t, generationPlan{}.traceOptions()) - assert.Nil(t, generationPlan{TraceDays: -1}.traceOptions()) - - opts := generationPlan{TraceDays: 7}.traceOptions() - require.NotNil(t, opts) - assert.Equal(t, 7, opts.Days) -} diff --git a/cli/azd/extensions/azure.ai.dataset/internal/cmd/instruction_test.go b/cli/azd/extensions/azure.ai.dataset/internal/cmd/instruction_test.go deleted file mode 100644 index 086c9be62e3..00000000000 --- a/cli/azd/extensions/azure.ai.dataset/internal/cmd/instruction_test.go +++ /dev/null @@ -1,132 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package cmd - -import ( - "bytes" - "context" - "encoding/json" - "net/http" - "net/http/httptest" - "testing" - - "azureaidataset/internal/pkg/gen_api" - - "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" - "github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -// contextServingAgent builds a datasetContext whose generation client answers -// the agent read with status and body. -func contextServingAgent(t *testing.T, status int, body string) *datasetContext { - t.Helper() - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.Header().Set("Content-Type", "application/json") - w.WriteHeader(status) - if body != "" { - _, _ = w.Write([]byte(body)) - } - })) - t.Cleanup(srv.Close) - - pipeline := runtime.NewPipeline("test", "v1", runtime.PipelineOptions{}, - &policy.ClientOptions{Retry: policy.RetryOptions{MaxRetries: -1}}) - return &datasetContext{genClient: gen_api.NewClientFromPipeline(srv.URL, pipeline)} -} - -// agentBody is a catalog agent as the service returns one: versions inlined, -// with only `latest` populated on a plain read. -func agentBody(t *testing.T, instructions string) string { - t.Helper() - raw, err := json.Marshal(map[string]any{ - "name": "support-bot", - "versions": map[string]any{ - "latest": map[string]any{ - "version": "1", - "definition": map[string]any{"instructions": instructions}, - }, - }, - }) - require.NoError(t, err) - return string(raw) -} - -// What the caller passed wins: --instruction is the explicit answer to the -// question the agent is only a fallback for. -func TestResolveGenerationInstructionPrefersTheExplicitValue(t *testing.T) { - dc := contextServingAgent(t, http.StatusOK, agentBody(t, "from the agent")) - var out bytes.Buffer - - got, err := dc.resolveGenerationInstruction( - context.Background(), "from the flag", "support-bot", &out, false) - - require.NoError(t, err) - assert.Equal(t, "from the flag", got) - assert.Empty(t, out.String(), "nothing was looked up, so nothing is announced") -} - -// With no agent to read there is nothing to fall back to, and an empty -// instruction is a valid request rather than an error. -func TestResolveGenerationInstructionWithoutAnAgent(t *testing.T) { - dc := contextServingAgent(t, http.StatusOK, agentBody(t, "unused")) - var out bytes.Buffer - - got, err := dc.resolveGenerationInstruction(context.Background(), "", "", &out, false) - - require.NoError(t, err) - assert.Empty(t, got) -} - -// The service accepts an agent source meant to pull these instructions, but it -// fails for every agent, so they are read here instead. Losing this read makes -// generation silently ignore the agent it was pointed at. -func TestResolveGenerationInstructionReadsTheAgent(t *testing.T) { - dc := contextServingAgent(t, http.StatusOK, agentBody(t, "Answer support questions politely.")) - var out bytes.Buffer - - got, err := dc.resolveGenerationInstruction( - context.Background(), "", "support-bot", &out, false) - - require.NoError(t, err) - assert.Equal(t, "Answer support questions politely.", got) - assert.Contains(t, out.String(), "support-bot", - "the user is told what the generation was seeded from") -} - -// Generation can still proceed from the agent source alone, so a failure to -// read the agent is reported and stepped over rather than ending the command. -func TestResolveGenerationInstructionWarnsButContinues(t *testing.T) { - dc := contextServingAgent(t, http.StatusNotFound, `{"error":{"code":"NotFound"}}`) - var out bytes.Buffer - - got, err := dc.resolveGenerationInstruction( - context.Background(), "", "missing-bot", &out, false) - - require.NoError(t, err, "an unreadable agent must not fail the generation") - assert.Empty(t, got) - assert.Contains(t, out.String(), "warning") - assert.Contains(t, out.String(), "missing-bot") -} - -// --quiet is for scripts, where the progress lines are noise on stdout that a -// caller may be parsing. -func TestResolveGenerationInstructionStaysQuiet(t *testing.T) { - var out bytes.Buffer - - found := contextServingAgent(t, http.StatusOK, agentBody(t, "Answer politely.")) - got, err := found.resolveGenerationInstruction( - context.Background(), "", "support-bot", &out, true) - require.NoError(t, err) - assert.Equal(t, "Answer politely.", got, "quiet changes what is printed, not what is resolved") - assert.Empty(t, out.String()) - - missing := contextServingAgent(t, http.StatusNotFound, `{}`) - _, err = missing.resolveGenerationInstruction( - context.Background(), "", "missing-bot", &out, true) - require.NoError(t, err) - assert.Empty(t, out.String(), "even the warning is suppressed") -} diff --git a/cli/azd/extensions/azure.ai.dataset/internal/cmd/job.go b/cli/azd/extensions/azure.ai.dataset/internal/cmd/job.go deleted file mode 100644 index 05860a57c77..00000000000 --- a/cli/azd/extensions/azure.ai.dataset/internal/cmd/job.go +++ /dev/null @@ -1,197 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package cmd - -import ( - "fmt" - - "azureaidataset/internal/pkg/gen_api" - - "github.com/spf13/cobra" -) - -// The job group nests under `dataset` rather than sitting at the root, because -// generation runs as two independent long-running resources — one for datasets, -// one for evaluators — sharing no collection. The evaluator half lives in -// `azure.ai.evaluations`; a shared top-level `job show ` would have to guess -// which endpoint to call from an id prefix that is not a documented contract. - -func newJobCommand() *cobra.Command { - cmd := &cobra.Command{ - Use: "job", - Short: "Inspect, cancel and delete dataset generation jobs.", - Long: "Inspect, cancel and delete dataset generation jobs.\n\n" + - "This is the resume path for `dataset generate`: a job started with " + - "--no-wait, or one whose client was interrupted, is reattached to here " + - "rather than restarted.", - } - cmd.AddCommand( - newJobListCommand(), - newJobShowCommand(), - newJobCancelCommand(), - newJobDeleteCommand(), - ) - return cmd -} - -func newJobListCommand() *cobra.Command { - var endpointFlg string - - cmd := &cobra.Command{ - Use: "list", - Short: "List the project's dataset generation jobs.", - Args: cobra.NoArgs, - RunE: func(cmd *cobra.Command, args []string) error { - ctx := cmd.Context() - dc, err := newDatasetContext(ctx, endpointFlg) - if err != nil { - return err - } - defer dc.Close() - - out, err := dc.genClient.ListDataGenerationJobs(ctx, ProjectEndpointAPIVersion) - if err != nil { - return fmt.Errorf("listing dataset generation jobs: %w", err) - } - jobs := out.Data - - if isJSON(cmd) { - return emitJSONList(cmd.OutOrStdout(), jobs) - } - if len(jobs) == 0 { - fmt.Fprintln(cmd.OutOrStdout(), "No dataset generation jobs found.") - return nil - } - table := make([][]string, 0, len(jobs)) - for _, j := range jobs { - table = append(table, []string{j.ID, j.Status}) - } - return emitTable(cmd.OutOrStdout(), []string{"JOB ID", "STATUS"}, table) - }, - } - - cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") - return cmd -} - -func newJobShowCommand() *cobra.Command { - var endpointFlg string - - cmd := &cobra.Command{ - Use: "show ", - Short: "Show a dataset generation job.", - Args: cobra.ExactArgs(1), - RunE: func(cmd *cobra.Command, args []string) error { - jobID := args[0] - - ctx := cmd.Context() - dc, err := newDatasetContext(ctx, endpointFlg) - if err != nil { - return err - } - defer dc.Close() - - job, err := dc.genClient.GetDataGenerationJob(ctx, jobID, ProjectEndpointAPIVersion) - if err != nil { - return jobLookupError(jobID, err) - } - - if isJSON(cmd) { - return emitJSON(cmd.OutOrStdout(), job) - } - fmt.Fprintf(cmd.OutOrStdout(), "%s %s\n", job.ID, job.Status) - if job.Error != nil && job.Error.Message != "" { - fmt.Fprintf(cmd.OutOrStdout(), "error: %s\n", job.Error.Message) - } - return nil - }, - } - - cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") - return cmd -} - -func newJobCancelCommand() *cobra.Command { - var endpointFlg string - - cmd := &cobra.Command{ - Use: "cancel ", - Short: "Cancel an in-flight dataset generation job.", - Args: cobra.ExactArgs(1), - RunE: func(cmd *cobra.Command, args []string) error { - jobID := args[0] - - ctx := cmd.Context() - dc, err := newDatasetContext(ctx, endpointFlg) - if err != nil { - return err - } - defer dc.Close() - - canceled, err := dc.genClient.CancelDataGenerationJob(ctx, jobID, ProjectEndpointAPIVersion) - if err != nil { - return jobLookupError(jobID, err) - } - - if isJSON(cmd) { - return emitJSON(cmd.OutOrStdout(), canceled) - } - fmt.Fprintf(cmd.OutOrStdout(), "Cancelled dataset generation job %s (%s)\n", - jobID, canceled.Status) - return nil - }, - } - - cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") - return cmd -} - -func newJobDeleteCommand() *cobra.Command { - var endpointFlg string - - cmd := &cobra.Command{ - Use: "delete ", - Short: "Delete a dataset generation job record.", - Long: "Delete a dataset generation job record.\n\n" + - "The dataset the job produced is already registered as its own version " + - "and is not affected.", - Args: cobra.ExactArgs(1), - RunE: func(cmd *cobra.Command, args []string) error { - jobID := args[0] - - ctx := cmd.Context() - dc, err := newDatasetContext(ctx, endpointFlg) - if err != nil { - return err - } - defer dc.Close() - - if err := dc.genClient.DeleteDataGenerationJob(ctx, jobID, ProjectEndpointAPIVersion); err != nil { - return jobLookupError(jobID, err) - } - - if isJSON(cmd) { - return emitJSON(cmd.OutOrStdout(), map[string]string{ - "id": jobID, "kind": "dataset", "status": "deleted", - }) - } - fmt.Fprintf(cmd.OutOrStdout(), "Deleted dataset generation job %s\n", jobID) - return nil - }, - } - - cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") - return cmd -} - -// jobLookupError names the evaluator group, because the two job types share an -// id shape and reaching for the wrong one is the likely mistake. -func jobLookupError(jobID string, err error) error { - if gen_api.IsNotFound(err) { - return fmt.Errorf( - "no dataset generation job %q in this project; if it generated an "+ - "evaluator, use `azd ai eval evaluator job` instead", jobID) - } - return fmt.Errorf("reading dataset generation job %s: %w", jobID, err) -} diff --git a/cli/azd/extensions/azure.ai.dataset/internal/cmd/root.go b/cli/azd/extensions/azure.ai.dataset/internal/cmd/root.go index e96c9d00927..c53f715bb58 100644 --- a/cli/azd/extensions/azure.ai.dataset/internal/cmd/root.go +++ b/cli/azd/extensions/azure.ai.dataset/internal/cmd/root.go @@ -17,7 +17,7 @@ func NewRootCommand() *cobra.Command { Name: "dataset", Use: "dataset [options]", Short: fmt.Sprintf( - "Create, generate and version Foundry datasets from your terminal. %s", + "Register and version Foundry datasets from your terminal. %s", color.YellowString("(Beta)"), ), }) @@ -40,15 +40,15 @@ func NewRootCommand() *cobra.Command { return nil } + // Generation stays with `azure.ai.evaluations`: it writes the `datasets:` + // entry in evals/eval.yaml, which is that extension's file. rootCmd.AddCommand( newDatasetCreateCommand(), newDatasetUpdateCommand(), - newDatasetGenerateCommand(), newDatasetListCommand(), newDatasetShowCommand(), newDatasetDeleteCommand(), newDatasetVersionsCommand(), - newJobCommand(), ) // The manifest declares the `metadata` capability, which azd uses to diff --git a/cli/azd/extensions/azure.ai.dataset/internal/cmd/surface_test.go b/cli/azd/extensions/azure.ai.dataset/internal/cmd/surface_test.go index 001039e3ac6..e8318e4580c 100644 --- a/cli/azd/extensions/azure.ai.dataset/internal/cmd/surface_test.go +++ b/cli/azd/extensions/azure.ai.dataset/internal/cmd/surface_test.go @@ -32,18 +32,13 @@ func walk(t *testing.T, cmd *cobra.Command, path []string, visit func(string, *c } } -// The command tree is the spec's `azd ai dataset` table. The groups moved here -// from azure.ai.evaluations, so this is also what says the move was complete. +// The command tree is the spec's `azd ai dataset` table. The CRUD groups moved +// here from azure.ai.evaluations; `generate` deliberately did not, because it +// writes the `datasets:` entry in evals/eval.yaml, which that extension owns. func TestCommandTreeMatchesTheSpec(t *testing.T) { want := []string{ "create", "delete", - "generate", - "job", - "job cancel", - "job delete", - "job list", - "job show", "list", "show", "update", @@ -108,20 +103,16 @@ func TestServiceCommandsTakeProjectEndpoint(t *testing.T) { }) } -// The spec says --from "selects one or more" of the sources, so it has to be -// repeatable. Declared as a plain string it would still accept every documented -// single-source invocation and silently keep only the last of a repeated one. -func TestGenerateFromTakesMoreThanOneSource(t *testing.T) { - flag := find(t, "generate").Flags().Lookup("from") - require.NotNil(t, flag, "generate must offer --from") - - assert.Equal(t, "stringSlice", flag.Value.Type(), - "--from selects one or more sources, so it cannot be a single string") - - for _, source := range generateSources { - assert.Containsf(t, flag.Usage, source, - "--from accepts %q, so its help has to say so", source) - } +// Generation stays with `azure.ai.evaluations`, so nothing here may grow a +// `--from`: a second generate would be a second place for the catalog write to +// go missing. +func TestNoGenerationCommandLandsHere(t *testing.T) { + walk(t, NewRootCommand(), nil, func(path string, cmd *cobra.Command) { + assert.Nilf(t, cmd.Flags().Lookup("from"), + "%s offers --from; generation belongs to azure.ai.evaluations", path) + assert.NotEqualf(t, "generate", cmd.Name(), + "%s is a generation command; it belongs to azure.ai.evaluations", path) + }) } // Messages that tell a user what to run next have to name a command that @@ -193,12 +184,10 @@ func TestSuggestedCommandsExist(t *testing.T) { // eval extension's version of a command it serves itself sends them somewhere // they may not have installed. // -// generate's "register this in an eval configuration" line is the one real -// exception, because eval.yaml genuinely belongs to the other extension. +// Nothing here may suggest one any more: `generate` was the only command with a +// reason to, and it stayed with azure.ai.evaluations. func TestNoStaleEvalDatasetSuggestions(t *testing.T) { - allowed := map[string]bool{ - "azd ai eval dataset create": true, // registering a generated file in eval.yaml - } + allowed := map[string]bool{} err := filepath.WalkDir("../..", func(path string, d fs.DirEntry, err error) error { if err != nil { @@ -229,7 +218,6 @@ func TestNoStaleEvalDatasetSuggestions(t *testing.T) { // siblingNamespaces are the other Foundry extensions this one points users at. var siblingNamespaces = map[string]bool{ - "eval": true, // registering a generated dataset in an eval configuration "project": true, // `azd ai project set` owns the shared endpoint context } diff --git a/cli/azd/extensions/azure.ai.dataset/internal/pkg/dataset_api/operations.go b/cli/azd/extensions/azure.ai.dataset/internal/pkg/dataset_api/operations.go index a37f97c79c0..23e59cc9e8b 100644 --- a/cli/azd/extensions/azure.ai.dataset/internal/pkg/dataset_api/operations.go +++ b/cli/azd/extensions/azure.ai.dataset/internal/pkg/dataset_api/operations.go @@ -167,6 +167,15 @@ func IsVersionConflict(err error) bool { return respErr.StatusCode == http.StatusConflict } +// IsNotFound reports whether the service answered 404. +func IsNotFound(err error) bool { + var respErr *azcore.ResponseError + if !errors.As(err, &respErr) { + return false + } + return respErr.StatusCode == http.StatusNotFound +} + // UploadNewVersion reads the first JSONL file from localDir, computes the next // version from currentVersion, and uploads it as a new dataset version using // the 3-step pending upload flow: diff --git a/cli/azd/extensions/azure.ai.dataset/internal/pkg/gen_api/errors.go b/cli/azd/extensions/azure.ai.dataset/internal/pkg/gen_api/errors.go deleted file mode 100644 index bd68ce76e6c..00000000000 --- a/cli/azd/extensions/azure.ai.dataset/internal/pkg/gen_api/errors.go +++ /dev/null @@ -1,49 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package gen_api - -import ( - "errors" - "net/http" - "strings" - - "github.com/Azure/azure-sdk-for-go/sdk/azcore" -) - -// IsConflict reports whether the service refused because the resource is busy. -func IsConflict(err error) bool { - var respErr *azcore.ResponseError - if !errors.As(err, &respErr) { - return false - } - return respErr.StatusCode == http.StatusConflict -} - -// IsNotFound reports whether the service answered 404. -func IsNotFound(err error) bool { - var respErr *azcore.ResponseError - if !errors.As(err, &respErr) { - return false - } - return respErr.StatusCode == http.StatusNotFound -} - -// IsTransientError reports whether err is worth retrying: throttling, a server -// fault, or a dropped connection. -func IsTransientError(err error) bool { - if err == nil { - return false - } - - var respErr *azcore.ResponseError - if errors.As(err, &respErr) { - return respErr.StatusCode == http.StatusTooManyRequests || - respErr.StatusCode >= http.StatusInternalServerError - } - - msg := err.Error() - return strings.Contains(msg, "connection reset") || - strings.Contains(msg, "connection refused") || - strings.Contains(msg, "EOF") -} diff --git a/cli/azd/extensions/azure.ai.dataset/internal/pkg/gen_api/generation.go b/cli/azd/extensions/azure.ai.dataset/internal/pkg/gen_api/generation.go deleted file mode 100644 index fd7af65c614..00000000000 --- a/cli/azd/extensions/azure.ai.dataset/internal/pkg/gen_api/generation.go +++ /dev/null @@ -1,180 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package gen_api - -import ( - "path/filepath" - "strings" - "time" -) - -// --------------------------------------------------------------------------- -// Generation source building -// --------------------------------------------------------------------------- - -// TraceOptions holds optional trace inclusion parameters for generation sources. -type TraceOptions struct { - Days int -} - -// WithoutAgentSource returns the sources with the agent entry removed. -// -// Agent-seeded data generation currently fails server-side for every agent, -// while the same request carrying only the prompt succeeds, so this is what a -// retry falls back to. -func WithoutAgentSource(sources []GenerationSource) []GenerationSource { - kept := make([]GenerationSource, 0, len(sources)) - for _, s := range sources { - if s.Type == "agent" { - continue - } - kept = append(kept, s) - } - return kept -} - -// HasPromptSource reports whether anything remains to generate from. -func HasPromptSource(sources []GenerationSource) bool { - for _, s := range sources { - if s.Type == "prompt" && s.Prompt != "" { - return true - } - } - return false -} - -// BuildGenerationSources emits the sources the caller selected, in a stable -// order, along with the ones it asked for and nothing could be built from. -// -// kinds is what --from named. An empty kinds means "whatever this plan has to -// offer" and reports nothing missing: the caller expressed no preference, so -// there is nothing to disappoint. Naming a kind explicitly is a request, and a -// request that cannot be built is worth saying out loud rather than quietly -// submitting a job seeded from less than was asked for. -func BuildGenerationSources( - kinds []string, - agentName, version, instruction string, - traces *TraceOptions, -) (sources []GenerationSource, unbuildable []string) { - want := map[string]bool{} - for _, k := range kinds { - want[k] = true - } - // Empty kinds selects everything available; a populated one selects only - // what it names. - selected := func(kind string) bool { - return len(want) == 0 || want[kind] - } - // asked distinguishes "the default swept this up" from "the user typed it", - // which is what decides whether an empty-handed source is an error. - asked := func(kind string) bool { return want[kind] } - - // The agent is settled first because whether it was built decides whether - // its instructions have anything to be the instructions of. - var agentSource *GenerationSource - if selected("agent") { - switch { - case agentName != "": - agentSource = &GenerationSource{Type: "agent", AgentName: agentName} - if version != "" { - agentSource.AgentVersion = version - } - case asked("agent"): - unbuildable = append(unbuildable, "agent") - } - } - - // Generating from an agent means generating from its instructions, so they - // travel with it as a prompt. That is also the only shape the service - // currently honours: the agent source alone fails for every agent, and the - // prompt is what the retry in generateDataset falls back to. Without this, - // `--from agent` would be a request that always fails. - promptCarriesTheAgent := agentSource != nil && asked("agent") - if selected("prompt") || promptCarriesTheAgent { - switch { - case instruction != "": - sources = append(sources, GenerationSource{ - Type: "prompt", - Prompt: instruction, - }) - case asked("prompt"): - unbuildable = append(unbuildable, "prompt") - } - } - - if agentSource != nil { - sources = append(sources, *agentSource) - } - - if selected("traces") { - // A window narrows the request; it does not authorize it. Asking for - // traces without one means every trace the agent has. - switch { - case traces != nil && traces.Days > 0: - sources = append(sources, GenerationSource{ - Type: "traces", - AgentName: agentName, - StartTime: time.Now().AddDate(0, 0, -traces.Days).Unix(), - }) - case asked("traces"): - sources = append(sources, GenerationSource{ - Type: "traces", - AgentName: agentName, - }) - } - } - - // The service takes a file's rows through the dataset upload path, not - // through a generation source, so there is nothing here to build one from. - if asked("file") { - unbuildable = append(unbuildable, "file") - } - - return sources, unbuildable -} - -// --------------------------------------------------------------------------- -// Request builders -// --------------------------------------------------------------------------- - -// NewDataGenerationJobRequest builds a DataGenerationJobRequest from the -// provided parameters. Currently, it's always "simple_qna" type with multiple sources -func NewDataGenerationJobRequest( - name, evalModel string, - maxSamples int, - sources []GenerationSource, -) *DataGenerationJobRequest { - return &DataGenerationJobRequest{ - Inputs: DataGenerationInputs{ - Name: name, - Scenario: "evaluation", - Options: DataGenerationOptions{ - Type: "simple_qna", - MaxSamples: maxSamples, - ModelOptions: ModelOptions{ - Model: evalModel, - }, - }, - Sources: sources, - }, - } -} - -// --------------------------------------------------------------------------- -// Dataset name detection -// --------------------------------------------------------------------------- - -// IsDatasetName returns true when the value looks like a registered dataset -// name rather than a local file path. A name has no path separators and no -// common data-file extension (.jsonl, .json, .csv). -func IsDatasetName(value string) bool { - if value == "" { - return false - } - if strings.ContainsAny(value, "/\\") { - return false - } - ext := strings.ToLower(filepath.Ext(value)) - return ext != ".jsonl" && ext != ".json" && ext != ".csv" -} diff --git a/cli/azd/extensions/azure.ai.dataset/internal/pkg/gen_api/generation_test.go b/cli/azd/extensions/azure.ai.dataset/internal/pkg/gen_api/generation_test.go deleted file mode 100644 index d6cce82e5e8..00000000000 --- a/cli/azd/extensions/azure.ai.dataset/internal/pkg/gen_api/generation_test.go +++ /dev/null @@ -1,232 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package gen_api - -import ( - "testing" - "time" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -// kindsOf reduces the built sources to what --from talks about, which is the -// only part these tests are asserting on. -func kindsOf(sources []GenerationSource) []string { - kinds := make([]string, 0, len(sources)) - for _, s := range sources { - kinds = append(kinds, s.Type) - } - return kinds -} - -// Naming a source is a request to send that one, not a hint. Everything the -// plan could otherwise have offered stays out of the request. -func TestBuildGenerationSources_SendsOnlyWhatFromNamed(t *testing.T) { - sources, unbuildable := BuildGenerationSources( - []string{"traces"}, - "support-agent", "3", "answer support questions", - &TraceOptions{Days: 7}, - ) - - assert.Equal(t, []string{"traces"}, kindsOf(sources)) - assert.Empty(t, unbuildable) -} - -// Generating from an agent means generating from its instructions, so asking -// for the agent carries them. It is also the only shape the service honours: -// the agent source on its own fails for every agent, so a `--from agent` that -// dropped the prompt would be a request that always fails. -func TestBuildGenerationSources_AgentCarriesItsInstructions(t *testing.T) { - sources, unbuildable := BuildGenerationSources( - []string{"agent"}, "support-agent", "3", "answer support questions", nil, - ) - - assert.Equal(t, []string{"prompt", "agent"}, kindsOf(sources)) - assert.Equal(t, "answer support questions", sources[0].Prompt) - assert.Empty(t, unbuildable) -} - -// The instructions ride along with the agent; they do not stand in for it. An -// agent nobody named is still nothing to generate from. -func TestBuildGenerationSources_InstructionsDoNotSubstituteForTheAgent(t *testing.T) { - sources, unbuildable := BuildGenerationSources( - []string{"agent"}, "", "", "answer support questions", nil, - ) - - assert.Empty(t, sources) - assert.Equal(t, []string{"agent"}, unbuildable) -} - -// The agent name travels with the traces source: it is what scopes the query -// to this agent's conversations rather than the whole project's. -func TestBuildGenerationSources_TracesCarryTheAgent(t *testing.T) { - sources, _ := BuildGenerationSources( - []string{"traces"}, "support-agent", "", "", &TraceOptions{Days: 7}, - ) - - require.Len(t, sources, 1) - assert.Equal(t, "support-agent", sources[0].AgentName) -} - -// A day window narrows the trace query; it is not what authorizes it. The -// documented `dataset generate --from traces` carries no window, and it -// has to mean "every trace" rather than "no traces". -func TestBuildGenerationSources_TracesWithoutAWindowAreUnbounded(t *testing.T) { - sources, unbuildable := BuildGenerationSources( - []string{"traces"}, "support-agent", "", "", nil, - ) - - require.Len(t, sources, 1) - assert.Equal(t, "traces", sources[0].Type) - assert.Zero(t, sources[0].StartTime, - "an absent window must leave start_time off the wire, not pin it to now") - assert.Empty(t, unbuildable) -} - -func TestBuildGenerationSources_TraceWindowBecomesAStartTime(t *testing.T) { - sources, _ := BuildGenerationSources( - []string{"traces"}, "support-agent", "", "", &TraceOptions{Days: 7}, - ) - - require.Len(t, sources, 1) - want := time.Now().AddDate(0, 0, -7).Unix() - assert.InDelta(t, want, sources[0].StartTime, 60) -} - -// No --from is no preference, so the plan sends everything it happens to have. -func TestBuildGenerationSources_EmptyFromSendsWhatThePlanHas(t *testing.T) { - sources, unbuildable := BuildGenerationSources( - nil, "support-agent", "3", "answer support questions", &TraceOptions{Days: 7}, - ) - - assert.Equal(t, []string{"prompt", "agent", "traces"}, kindsOf(sources)) - assert.Empty(t, unbuildable) -} - -// Expressing no preference cannot disappoint one, so an empty --from reports -// nothing missing however little the plan turns out to hold. -func TestBuildGenerationSources_EmptyFromNeverReportsMissingSources(t *testing.T) { - sources, unbuildable := BuildGenerationSources(nil, "", "", "", nil) - - assert.Empty(t, sources) - assert.Empty(t, unbuildable) -} - -// Asking for a source the plan cannot build has to surface, because the job is -// billed and what comes back looks the same either way. -func TestBuildGenerationSources_ReportsWhatItCouldNotBuild(t *testing.T) { - tests := []struct { - name string - kinds []string - agentName string - instruction string - want []string - }{ - { - name: "prompt without an instruction", - kinds: []string{"prompt"}, - want: []string{"prompt"}, - }, - { - name: "agent without a target", - kinds: []string{"agent"}, - want: []string{"agent"}, - }, - { - name: "file is not a generation source at all", - kinds: []string{"file"}, - want: []string{"file"}, - }, - { - name: "several at once", - kinds: []string{"prompt", "agent"}, - want: []string{"agent", "prompt"}, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - sources, unbuildable := BuildGenerationSources( - tt.kinds, tt.agentName, "", tt.instruction, nil, - ) - - assert.Empty(t, sources) - assert.Equal(t, tt.want, unbuildable) - }) - } -} - -// A request that names two sources and can only build one still reports the -// one it could not, rather than being satisfied by the other's success. -func TestBuildGenerationSources_OneBuiltSourceDoesNotExcuseAMissingOne(t *testing.T) { - sources, unbuildable := BuildGenerationSources( - []string{"agent", "prompt"}, "support-agent", "", "", nil, - ) - - assert.Equal(t, []string{"agent"}, kindsOf(sources)) - assert.Equal(t, []string{"prompt"}, unbuildable) -} - -// `file` is only unbuildable when it was asked for. The default sweep must not -// invent a complaint about a source nobody named. -func TestBuildGenerationSources_FileIsOnlyReportedWhenAskedFor(t *testing.T) { - _, unbuildable := BuildGenerationSources( - nil, "support-agent", "", "instruction", &TraceOptions{Days: 7}, - ) - - assert.Empty(t, unbuildable) -} - -func TestBuildGenerationSources_AgentVersionIsOptional(t *testing.T) { - withVersion, _ := BuildGenerationSources([]string{"agent"}, "support-agent", "3", "", nil) - require.Len(t, withVersion, 1) - assert.Equal(t, "3", withVersion[0].AgentVersion) - - withoutVersion, _ := BuildGenerationSources([]string{"agent"}, "support-agent", "", "", nil) - require.Len(t, withoutVersion, 1) - assert.Empty(t, withoutVersion[0].AgentVersion) -} - -// The retry that saves the documented flow: agent-seeded generation fails -// server-side for every agent, and the same request without the agent source -// succeeds. -func TestWithoutAgentSource(t *testing.T) { - sources := []GenerationSource{ - {Type: "prompt", Prompt: "be helpful"}, - {Type: "agent", AgentName: "support"}, - {Type: "traces", AgentName: "support"}, - } - - kept := WithoutAgentSource(sources) - - assert.Equal(t, []string{"prompt", "traces"}, kindsOf(kept)) - assert.Len(t, sources, 3, "the original must not be modified; it is retried from") -} - -// The retry only happens when something is left to generate from, so this is -// what stops a second billed job that would fail the same way. -func TestHasPromptSource(t *testing.T) { - assert.True(t, HasPromptSource([]GenerationSource{{Type: "prompt", Prompt: "x"}})) - assert.False(t, HasPromptSource([]GenerationSource{{Type: "prompt"}}), - "an empty prompt is nothing to generate from") - assert.False(t, HasPromptSource([]GenerationSource{{Type: "agent", AgentName: "s"}})) - assert.False(t, HasPromptSource(nil)) -} - -// The request body is what the service validates, so the fields it keys on are -// pinned rather than left to whatever the builder happens to set. -func TestNewDataGenerationJobRequest(t *testing.T) { - sources := []GenerationSource{{Type: "prompt", Prompt: "be helpful"}} - - req := NewDataGenerationJobRequest("support-regression", "gpt-4o", 15, sources) - - require.NotNil(t, req) - assert.Equal(t, "support-regression", req.Inputs.Name) - assert.Equal(t, "evaluation", req.Inputs.Scenario) - assert.Equal(t, "simple_qna", req.Inputs.Options.Type) - assert.Equal(t, 15, req.Inputs.Options.MaxSamples) - assert.Equal(t, "gpt-4o", req.Inputs.Options.ModelOptions.Model) - assert.Equal(t, sources, req.Inputs.Sources) -} diff --git a/cli/azd/extensions/azure.ai.dataset/internal/pkg/gen_api/models.go b/cli/azd/extensions/azure.ai.dataset/internal/pkg/gen_api/models.go deleted file mode 100644 index de6ca795b21..00000000000 --- a/cli/azd/extensions/azure.ai.dataset/internal/pkg/gen_api/models.go +++ /dev/null @@ -1,144 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package gen_api - -import ( - "encoding/json" - "strings" -) - -// This is the data-generation half of the evaluation service's API. The -// evaluator half stays with `azure.ai.evaluations`, because only that extension -// generates evaluators. azd extensions share no code, so the shapes both need -// are spelled out in each rather than imported. - -// DataGenerationJobRequest is the request body for CreateDataGenerationJob. -type DataGenerationJobRequest struct { - Inputs DataGenerationInputs `json:"inputs"` -} - -// DataGenerationInputs holds the inputs for a data generation job. -type DataGenerationInputs struct { - Name string `json:"name"` - Scenario string `json:"scenario"` - Options DataGenerationOptions `json:"options"` - Sources []GenerationSource `json:"sources"` -} - -// DataGenerationOptions holds configuration for data generation. -type DataGenerationOptions struct { - Type string `json:"type"` - MaxSamples int `json:"max_samples"` - ModelOptions ModelOptions `json:"model_options"` -} - -// ModelOptions holds the model selection for generation. -type ModelOptions struct { - Model string `json:"model"` -} - -// GenerationSource describes a source used for dataset generation. -type GenerationSource struct { - Type string `json:"type"` - Prompt string `json:"prompt,omitempty"` - AgentName string `json:"agent_name,omitempty"` - AgentVersion string `json:"agent_version,omitempty"` - StartTime int64 `json:"start_time,omitempty"` -} - -// Agent is the part of a catalog agent that describes what it does. -// -// An agent is returned with its versions inlined rather than as a list, and -// only `latest` is populated on a plain read. -type Agent struct { - Name string `json:"name"` - Versions struct { - Latest *AgentVersion `json:"latest"` - } `json:"versions"` -} - -// AgentVersion is one published revision of an agent. -type AgentVersion struct { - Version string `json:"version"` - Definition struct { - Model string `json:"model"` - Instructions string `json:"instructions"` - } `json:"definition"` -} - -// Instructions returns the newest version's system prompt, or "" when the agent -// has no published version. -func (a *Agent) Instructions() string { - if a == nil || a.Versions.Latest == nil { - return "" - } - return strings.TrimSpace(a.Versions.Latest.Definition.Instructions) -} - -// GenerationJob is the response for data generation job operations. -type GenerationJob struct { - ID string `json:"id"` - Status string `json:"status"` - Result json.RawMessage `json:"result,omitempty"` - Error *JobError `json:"error,omitempty"` -} - -// JobError captures error details from a failed generation job. -type JobError struct { - Code string `json:"code,omitempty"` - Message string `json:"message,omitempty"` -} - -// GenerationJobList is the listing envelope the job routes answer with. It is -// `data`, not the `value` the dataset routes use. -type GenerationJobList struct { - Data []GenerationJob `json:"data"` -} - -// ResolvedNameVersion extracts the name and version from the generation job -// result. An empty name means there is no result to read; an empty version -// means the service left it to be resolved as `latest`. -func (j *GenerationJob) ResolvedNameVersion() (string, string) { - name := j.resultStringField("name") - if name == "" { - return "", "" - } - version := j.resultStringField("version") - if version == "" { - version = "latest" - } - return name, version -} - -// resultStringField reads a string field out of the raw Result JSON, trying a -// top-level key before the nested outputs[0] shape the service also returns. -func (j *GenerationJob) resultStringField(key string) string { - if len(j.Result) == 0 { - return "" - } - var m map[string]json.RawMessage - if err := json.Unmarshal(j.Result, &m); err != nil { - return "" - } - - if raw, ok := m[key]; ok { - var s string - if err := json.Unmarshal(raw, &s); err == nil && s != "" { - return s - } - } - - if rawOutputs, ok := m["outputs"]; ok { - var outputs []map[string]json.RawMessage - if err := json.Unmarshal(rawOutputs, &outputs); err == nil && len(outputs) > 0 { - if raw, ok := outputs[0][key]; ok { - var s string - if err := json.Unmarshal(raw, &s); err == nil { - return s - } - } - } - } - return "" -} diff --git a/cli/azd/extensions/azure.ai.dataset/internal/pkg/gen_api/models_test.go b/cli/azd/extensions/azure.ai.dataset/internal/pkg/gen_api/models_test.go deleted file mode 100644 index ff3b17aa425..00000000000 --- a/cli/azd/extensions/azure.ai.dataset/internal/pkg/gen_api/models_test.go +++ /dev/null @@ -1,179 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package gen_api - -import ( - "encoding/json" - "errors" - "net/http" - "testing" - - "github.com/Azure/azure-sdk-for-go/sdk/azcore" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -// The generated dataset's name and version come back inside the job result, -// and the service answers in two shapes. Reading only one leaves the download -// looking for a dataset the job never named. -func TestGenerationJob_ResolvedNameVersion(t *testing.T) { - tests := []struct { - name string - result string - wantName string - wantVersion string - }{ - { - name: "top-level fields", - result: `{"name":"support-regression","version":"3"}`, - wantName: "support-regression", - wantVersion: "3", - }, - { - name: "nested under outputs", - result: `{"outputs":[{"name":"support-regression","version":"2"}]}`, - wantName: "support-regression", - wantVersion: "2", - }, - { - name: "a name with no version resolves as latest", - result: `{"name":"support-regression"}`, - wantName: "support-regression", - wantVersion: "latest", - }, - { - name: "top-level wins over outputs", - result: `{"name":"top","outputs":[{"name":"nested"}]}`, - wantName: "top", - wantVersion: "latest", - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - job := &GenerationJob{Result: json.RawMessage(tt.result)} - - gotName, gotVersion := job.ResolvedNameVersion() - - assert.Equal(t, tt.wantName, gotName) - assert.Equal(t, tt.wantVersion, gotVersion) - }) - } -} - -// No name means no result to read, and both halves come back empty so a caller -// checking either sees the same thing. -func TestGenerationJob_ResolvedNameVersion_NoResult(t *testing.T) { - for _, result := range []string{``, `{}`, `not json`, `{"version":"3"}`} { - job := &GenerationJob{Result: json.RawMessage(result)} - - name, version := job.ResolvedNameVersion() - - assert.Emptyf(t, name, "result %q", result) - assert.Emptyf(t, version, "result %q", result) - } -} - -// The agent's newest instructions are what generation is seeded from, so an -// agent with no published version has nothing to offer rather than an error. -func TestAgent_Instructions(t *testing.T) { - var agent *Agent - assert.Empty(t, agent.Instructions(), "a nil agent is not a panic") - - agent = &Agent{Name: "support"} - assert.Empty(t, agent.Instructions(), "no published version, nothing to read") - - agent.Versions.Latest = &AgentVersion{Version: "3"} - agent.Versions.Latest.Definition.Instructions = "\n Answer politely.\n" - assert.Equal(t, "Answer politely.", agent.Instructions(), - "surrounding whitespace would travel into the prompt") -} - -// A status the service spells differently must not read as still-running, or -// the poller waits out its budget on a job that finished. -func TestParseJobStatus(t *testing.T) { - assert.Equal(t, JobStatusRunning, ParseJobStatus(""), "unset means still running") - assert.Equal(t, JobStatusCompleted, ParseJobStatus("Completed")) - assert.Equal(t, JobStatusSucceeded, ParseJobStatus("SUCCEEDED")) - assert.Equal(t, JobStatusFailed, ParseJobStatus("failed")) -} - -func TestJobStatus_TerminalAndFailed(t *testing.T) { - terminal := []JobStatus{ - JobStatusCompleted, JobStatusSucceeded, - JobStatusFailed, JobStatusCancelled, JobStatusCanceled, - } - for _, s := range terminal { - assert.Truef(t, s.IsTerminal(), "%s is a final state", s) - } - assert.False(t, JobStatusRunning.IsTerminal()) - - // Both spellings of cancelled count as a failure, because the service uses - // one and the other is what half the callers will type. - for _, s := range []JobStatus{JobStatusFailed, JobStatusCancelled, JobStatusCanceled} { - assert.Truef(t, s.IsFailed(), "%s did not produce an artifact", s) - } - for _, s := range []JobStatus{JobStatusCompleted, JobStatusSucceeded, JobStatusRunning} { - assert.Falsef(t, s.IsFailed(), "%s is not a failure", s) - } -} - -// The poller reports why it gave up, and a job that failed with a service -// message has to carry that message rather than only its status. -func TestJobFailedError(t *testing.T) { - bare := &JobFailedError{Status: JobStatusFailed} - assert.Contains(t, bare.Error(), "failed") - - withMessage := &JobFailedError{ - Status: JobStatusFailed, - Job: &GenerationJob{Error: &JobError{Message: "quota exceeded"}}, - } - assert.Contains(t, withMessage.Error(), "quota exceeded", - "the service said why; repeating only the status loses it") -} - -// A transient failure is retried and a terminal one is not, so the difference -// decides whether a caller waits or is told. -func TestIsTransientError(t *testing.T) { - assert.False(t, IsTransientError(nil)) - - for _, code := range []int{http.StatusTooManyRequests, http.StatusInternalServerError, - http.StatusBadGateway, http.StatusServiceUnavailable} { - err := &azcore.ResponseError{StatusCode: code} - assert.Truef(t, IsTransientError(err), "%d is worth retrying", code) - } - for _, code := range []int{http.StatusBadRequest, http.StatusNotFound, http.StatusConflict} { - err := &azcore.ResponseError{StatusCode: code} - assert.Falsef(t, IsTransientError(err), "%d will not change on a retry", code) - } - - for _, msg := range []string{"connection reset by peer", "connection refused", "unexpected EOF"} { - assert.Truef(t, IsTransientError(errors.New(msg)), "%q is a dropped connection", msg) - } - assert.False(t, IsTransientError(errors.New("invalid dataset name"))) -} - -func TestIsNotFoundAndIsConflict(t *testing.T) { - notFound := &azcore.ResponseError{StatusCode: http.StatusNotFound} - conflict := &azcore.ResponseError{StatusCode: http.StatusConflict} - - assert.True(t, IsNotFound(notFound)) - assert.False(t, IsNotFound(conflict)) - assert.False(t, IsNotFound(errors.New("boom"))) - - assert.True(t, IsConflict(conflict)) - assert.False(t, IsConflict(notFound)) - assert.False(t, IsConflict(nil)) -} - -// Wrapped errors have to keep answering, because the client wraps everything it -// returns with context about the call. -func TestIsNotFound_ThroughAWrap(t *testing.T) { - wrapped := errors.Join( - errors.New("reading dataset \"x\""), - &azcore.ResponseError{StatusCode: http.StatusNotFound}, - ) - - require.True(t, IsNotFound(wrapped)) -} diff --git a/cli/azd/extensions/azure.ai.dataset/internal/pkg/gen_api/operations.go b/cli/azd/extensions/azure.ai.dataset/internal/pkg/gen_api/operations.go deleted file mode 100644 index 7598f290a38..00000000000 --- a/cli/azd/extensions/azure.ai.dataset/internal/pkg/gen_api/operations.go +++ /dev/null @@ -1,233 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package gen_api - -import ( - "bytes" - "context" - "encoding/json" - "fmt" - "io" - "log" - "net/http" - "net/url" - - "azureaidataset/internal/version" - - "github.com/Azure/azure-sdk-for-go/sdk/azcore" - "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" - "github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime" - "github.com/Azure/azure-sdk-for-go/sdk/azcore/streaming" - "github.com/azure/azure-dev/cli/azd/pkg/azsdk" -) - -const ( - pathDataGenerationJobs = "/data_generation_jobs" - pathAgents = "/agents" -) - -// Client talks to the evaluation service's data-generation routes. -// -// Datasets are registered through the dataset API, but they are *generated* by -// the evaluation service, so this extension speaks to both. -type Client struct { - endpoint string - pipeline runtime.Pipeline -} - -// NewClient creates a Client for the given project endpoint. -func NewClient(endpoint string, cred azcore.TokenCredential) *Client { - userAgent := fmt.Sprintf("azd-ext-azure-ai-dataset/%s", version.Version) - - clientOptions := &policy.ClientOptions{ - Logging: policy.LogOptions{ - AllowedHeaders: []string{"X-Ms-Correlation-Request-Id", "X-Request-Id"}, - IncludeBody: false, - }, - PerCallPolicies: []policy.Policy{ - runtime.NewBearerTokenPolicy(cred, []string{"https://ai.azure.com/.default"}, nil), - azsdk.NewMsCorrelationPolicy(), - azsdk.NewUserAgentPolicy(userAgent), - }, - } - - pipeline := runtime.NewPipeline( - "azure-ai-dataset", - "v1.0.0", - runtime.PipelineOptions{}, - clientOptions, - ) - - return &Client{endpoint: endpoint, pipeline: pipeline} -} - -// NewClientFromPipeline creates a Client with a pre-built pipeline, for tests -// that need to bypass auth policies. -func NewClientFromPipeline(endpoint string, pipeline runtime.Pipeline) *Client { - return &Client{endpoint: endpoint, pipeline: pipeline} -} - -// CreateDataGenerationJob starts a dataset generation job. -func (c *Client) CreateDataGenerationJob( - ctx context.Context, - request *DataGenerationJobRequest, - apiVersion string, -) (*GenerationJob, error) { - return doRequestTyped[GenerationJob]( - c, ctx, http.MethodPost, pathDataGenerationJobs, request, apiVersion) -} - -// GetDataGenerationJob gets the current state of a dataset generation job. -func (c *Client) GetDataGenerationJob( - ctx context.Context, - operationID string, - apiVersion string, -) (*GenerationJob, error) { - path := pathDataGenerationJobs + "/" + url.PathEscape(operationID) - return doRequestTyped[GenerationJob](c, ctx, http.MethodGet, path, nil, apiVersion) -} - -// ListDataGenerationJobs returns the project's dataset generation jobs. -func (c *Client) ListDataGenerationJobs( - ctx context.Context, - apiVersion string, -) (*GenerationJobList, error) { - return doRequestTyped[GenerationJobList]( - c, ctx, http.MethodGet, pathDataGenerationJobs, nil, apiVersion) -} - -// CancelDataGenerationJob stops a dataset generation job. -// -// The separator is a colon, not a path segment: `{id}/cancel` is a 404 while -// `{id}:cancel` reaches the action. The empty object is what carries a content -// type, without which the route answers 415. -func (c *Client) CancelDataGenerationJob( - ctx context.Context, - operationID string, - apiVersion string, -) (*GenerationJob, error) { - path := pathDataGenerationJobs + "/" + url.PathEscape(operationID) + ":cancel" - return doRequestTyped[GenerationJob]( - c, ctx, http.MethodPost, path, json.RawMessage(`{}`), apiVersion) -} - -// DeleteDataGenerationJob discards the job record. The dataset the job produced -// is already registered and is not affected. -func (c *Client) DeleteDataGenerationJob( - ctx context.Context, - operationID string, - apiVersion string, -) error { - path := pathDataGenerationJobs + "/" + url.PathEscape(operationID) - _, err := c.doRequest(ctx, http.MethodDelete, path, nil, apiVersion) - return err -} - -// GetAgent reads an agent from the project's catalog. -// -// Only the newest version is returned, which is the one generation is seeded -// from: the point is to describe what the agent does now. -func (c *Client) GetAgent( - ctx context.Context, - name string, - apiVersion string, -) (*Agent, error) { - path := pathAgents + "/" + url.PathEscape(name) - return doRequestTyped[Agent](c, ctx, http.MethodGet, path, nil, apiVersion) -} - -func (c *Client) doRequest( - ctx context.Context, - method string, - path string, - body any, - apiVersion string, -) ([]byte, error) { - u, err := url.Parse(c.endpoint) - if err != nil { - return nil, fmt.Errorf("invalid endpoint URL: %w", err) - } - - // Callers escape the ids they interpolate, so the path is set as the raw - // one. Assigning it to u.Path re-escapes the percent signs, and a job id - // carrying a separator then addresses a literally-named resource. - escapedPath := u.EscapedPath() + path - decodedPath, err := url.PathUnescape(escapedPath) - if err != nil { - return nil, fmt.Errorf("invalid request path %q: %w", escapedPath, err) - } - u.Path, u.RawPath = decodedPath, escapedPath - - q := u.Query() - if apiVersion != "" { - q.Set("api-version", apiVersion) - } - u.RawQuery = q.Encode() - - req, err := runtime.NewRequest(ctx, method, u.String()) - if err != nil { - return nil, fmt.Errorf("failed to create request: %w", err) - } - - log.Printf("[gen_api] %s %s", method, u.Redacted()) - - if body != nil { - payload, err := json.Marshal(body) - if err != nil { - return nil, fmt.Errorf("failed to marshal request: %w", err) - } - if err := req.SetBody(streaming.NopCloser(bytes.NewReader(payload)), "application/json"); err != nil { - return nil, fmt.Errorf("failed to set request body: %w", err) - } - } - - resp, err := c.pipeline.Do(req) - if err != nil { - return nil, fmt.Errorf("HTTP request failed: %w", err) - } - defer resp.Body.Close() - - respBody, err := io.ReadAll(resp.Body) - if err != nil { - return nil, fmt.Errorf("failed to read response body: %w", err) - } - - log.Printf("[gen_api] response status: %d", resp.StatusCode) - - // 204 belongs here: a delete that removed the resource answers No Content, - // and treating that as a failure reports every successful delete as an - // error. - if !runtime.HasStatusCode(resp, - http.StatusOK, http.StatusCreated, http.StatusAccepted, http.StatusNoContent) { - resp.Body = io.NopCloser(bytes.NewReader(respBody)) - return nil, runtime.NewResponseError(resp) - } - - return respBody, nil -} - -func doRequestTyped[T any]( - c *Client, - ctx context.Context, - method string, - path string, - body any, - apiVersion string, -) (*T, error) { - respBody, err := c.doRequest(ctx, method, path, body, apiVersion) - if err != nil { - return nil, err - } - - if len(respBody) == 0 { - return new(T), nil - } - - var result T - if err := json.Unmarshal(respBody, &result); err != nil { - return nil, fmt.Errorf("failed to parse response: %w", err) - } - - return &result, nil -} diff --git a/cli/azd/extensions/azure.ai.dataset/internal/pkg/gen_api/operations_test.go b/cli/azd/extensions/azure.ai.dataset/internal/pkg/gen_api/operations_test.go deleted file mode 100644 index e7d02385414..00000000000 --- a/cli/azd/extensions/azure.ai.dataset/internal/pkg/gen_api/operations_test.go +++ /dev/null @@ -1,188 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package gen_api - -import ( - "context" - "io" - "net/http" - "net/http/httptest" - "net/url" - "testing" - - "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" - "github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -// call is what the client actually put on the wire, which is the part of these -// operations that can be wrong without anything failing to compile. -type call struct { - method string - path string - // rawPath is the path as it went over the wire, where escaping is still - // visible. path has been decoded and cannot tell %2F from a separator. - rawPath string - query url.Values - body string -} - -// recorder answers every request with status and body, remembering the last one. -func recorder(t *testing.T, status int, body string) (*Client, *call) { - t.Helper() - var last call - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - raw, _ := io.ReadAll(r.Body) - last = call{ - method: r.Method, - path: r.URL.Path, - rawPath: r.URL.EscapedPath(), - query: r.URL.Query(), - body: string(raw), - } - w.Header().Set("Content-Type", "application/json") - w.WriteHeader(status) - if body != "" { - _, _ = w.Write([]byte(body)) - } - })) - t.Cleanup(server.Close) - - // MaxRetries -1 disables the SDK's retry policy. Without it a test that - // answers 5xx on purpose spends ten seconds being retried. - client := NewClientFromPipeline(server.URL, runtime.NewPipeline( - "test", "v1.0.0", runtime.PipelineOptions{}, - &policy.ClientOptions{Retry: policy.RetryOptions{MaxRetries: -1}})) - return client, &last -} - -// The cancel route takes a colon, not a path segment. `{id}/cancel` is a 404 -// while `{id}:cancel` reaches the action, and nothing but the URL says so. -func TestCancelDataGenerationJob_UsesTheColonForm(t *testing.T) { - client, last := recorder(t, http.StatusOK, `{"id":"dgj_1","status":"cancelled"}`) - - _, err := client.CancelDataGenerationJob(context.Background(), "dgj_1", "v1") - - require.NoError(t, err) - assert.Equal(t, http.MethodPost, last.method) - assert.Equal(t, "/data_generation_jobs/dgj_1:cancel", last.path) - assert.Equal(t, "{}", last.body, - "the empty object is what carries a content type; without it the route answers 415") -} - -// A delete that removed the record answers 204 with no body. Treating that as a -// failure would report every successful delete as an error. -func TestDeleteDataGenerationJob_AcceptsNoContent(t *testing.T) { - client, last := recorder(t, http.StatusNoContent, "") - - require.NoError(t, client.DeleteDataGenerationJob(context.Background(), "dgj_1", "v1")) - - assert.Equal(t, http.MethodDelete, last.method) - assert.Equal(t, "/data_generation_jobs/dgj_1", last.path) -} - -// The job routes answer with `data`, not the `value` the dataset routes use. -// Reading the wrong key returns an empty list from a full response. -func TestListDataGenerationJobs_ReadsTheDataEnvelope(t *testing.T) { - client, last := recorder(t, http.StatusOK, - `{"data":[{"id":"dgj_1","status":"completed"},{"id":"dgj_2","status":"running"}]}`) - - list, err := client.ListDataGenerationJobs(context.Background(), "v1") - - require.NoError(t, err) - assert.Equal(t, "/data_generation_jobs", last.path) - require.Len(t, list.Data, 2) - assert.Equal(t, "dgj_1", list.Data[0].ID) -} - -// The request body is what the service validates, so what reaches the wire is -// pinned rather than left to whatever the builder happened to set. -func TestCreateDataGenerationJob_SendsTheBuiltRequest(t *testing.T) { - client, last := recorder(t, http.StatusOK, `{"id":"dgj_1","status":"running"}`) - - req := NewDataGenerationJobRequest("support-regression", "gpt-4o", 15, - []GenerationSource{{Type: "prompt", Prompt: "be helpful"}}) - _, err := client.CreateDataGenerationJob(context.Background(), req, "v1") - - require.NoError(t, err) - assert.Equal(t, http.MethodPost, last.method) - assert.Equal(t, "/data_generation_jobs", last.path) - assert.Contains(t, last.body, `"name":"support-regression"`) - assert.Contains(t, last.body, `"max_samples":15`) - assert.Contains(t, last.body, `"scenario":"evaluation"`) -} - -// An id goes into the path, so one containing a separator has to be escaped or -// it silently addresses a different route. -// -// The assertion is on the wire form: the decoded path shows the separators -// again, so it cannot tell a correctly escaped id from an unescaped one. -func TestOperations_EscapeIdsInThePath(t *testing.T) { - client, last := recorder(t, http.StatusOK, `{"id":"x"}`) - - _, err := client.GetDataGenerationJob(context.Background(), "dgj/../evil", "v1") - - require.NoError(t, err) - assert.Equal(t, "/data_generation_jobs/dgj%2F..%2Fevil", last.rawPath, - "the id stays one segment; escaping it twice would send %252F and address a differently named job") -} - -func TestOperations_SendTheApiVersion(t *testing.T) { - client, last := recorder(t, http.StatusOK, `{"id":"x"}`) - - _, err := client.GetDataGenerationJob(context.Background(), "dgj_1", "2025-11-15-preview") - - require.NoError(t, err) - assert.Equal(t, "2025-11-15-preview", last.query.Get("api-version")) -} - -// Only the newest agent version seeds generation: the point is to describe what -// the agent does now. -func TestGetAgent_ReadsTheCatalogEntry(t *testing.T) { - client, last := recorder(t, http.StatusOK, - `{"name":"support","versions":{"latest":{"version":"3",`+ - `"definition":{"instructions":"Be helpful."}}}}`) - - agent, err := client.GetAgent(context.Background(), "support", "v1") - - require.NoError(t, err) - assert.Equal(t, "/agents/support", last.path) - assert.Equal(t, "Be helpful.", agent.Instructions()) -} - -// A 404 has to arrive as one, because jobLookupError branches on it to point at -// the evaluator group rather than reporting a transport failure. -func TestOperations_NotFoundIsRecognizable(t *testing.T) { - client, _ := recorder(t, http.StatusNotFound, `{"error":{"code":"NotFound"}}`) - - _, err := client.GetDataGenerationJob(context.Background(), "dgj_missing", "v1") - - require.Error(t, err) - assert.True(t, IsNotFound(err)) -} - -// A server fault is worth retrying and has to be recognizable as such, or the -// poller gives up on a job the service is still working on. -func TestOperations_ServerFaultIsTransient(t *testing.T) { - client, _ := recorder(t, http.StatusBadGateway, `{"error":{"code":"BadGateway"}}`) - - _, err := client.GetDataGenerationJob(context.Background(), "dgj_1", "v1") - - require.Error(t, err) - assert.True(t, IsTransientError(err)) - assert.False(t, IsNotFound(err)) -} - -// An empty body on a success is not a parse failure: a 204 carries none, and -// the typed helper has to hand back a zero value rather than an error. -func TestOperations_EmptyBodyIsNotAnError(t *testing.T) { - client, _ := recorder(t, http.StatusOK, "") - - job, err := client.GetDataGenerationJob(context.Background(), "dgj_1", "v1") - - require.NoError(t, err) - require.NotNil(t, job) - assert.Empty(t, job.ID) -} diff --git a/cli/azd/extensions/azure.ai.dataset/internal/pkg/gen_api/poller.go b/cli/azd/extensions/azure.ai.dataset/internal/pkg/gen_api/poller.go deleted file mode 100644 index 825b980ec2e..00000000000 --- a/cli/azd/extensions/azure.ai.dataset/internal/pkg/gen_api/poller.go +++ /dev/null @@ -1,193 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package gen_api - -import ( - "context" - "fmt" - "log" - "strings" - "time" -) - -// --------------------------------------------------------------------------- -// JobStatus — typed status with terminal/failed semantics -// --------------------------------------------------------------------------- - -// JobStatus represents the normalized status of a generation job. -type JobStatus string - -const ( - JobStatusRunning JobStatus = "running" - JobStatusCompleted JobStatus = "completed" - JobStatusSucceeded JobStatus = "succeeded" - JobStatusFailed JobStatus = "failed" - JobStatusCancelled JobStatus = "cancelled" - JobStatusCanceled JobStatus = "canceled" -) - -// ParseJobStatus normalizes a raw status string into a JobStatus. -// An empty string is treated as "running". -func ParseJobStatus(s string) JobStatus { - if s == "" { - return JobStatusRunning - } - return JobStatus(strings.ToLower(s)) -} - -// IsTerminal returns true when the status represents a final state. -func (s JobStatus) IsTerminal() bool { - switch s { - case JobStatusCompleted, JobStatusSucceeded, JobStatusFailed, JobStatusCancelled, JobStatusCanceled: - return true - } - return false -} - -// IsFailed returns true when the status represents a failure or cancellation. -func (s JobStatus) IsFailed() bool { - switch s { - case JobStatusFailed, JobStatusCancelled, JobStatusCanceled: - return true - } - return false -} - -// String returns the status as a plain string. -func (s JobStatus) String() string { - return string(s) -} - -// --------------------------------------------------------------------------- -// JobFailedError — returned when a polled job reaches a failed state -// --------------------------------------------------------------------------- - -// JobFailedError is returned when a generation job reaches a failed terminal state. -type JobFailedError struct { - Job *GenerationJob - Status JobStatus -} - -func (e *JobFailedError) Error() string { - if e.Job != nil && e.Job.Error != nil && e.Job.Error.Message != "" { - return fmt.Sprintf("job failed with status %q: %s", e.Status, e.Job.Error.Message) - } - return fmt.Sprintf("job failed with status %q", e.Status) -} - -// --------------------------------------------------------------------------- -// PollerTimeoutError — returned when polling exhausts all attempts -// --------------------------------------------------------------------------- - -// PollerTimeoutError is returned when a generation job has not reached a -// terminal state within the configured number of polling attempts. -type PollerTimeoutError struct { - OperationID string - Attempts int -} - -func (e *PollerTimeoutError) Error() string { - return fmt.Sprintf( - "operation %s did not complete within %d attempts", - e.OperationID, e.Attempts, - ) -} - -// --------------------------------------------------------------------------- -// GetJobFunc — callback type for fetching job state -// --------------------------------------------------------------------------- - -// GetJobFunc fetches the current state of a generation job by operation ID. -type GetJobFunc func(ctx context.Context, operationID, apiVersion string) (*GenerationJob, error) - -// --------------------------------------------------------------------------- -// PollerOptions — configurable polling behavior -// --------------------------------------------------------------------------- - -// PollerOptions configures the polling interval and attempt limit. -type PollerOptions struct { - Interval time.Duration - MaxAttempts int -} - -// DefaultPollerOptions returns sensible defaults: 2 s interval, 300 attempts (~10 min). -func DefaultPollerOptions() PollerOptions { - return PollerOptions{ - Interval: 2 * time.Second, - MaxAttempts: 300, - } -} - -// --------------------------------------------------------------------------- -// Poller — polls a generation job until it reaches a terminal state -// --------------------------------------------------------------------------- - -// Poller polls a GenerationJob until it reaches a terminal status. -type Poller struct { - OperationID string - APIVersion string - GetJob GetJobFunc - Options PollerOptions - // OnPoll is called after each successful poll with the latest status. - // Callers can use this for progress reporting (e.g. debug logging). - OnPoll func(status JobStatus) -} - -// NewPoller creates a Poller with default options. -func NewPoller(operationID, apiVersion string, getJob GetJobFunc) *Poller { - return &Poller{ - OperationID: operationID, - APIVersion: apiVersion, - GetJob: getJob, - Options: DefaultPollerOptions(), - } -} - -// Poll blocks until the job reaches a terminal state, the context is -// cancelled, or the maximum number of attempts is exhausted. -// -// On success it returns the completed GenerationJob. -// On failure it returns a *JobFailedError (which wraps the job for inspection). -// On timeout it returns a plain error. -func (p *Poller) Poll(ctx context.Context) (*GenerationJob, error) { - if p.OperationID == "" { - return nil, fmt.Errorf("operation ID is empty") - } - - for range p.Options.MaxAttempts { - select { - case <-ctx.Done(): - return nil, ctx.Err() - case <-time.After(p.Options.Interval): - } - - job, err := p.GetJob(ctx, p.OperationID, p.APIVersion) - if err != nil { - if IsTransientError(err) { - log.Printf("[poller] transient error polling %s, will retry: %v", p.OperationID, err) - continue - } - return nil, err - } - - status := ParseJobStatus(job.Status) - log.Printf("[poller] operationID=%s status=%s", p.OperationID, status) - - if p.OnPoll != nil { - p.OnPoll(status) - } - - if status.IsTerminal() { - if status.IsFailed() { - return nil, &JobFailedError{Job: job, Status: status} - } - return job, nil - } - } - - return nil, &PollerTimeoutError{ - OperationID: p.OperationID, - Attempts: p.Options.MaxAttempts, - } -} From bb07f764dda44dc8ef6cfe76c93dcfeb021f997d Mon Sep 17 00:00:00 2001 From: mohessie Date: Thu, 6 Aug 2026 19:24:16 +0300 Subject: [PATCH 134/320] Say what went into the catalog, not that something did The spec's Scenario 2 transcript reads (v) Done: Added dataset 'support-agent-regression' (version 1) to evals/eval.yaml and the command printed "Added catalog entry to evals/eval.yaml" -- neither the name nor the version. The line above it names the file, so a reader could infer the name, but the version is what pins the entry and it appeared nowhere. Both generation jobs already resolve a version on the way past; it just was not carried. ArtifactRef now holds it, and the catalog line reports it. A version of "latest" is the service declining to say which, so it prints as no version rather than as one. --- .../internal/cmd/catalog.go | 25 +++- .../internal/cmd/catalog_test.go | 111 ++++++++++++++++++ .../internal/cmd/generate.go | 13 +- .../internal/project/artifacts.go | 3 + 4 files changed, 145 insertions(+), 7 deletions(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/catalog_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/catalog.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/catalog.go index 2e34a944b12..815c84b23c2 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/catalog.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/catalog.go @@ -22,7 +22,7 @@ func addDatasetToCatalog(cmd *cobra.Command, evalDir string, ref *project.Artifa if ref == nil { return nil } - return updateCatalog(cmd, evalDir, func(cfg *project.EvalConfig) bool { + return updateCatalog(cmd, evalDir, "dataset", ref, func(cfg *project.EvalConfig) bool { for i := range cfg.Datasets { if cfg.Datasets[i].Name == ref.Name { // Regeneration overwrites the file in place, so the entry only @@ -47,7 +47,7 @@ func addEvaluatorToCatalog(cmd *cobra.Command, evalDir string, ref *project.Arti if ref == nil { return nil } - return updateCatalog(cmd, evalDir, func(cfg *project.EvalConfig) bool { + return updateCatalog(cmd, evalDir, "evaluator", ref, func(cfg *project.EvalConfig) bool { for i := range cfg.Evaluators { if cfg.Evaluators[i].Name == ref.Name { if cfg.Evaluators[i].Source == ref.Source { @@ -74,6 +74,8 @@ func addEvaluatorToCatalog(cmd *cobra.Command, evalDir string, ref *project.Arti func updateCatalog( cmd *cobra.Command, evalDir string, + kind string, + ref *project.ArtifactRef, apply func(*project.EvalConfig) bool, ) error { cfg, err := project.OpenEvalConfig(evalDir) @@ -94,10 +96,23 @@ func updateCatalog( if !isJSON(cmd) { path := filepath.ToSlash(project.EvalConfigPath(evalDir)) if created { - fmt.Fprintf(cmd.OutOrStdout(), "(✓) Done: Created %s with the catalog entry\n", path) - } else { - fmt.Fprintf(cmd.OutOrStdout(), "(✓) Done: Added catalog entry to %s\n", path) + fmt.Fprintf(cmd.OutOrStdout(), + "%s Created %s with the catalog entry\n", doneMark, path) } + fmt.Fprintf(cmd.OutOrStdout(), + "%s Added %s %s to %s\n", doneMark, kind, describeArtifact(ref), path) } return nil } + +// describeArtifact names what was recorded, with the published version when the +// job reported one, so a reader can pin it without going to look. +// +// Single-quoted to match the spec's transcripts; the surrounding Done: lines +// carry bare values, but a dataset name can hold a space and these cannot. +func describeArtifact(ref *project.ArtifactRef) string { + if ref.Version == "" || ref.Version == "latest" { + return fmt.Sprintf("'%s'", ref.Name) + } + return fmt.Sprintf("'%s' (version %s)", ref.Name, ref.Version) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/catalog_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/catalog_test.go new file mode 100644 index 00000000000..126c36a35ba --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/catalog_test.go @@ -0,0 +1,111 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "bytes" + "os" + "path/filepath" + "testing" + + "azureaieval/internal/project" + + "github.com/spf13/cobra" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// catalogCommand builds a command whose output can be read back. +func catalogCommand(t *testing.T, buf *bytes.Buffer) *cobra.Command { + t.Helper() + cmd := &cobra.Command{Use: "generate"} + cmd.Flags().StringP("output", "o", "", "") + cmd.SetOut(buf) + return cmd +} + +// The spec's Scenario 2 transcript names what was recorded and the version it +// was published at. "Added catalog entry" says neither, which leaves the reader +// to infer both from the line above it. +func TestCatalogLineNamesTheArtifactAndVersion(t *testing.T) { + dir := t.TempDir() + var buf bytes.Buffer + + require.NoError(t, addDatasetToCatalog(catalogCommand(t, &buf), dir, &project.ArtifactRef{ + Name: "support-agent-regression", + Source: "datasets/support-agent-regression.jsonl", + Version: "1", + })) + + out := buf.String() + assert.Contains(t, out, `Added dataset 'support-agent-regression' (version 1) to`) + assert.Contains(t, out, "eval.yaml") +} + +// The evaluator half of the same transcript. +func TestCatalogLineForAnEvaluator(t *testing.T) { + dir := t.TempDir() + var buf bytes.Buffer + + require.NoError(t, addEvaluatorToCatalog(catalogCommand(t, &buf), dir, &project.ArtifactRef{ + Name: "support-agent-quality", + Source: "evaluators/support-agent-quality.json", + Version: "1", + })) + + assert.Contains(t, buf.String(), `Added evaluator 'support-agent-quality' (version 1) to`) +} + +// A job that reported no version still has to name what it recorded, rather +// than printing an empty parenthesis or the word "latest" as if it were one. +func TestCatalogLineWithoutAVersion(t *testing.T) { + for _, version := range []string{"", "latest"} { + dir := t.TempDir() + var buf bytes.Buffer + + require.NoError(t, addDatasetToCatalog(catalogCommand(t, &buf), dir, &project.ArtifactRef{ + Name: "golden", Source: "datasets/golden.jsonl", Version: version, + })) + + out := buf.String() + assert.Contains(t, out, `Added dataset 'golden' to`) + assert.NotContains(t, out, "version", "version %q is not one to print", version) + } +} + +// The first generate in a repository has no eval.yaml to append to, so it says +// the file was created as well as what went into it. +func TestCatalogLineWhenTheFileIsCreated(t *testing.T) { + dir := t.TempDir() + var buf bytes.Buffer + + require.NoError(t, addDatasetToCatalog(catalogCommand(t, &buf), dir, &project.ArtifactRef{ + Name: "golden", Source: "datasets/golden.jsonl", Version: "1", + })) + + out := buf.String() + assert.Contains(t, out, "Created") + assert.Contains(t, out, `Added dataset 'golden' (version 1) to`, + "creating the file still has to say what was put in it") + + // The entry is really on disk, not just announced. + body, err := os.ReadFile(filepath.Join(dir, "eval.yaml")) + require.NoError(t, err) + assert.Contains(t, string(body), "golden") +} + +// Regenerating the same artifact to the same path changes nothing, so it must +// not claim it did. +func TestCatalogSaysNothingWhenNothingChanged(t *testing.T) { + dir := t.TempDir() + ref := &project.ArtifactRef{Name: "golden", Source: "datasets/golden.jsonl", Version: "1"} + + var first bytes.Buffer + require.NoError(t, addDatasetToCatalog(catalogCommand(t, &first), dir, ref)) + require.Contains(t, first.String(), "Added dataset") + + var second bytes.Buffer + require.NoError(t, addDatasetToCatalog(catalogCommand(t, &second), dir, ref)) + assert.Empty(t, second.String(), "an unchanged catalog is not an edit") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go index d88cc539cc1..010abc3ca5f 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go @@ -222,7 +222,12 @@ func (ec *evalContext) generateRubric( } fmt.Fprintf(out, " wrote %s\n", path) - return &project.ArtifactRef{Name: plan.Name, Source: relativeSource(plan.BaseDir, path)}, nil + _, version := completed.ResolvedNameVersion() + return &project.ArtifactRef{ + Name: plan.Name, + Source: relativeSource(plan.BaseDir, path), + Version: version, + }, nil } // refuseUnbuildableSources reports a --from the plan could not honour. @@ -342,7 +347,11 @@ func (ec *evalContext) generateDataset( } fmt.Fprintf(out, " wrote %s\n", path) - return &project.ArtifactRef{Name: plan.Name, Source: relativeSource(plan.BaseDir, path)}, nil + return &project.ArtifactRef{ + Name: plan.Name, + Source: relativeSource(plan.BaseDir, path), + Version: version, + }, nil } // isAgentSeededGenerationFailure recognizes the service-side failure that hits diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/artifacts.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/artifacts.go index 9f5a6320684..a5e1462e4a6 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/artifacts.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/artifacts.go @@ -20,6 +20,9 @@ const ( type ArtifactRef struct { Name string `json:"name"` Source string `json:"source"` + // Version is what the generation job published, which the catalog line + // names so a reader can pin it without going to look. + Version string `json:"version,omitempty"` } // Sample-count bounds enforced by the generation service. From 08725da026690f34730d78c82e42fa81c5e28f70 Mon Sep 17 00:00:00 2001 From: mohessie Date: Thu, 6 Aug 2026 21:56:37 +0300 Subject: [PATCH 135/320] Let run find a configuration that init did not put in ./evals --path is a documented init flag, and using it produced a project whose next step -- the one init itself prints -- could not work: azd ai eval init --path ./quality Created quality/eval.yaml, azure.yaml service with $ref: ./quality/eval.yaml Next: azd up / azd ai eval run start azd ai eval run start ERROR: no eval was named and none is declared in evals\eval.yaml run hardcoded the default directory and had no --path at all, so the only layout it could reach was the one it assumed. azure.yaml pointed at the real file the whole time; only the deploy path reads that. run and run output now resolve the directory through --path, then the path init recorded in the azd environment, then ./evals. The middle level is what stops --path from having to be repeated on every later command: init writes EVAL_CONFIG_PATH once and the rest of the session follows it. The flag defaults to empty rather than to ./evals, because a non-empty default is indistinguishable from an explicit value and would shadow the recorded path. Recording is best effort: init works outside an azd environment, where there is nowhere to write. That case still needs --path, which is now available. --- .../internal/cmd/context.go | 23 ++++++++ .../internal/cmd/evaldir_test.go | 58 +++++++++++++++++++ .../azure.ai.evaluations/internal/cmd/init.go | 29 ++++++++++ .../azure.ai.evaluations/internal/cmd/run.go | 4 +- .../internal/cmd/run_output.go | 13 ++++- 5 files changed, 124 insertions(+), 3 deletions(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaldir_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/context.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/context.go index 21eb8b0bee8..75a6647f2d8 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/context.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/context.go @@ -216,4 +216,27 @@ const ( envKeyEvalRunID = "EVAL_RUN_ID" envKeyDatasetVersion = "EVAL_DATASET_VERSION" envKeyFingerprintPrefix = "EVAL_FINGERPRINT_" + // envKeyEvalPath records where `init` put the configuration, so the + // commands that read it afterwards do not each need --path repeated. + envKeyEvalPath = "EVAL_CONFIG_PATH" ) + +// evalDir resolves where the configuration lives: +// +// 1. --path +// 2. the path `init` recorded in the azd environment +// 3. ./evals +// +// The middle level is what stops `--path` from having to be repeated on every +// later command. Without it, `init --path ./quality` wrote a configuration that +// `run` then looked for under ./evals and reported as missing -- while +// azure.yaml's $ref pointed at it correctly the whole time. +func (ec *evalContext) evalDir(ctx context.Context, flagValue string) string { + if flagValue != "" { + return flagValue + } + if recorded := ec.getEnvValue(ctx, envKeyEvalPath); recorded != "" { + return recorded + } + return project.DefaultEvalDir +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaldir_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaldir_test.go new file mode 100644 index 00000000000..aaba43e160a --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaldir_test.go @@ -0,0 +1,58 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "context" + "testing" + + "azureaieval/internal/project" + + "github.com/spf13/cobra" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// `init --path ./quality` wrote a configuration that `run` then looked for +// under ./evals and reported as missing, while azure.yaml's $ref pointed at it +// correctly the whole time. The path init used is remembered so the flag does +// not have to be repeated on every later command. +func TestEvalDirCascade(t *testing.T) { + // No azd environment: getEnvValue returns empty, so only flag and default apply. + ec := &evalContext{} + + assert.Equal(t, project.DefaultEvalDir, ec.evalDir(context.Background(), ""), + "nothing given anywhere is ./evals") + assert.Equal(t, "quality", ec.evalDir(context.Background(), "quality"), + "--path wins") +} + +// Every command that reads the configuration has to be able to say where it is, +// or a project scaffolded with --path is unreachable from that command. +func TestCommandsReadingTheConfigTakePath(t *testing.T) { + for _, path := range []string{"run start", "init", "dataset generate", "evaluator generate"} { + cmd := find(t, path) + assert.NotNilf(t, cmd.Flags().Lookup("path"), + "%s reads the configuration, so it must accept --path", path) + } +} + +// --path defaults to empty, not to ./evals, so "not given" stays +// distinguishable from "given the default". Defaulting it to ./evals would +// shadow the path init recorded and reintroduce the bug. +func TestRunPathFlagDefaultsToEmpty(t *testing.T) { + flag := find(t, "run start").Flags().Lookup("path") + require.NotNil(t, flag) + assert.Empty(t, flag.DefValue, + "a non-empty default would always win over the recorded path") +} + +// The recorded key is what `init` writes and what the other commands read; a +// rename on one side alone silently stops the hand-off working. +func TestEvalPathEnvKey(t *testing.T) { + assert.Equal(t, "EVAL_CONFIG_PATH", envKeyEvalPath) +} + +// find is shared with surface_test.go; this keeps the compiler honest about it. +var _ = func(t *testing.T) *cobra.Command { return find(t, "run start") } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go index 306232e24e3..ea7e19e5a5c 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go @@ -144,6 +144,8 @@ func newInitCommand() *cobra.Command { return err } + recordEvalPath(cmd.Context(), path) + if isJSON(cmd) { return emitJSON(out, map[string]any{ "eval": evalName, @@ -488,6 +490,33 @@ func detectModelDeployment(proj *azdext.ProjectConfig) string { return "" } +// recordEvalPath remembers where the configuration was written, so the commands +// that read it afterwards do not need --path repeated. +// +// Best effort: `init` works outside an azd environment, and a path that could +// not be recorded only costs the caller a flag later. It is never a reason to +// fail a scaffold that already succeeded. +func recordEvalPath(ctx context.Context, path string) { + if path == "" || path == project.DefaultEvalDir { + return + } + azdClient, err := azdext.NewAzdClient() + if err != nil { + return + } + defer azdClient.Close() + + env, err := azdClient.Environment().GetCurrent(ctx, &azdext.EmptyRequest{}) + if err != nil || env.GetEnvironment() == nil { + return + } + _, _ = azdClient.Environment().SetValue(ctx, &azdext.SetEnvRequest{ + EnvName: env.GetEnvironment().GetName(), + Key: envKeyEvalPath, + Value: filepath.ToSlash(path), + }) +} + // ensureRootEvalService declares the eval service in azd's project file. // // azd acts on nothing until the service exists, so the reference is made rather diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go index efd5ae895ab..9c10fc7546f 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go @@ -78,6 +78,7 @@ func buildRunCommand(use, short string) *cobra.Command { wait bool failOn string endpointFlg string + evalPath string ) cmd := &cobra.Command{ @@ -103,7 +104,7 @@ func buildRunCommand(use, short string) *cobra.Command { // One flag takes a name or an id. A declared name also brings the // declaration, which is what says where rows come from; a bare id // has none, so the pairing comes from the eval's previous run. - ref, err := ec.resolveEvalRef(ctx, project.DefaultEvalDir, groupName) + ref, err := ec.resolveEvalRef(ctx, ec.evalDir(ctx, evalPath), groupName) if err != nil { return err } @@ -236,6 +237,7 @@ func buildRunCommand(use, short string) *cobra.Command { } cmd.MarkFlagsMutuallyExclusive("wait", "no-wait") cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") + addEvalPathFlag(cmd, &evalPath) return cmd } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output.go index 28f2170909b..f31b868a099 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output.go @@ -13,7 +13,6 @@ import ( "strings" "azureaieval/internal/pkg/eval_api" - "azureaieval/internal/project" "github.com/spf13/cobra" ) @@ -250,7 +249,7 @@ func resolveEvalID( } if groupName != "" { - ref, err := ec.resolveEvalRef(cmd.Context(), project.DefaultEvalDir, groupName) + ref, err := ec.resolveEvalRef(cmd.Context(), ec.evalDir(cmd.Context(), ""), groupName) if err != nil { return "", err } @@ -272,6 +271,16 @@ func addEvalFlag(cmd *cobra.Command, target *string) { "Name of the eval declared in the configuration, or its id.") } +// addEvalPathFlag registers --path on a command that reads the configuration. +// +// It defaults to empty rather than to ./evals so that "not given" stays +// distinguishable from "given the default", which is what lets the path `init` +// recorded take effect in between. +func addEvalPathFlag(cmd *cobra.Command, target *string) { + cmd.Flags().StringVar(target, "path", "", + "Directory holding eval.yaml. Defaults to the path `init` used, then ./evals.") +} + // latestOrNamedRun returns the named run, or the most recent one for the eval. // // explicit says whether the caller named the run rather than leaving it to From 160e75b884d2169ef38f43cb2940936503bb166e Mon Sep 17 00:00:00 2001 From: mohessie Date: Thu, 6 Aug 2026 23:18:48 +0300 Subject: [PATCH 136/320] Write down the two files that decide whether this ships Neither extension is in registry.json, so `azd extension install` cannot resolve either of them -- today they are reachable only through `azd x pack` and `azd x publish` into a local source registry. Neither is in CODEOWNERS either, though every sibling Foundry extension is, so PRs here route to nobody. Both files belong to the azd extensions team, so this records the work rather than doing it. The ordering matters and is easy to get wrong: the microsoft.foundry dependency must come after the registry entry, because a declared dependency that cannot resolve breaks installing the bundle. --- cli/azd/extensions/azure.ai.dataset/README.md | 15 +++++++++++++++ cli/azd/extensions/azure.ai.evaluations/README.md | 15 +++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/cli/azd/extensions/azure.ai.dataset/README.md b/cli/azd/extensions/azure.ai.dataset/README.md index 544310c1047..e626e40b569 100644 --- a/cli/azd/extensions/azure.ai.dataset/README.md +++ b/cli/azd/extensions/azure.ai.dataset/README.md @@ -46,3 +46,18 @@ Every command resolves the Foundry project endpoint in this order: $ go build ./... $ go test ./... ``` + +## TODO before release + +Both are files the azd extensions team owns, so they are not changed here: + +- [ ] **`cli/azd/extensions/registry.json`** — add the `azure.ai.dataset` entry. + Until it exists `azd extension install azure.ai.dataset` cannot resolve, so + the extension is only reachable through `azd x pack` + `azd x publish` into + the local source registry. +- [ ] **`.github/CODEOWNERS`** — add `/cli/azd/extensions/azure.ai.dataset/`. + Every sibling Foundry extension has an entry; without one, PRs here get no + reviewer routing. +- [ ] **`microsoft.foundry/extension.yaml`** — add the dependency, but only + after the registry entry lands. Declaring a dependency that cannot resolve + breaks installing the bundle. diff --git a/cli/azd/extensions/azure.ai.evaluations/README.md b/cli/azd/extensions/azure.ai.evaluations/README.md index 2a5c80e62fe..86398f2ef46 100644 --- a/cli/azd/extensions/azure.ai.evaluations/README.md +++ b/cli/azd/extensions/azure.ai.evaluations/README.md @@ -156,3 +156,18 @@ They clean up every resource they create. Request tracing is off by default. `--debug`, or `AZD_EXT_DEBUG=true`, writes it to a dated log file rather than the terminal. + +## TODO before release + +Both are files the azd extensions team owns, so they are not changed here: + +- [ ] **`cli/azd/extensions/registry.json`** — add the `azure.ai.evaluations` + entry. Until it exists `azd extension install azure.ai.evaluations` cannot + resolve, so the extension is only reachable through `azd x pack` + + `azd x publish` into the local source registry. +- [ ] **`.github/CODEOWNERS`** — add `/cli/azd/extensions/azure.ai.evaluations/`. + Every sibling Foundry extension has an entry; without one, PRs here get no + reviewer routing. +- [ ] **`microsoft.foundry/extension.yaml`** — add the dependency, but only + after the registry entry lands. Declaring a dependency that cannot resolve + breaks installing the bundle. From 553542d9afaef9fb4b8414784e444c6d9dd1336a Mon Sep 17 00:00:00 2001 From: mohessie Date: Sun, 9 Aug 2026 19:18:56 +0300 Subject: [PATCH 137/320] Keep the eval branch to the eval extension The dataset extension ships as its own extension from its own branch, off main, with no dependency on this one. It was only ever here because it was carved out of this code; carrying it in the eval PR would make one review of two independent things. --- .../workflows/lint-ext-azure-ai-dataset.yml | 22 - .../extensions/azure.ai.dataset/.gitignore | 5 - .../azure.ai.dataset/.golangci.yaml | 21 - .../extensions/azure.ai.dataset/CHANGELOG.md | 11 - cli/azd/extensions/azure.ai.dataset/README.md | 63 -- cli/azd/extensions/azure.ai.dataset/build.ps1 | 78 --- cli/azd/extensions/azure.ai.dataset/build.sh | 66 -- .../extensions/azure.ai.dataset/ci-build.ps1 | 114 ---- .../extensions/azure.ai.dataset/ci-test.ps1 | 56 -- .../extensions/azure.ai.dataset/cspell.yaml | 27 - .../azure.ai.dataset/extension.yaml | 24 - cli/azd/extensions/azure.ai.dataset/go.mod | 105 --- cli/azd/extensions/azure.ai.dataset/go.sum | 316 --------- .../internal/cmd/apiversions.go | 13 - .../internal/cmd/artifacts.go | 23 - .../azure.ai.dataset/internal/cmd/context.go | 140 ---- .../azure.ai.dataset/internal/cmd/dataset.go | 321 --------- .../azure.ai.dataset/internal/cmd/debug.go | 65 -- .../internal/cmd/helpers_test.go | 26 - .../azure.ai.dataset/internal/cmd/output.go | 121 ---- .../internal/cmd/output_test.go | 177 ----- .../azure.ai.dataset/internal/cmd/root.go | 62 -- .../internal/cmd/surface_test.go | 231 ------- .../internal/cmd/table_test.go | 70 -- .../internal/exterrors/codes.go | 71 -- .../internal/exterrors/errors.go | 155 ----- .../internal/foundry/projectctx/resolver.go | 152 ----- .../foundry/projectctx/resolver_test.go | 259 ------- .../internal/foundry/projectctx/store.go | 40 -- .../internal/foundry/projectctx/types.go | 81 --- .../internal/foundry/projectctx/validator.go | 113 ---- .../internal/pkg/dataset_api/download_test.go | 47 -- .../pkg/dataset_api/download_wire_test.go | 255 ------- .../internal/pkg/dataset_api/list.go | 103 --- .../internal/pkg/dataset_api/models.go | 210 ------ .../internal/pkg/dataset_api/operations.go | 640 ------------------ .../pkg/dataset_api/operations_wire_test.go | 205 ------ .../pkg/dataset_api/upload_version_test.go | 157 ----- .../internal/pkg/dataset_api/uri_test.go | 75 -- .../internal/pkg/dataset_api/version_test.go | 68 -- .../internal/version/version.go | 11 - cli/azd/extensions/azure.ai.dataset/main.go | 14 - .../extensions/azure.ai.dataset/version.txt | 1 - .../release-ext-azure-ai-dataset.yml | 45 -- 44 files changed, 4859 deletions(-) delete mode 100644 .github/workflows/lint-ext-azure-ai-dataset.yml delete mode 100644 cli/azd/extensions/azure.ai.dataset/.gitignore delete mode 100644 cli/azd/extensions/azure.ai.dataset/.golangci.yaml delete mode 100644 cli/azd/extensions/azure.ai.dataset/CHANGELOG.md delete mode 100644 cli/azd/extensions/azure.ai.dataset/README.md delete mode 100644 cli/azd/extensions/azure.ai.dataset/build.ps1 delete mode 100644 cli/azd/extensions/azure.ai.dataset/build.sh delete mode 100644 cli/azd/extensions/azure.ai.dataset/ci-build.ps1 delete mode 100644 cli/azd/extensions/azure.ai.dataset/ci-test.ps1 delete mode 100644 cli/azd/extensions/azure.ai.dataset/cspell.yaml delete mode 100644 cli/azd/extensions/azure.ai.dataset/extension.yaml delete mode 100644 cli/azd/extensions/azure.ai.dataset/go.mod delete mode 100644 cli/azd/extensions/azure.ai.dataset/go.sum delete mode 100644 cli/azd/extensions/azure.ai.dataset/internal/cmd/apiversions.go delete mode 100644 cli/azd/extensions/azure.ai.dataset/internal/cmd/artifacts.go delete mode 100644 cli/azd/extensions/azure.ai.dataset/internal/cmd/context.go delete mode 100644 cli/azd/extensions/azure.ai.dataset/internal/cmd/dataset.go delete mode 100644 cli/azd/extensions/azure.ai.dataset/internal/cmd/debug.go delete mode 100644 cli/azd/extensions/azure.ai.dataset/internal/cmd/helpers_test.go delete mode 100644 cli/azd/extensions/azure.ai.dataset/internal/cmd/output.go delete mode 100644 cli/azd/extensions/azure.ai.dataset/internal/cmd/output_test.go delete mode 100644 cli/azd/extensions/azure.ai.dataset/internal/cmd/root.go delete mode 100644 cli/azd/extensions/azure.ai.dataset/internal/cmd/surface_test.go delete mode 100644 cli/azd/extensions/azure.ai.dataset/internal/cmd/table_test.go delete mode 100644 cli/azd/extensions/azure.ai.dataset/internal/exterrors/codes.go delete mode 100644 cli/azd/extensions/azure.ai.dataset/internal/exterrors/errors.go delete mode 100644 cli/azd/extensions/azure.ai.dataset/internal/foundry/projectctx/resolver.go delete mode 100644 cli/azd/extensions/azure.ai.dataset/internal/foundry/projectctx/resolver_test.go delete mode 100644 cli/azd/extensions/azure.ai.dataset/internal/foundry/projectctx/store.go delete mode 100644 cli/azd/extensions/azure.ai.dataset/internal/foundry/projectctx/types.go delete mode 100644 cli/azd/extensions/azure.ai.dataset/internal/foundry/projectctx/validator.go delete mode 100644 cli/azd/extensions/azure.ai.dataset/internal/pkg/dataset_api/download_test.go delete mode 100644 cli/azd/extensions/azure.ai.dataset/internal/pkg/dataset_api/download_wire_test.go delete mode 100644 cli/azd/extensions/azure.ai.dataset/internal/pkg/dataset_api/list.go delete mode 100644 cli/azd/extensions/azure.ai.dataset/internal/pkg/dataset_api/models.go delete mode 100644 cli/azd/extensions/azure.ai.dataset/internal/pkg/dataset_api/operations.go delete mode 100644 cli/azd/extensions/azure.ai.dataset/internal/pkg/dataset_api/operations_wire_test.go delete mode 100644 cli/azd/extensions/azure.ai.dataset/internal/pkg/dataset_api/upload_version_test.go delete mode 100644 cli/azd/extensions/azure.ai.dataset/internal/pkg/dataset_api/uri_test.go delete mode 100644 cli/azd/extensions/azure.ai.dataset/internal/pkg/dataset_api/version_test.go delete mode 100644 cli/azd/extensions/azure.ai.dataset/internal/version/version.go delete mode 100644 cli/azd/extensions/azure.ai.dataset/main.go delete mode 100644 cli/azd/extensions/azure.ai.dataset/version.txt delete mode 100644 eng/pipelines/release-ext-azure-ai-dataset.yml diff --git a/.github/workflows/lint-ext-azure-ai-dataset.yml b/.github/workflows/lint-ext-azure-ai-dataset.yml deleted file mode 100644 index b0b4cfd13bd..00000000000 --- a/.github/workflows/lint-ext-azure-ai-dataset.yml +++ /dev/null @@ -1,22 +0,0 @@ -name: ext-azure-ai-dataset-ci - -on: - pull_request: - paths: - - "cli/azd/extensions/azure.ai.dataset/**" - - ".github/workflows/lint-ext-azure-ai-dataset.yml" - branches: [main] - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number }} - cancel-in-progress: true - -permissions: - contents: read - pull-requests: write # required by reusable workflow lint-go.yml - -jobs: - lint: - uses: ./.github/workflows/lint-go.yml - with: - working-directory: cli/azd/extensions/azure.ai.dataset diff --git a/cli/azd/extensions/azure.ai.dataset/.gitignore b/cli/azd/extensions/azure.ai.dataset/.gitignore deleted file mode 100644 index 0d5b6d76489..00000000000 --- a/cli/azd/extensions/azure.ai.dataset/.gitignore +++ /dev/null @@ -1,5 +0,0 @@ -# Test report written by ci-test.ps1 for the pipeline to publish. -junitTestReport.xml - -# Debug log written when --debug or AZD_EXT_DEBUG is set. -azd-ai-eval-*.log diff --git a/cli/azd/extensions/azure.ai.dataset/.golangci.yaml b/cli/azd/extensions/azure.ai.dataset/.golangci.yaml deleted file mode 100644 index 9777522d023..00000000000 --- a/cli/azd/extensions/azure.ai.dataset/.golangci.yaml +++ /dev/null @@ -1,21 +0,0 @@ -version: "2" - -linters: - default: none - enable: - - gosec - - lll - - unused - - errorlint - settings: - lll: - line-length: 220 - tab-width: 4 - gosec: - excludes: - - G204 # Subprocess launched with variable (bicep build invoked in tests) - - G304 # Potential file inclusion via variable - -formatters: - enable: - - gofmt diff --git a/cli/azd/extensions/azure.ai.dataset/CHANGELOG.md b/cli/azd/extensions/azure.ai.dataset/CHANGELOG.md deleted file mode 100644 index 17c3d3ee366..00000000000 --- a/cli/azd/extensions/azure.ai.dataset/CHANGELOG.md +++ /dev/null @@ -1,11 +0,0 @@ -# Release History - -## 1.0.0-beta.1 (Unreleased) - -### Features Added - -- Initial release. `create`, `update`, `list`, `show`, `delete`, and - `versions list`. -- The CRUD groups moved here from `azure.ai.evaluations`. Generation stayed - there: `dataset generate` writes the `datasets:` entry in `evals/eval.yaml`, - and that file belongs to the evaluation extension. diff --git a/cli/azd/extensions/azure.ai.dataset/README.md b/cli/azd/extensions/azure.ai.dataset/README.md deleted file mode 100644 index e626e40b569..00000000000 --- a/cli/azd/extensions/azure.ai.dataset/README.md +++ /dev/null @@ -1,63 +0,0 @@ -# Foundry datasets (Beta) - -Register and version Foundry datasets from your terminal. - -```console -$ azd extension install azure.ai.dataset -$ azd ai dataset --help -``` - -A dataset is a general Foundry asset: evaluation needs one, and so do -fine-tuning and other scenarios. That is why these commands live here rather -than inside `azure.ai.evaluations`. - -## Commands - -| Command | What it does | -|---|---| -| `azd ai dataset create --from-file ` | Register a dataset, publishing its first version | -| `azd ai dataset update --from-file ` | Publish a further version | -| `azd ai dataset list` | List the project's datasets | -| `azd ai dataset show ` | Show one dataset | -| `azd ai dataset delete ` | Delete a dataset version | -| `azd ai dataset versions list ` | List a dataset's versions | - -## Generating a dataset - -Generation is `azd ai eval dataset generate`, in `azure.ai.evaluations`, and -stays there: it writes the `datasets:` entry in `evals/eval.yaml`, which is that -extension's file. Splitting the two would leave a generated dataset registered -with the service but absent from the configuration, so `azd up` would not -reconcile it and no eval could name it. - -Once a file exists, `create` registers it here. - -## Project endpoint - -Every command resolves the Foundry project endpoint in this order: - -1. `--project-endpoint` -2. `FOUNDRY_PROJECT_ENDPOINT` in the active azd environment -3. the host environment variable of the same name - -## Building - -```console -$ go build ./... -$ go test ./... -``` - -## TODO before release - -Both are files the azd extensions team owns, so they are not changed here: - -- [ ] **`cli/azd/extensions/registry.json`** — add the `azure.ai.dataset` entry. - Until it exists `azd extension install azure.ai.dataset` cannot resolve, so - the extension is only reachable through `azd x pack` + `azd x publish` into - the local source registry. -- [ ] **`.github/CODEOWNERS`** — add `/cli/azd/extensions/azure.ai.dataset/`. - Every sibling Foundry extension has an entry; without one, PRs here get no - reviewer routing. -- [ ] **`microsoft.foundry/extension.yaml`** — add the dependency, but only - after the registry entry lands. Declaring a dependency that cannot resolve - breaks installing the bundle. diff --git a/cli/azd/extensions/azure.ai.dataset/build.ps1 b/cli/azd/extensions/azure.ai.dataset/build.ps1 deleted file mode 100644 index f37f80cabf0..00000000000 --- a/cli/azd/extensions/azure.ai.dataset/build.ps1 +++ /dev/null @@ -1,78 +0,0 @@ -# Ensure script fails on any error -$ErrorActionPreference = 'Stop' - -# Get the directory of the script -$EXTENSION_DIR = Split-Path -Parent $MyInvocation.MyCommand.Path - -# Change to the script directory -Set-Location -Path $EXTENSION_DIR - -# Create a safe version of EXTENSION_ID replacing dots with dashes -$EXTENSION_ID_SAFE = $env:EXTENSION_ID -replace '\.', '-' - -# Define output directory -$OUTPUT_DIR = if ($env:OUTPUT_DIR) { $env:OUTPUT_DIR } else { Join-Path $EXTENSION_DIR "bin" } - -# Create output directory if it doesn't exist -if (-not (Test-Path -Path $OUTPUT_DIR)) { - New-Item -ItemType Directory -Path $OUTPUT_DIR | Out-Null -} - -# Get Git commit hash and build date -$COMMIT = git rev-parse HEAD -if ($LASTEXITCODE -ne 0) { - Write-Host "Error: Failed to get git commit hash" - exit 1 -} -$BUILD_DATE = (Get-Date -Format "yyyy-MM-ddTHH:mm:ssZ") - -# List of OS and architecture combinations -if ($env:EXTENSION_PLATFORM) { - $PLATFORMS = @($env:EXTENSION_PLATFORM) -} -else { - $PLATFORMS = @( - "windows/amd64", - "windows/arm64", - "darwin/amd64", - "darwin/arm64", - "linux/amd64", - "linux/arm64" - ) -} - -$VERSION_PATH = "azureaieval/internal/version" - -# Loop through platforms and build -foreach ($PLATFORM in $PLATFORMS) { - $OS, $ARCH = $PLATFORM -split '/' - - $OUTPUT_NAME = Join-Path $OUTPUT_DIR "$EXTENSION_ID_SAFE-$OS-$ARCH" - - if ($OS -eq "windows") { - $OUTPUT_NAME += ".exe" - } - - Write-Host "Building for $OS/$ARCH..." - - # Delete the output file if it already exists - if (Test-Path -Path $OUTPUT_NAME) { - Remove-Item -Path $OUTPUT_NAME -Force - } - - # Set environment variables for Go build - $env:GOOS = $OS - $env:GOARCH = $ARCH - - go build ` - -ldflags="-X '$VERSION_PATH.Version=$env:EXTENSION_VERSION' -X '$VERSION_PATH.Commit=$COMMIT' -X '$VERSION_PATH.BuildDate=$BUILD_DATE'" ` - -o $OUTPUT_NAME - - if ($LASTEXITCODE -ne 0) { - Write-Host "An error occurred while building for $OS/$ARCH" - exit 1 - } -} - -Write-Host "Build completed successfully!" -Write-Host "Binaries are located in the $OUTPUT_DIR directory." diff --git a/cli/azd/extensions/azure.ai.dataset/build.sh b/cli/azd/extensions/azure.ai.dataset/build.sh deleted file mode 100644 index 4165a516ac4..00000000000 --- a/cli/azd/extensions/azure.ai.dataset/build.sh +++ /dev/null @@ -1,66 +0,0 @@ -#!/bin/bash - -# Get the directory of the script -EXTENSION_DIR="$(cd "$(dirname "$0")" && pwd)" - -# Change to the script directory -cd "$EXTENSION_DIR" || exit - -# Create a safe version of EXTENSION_ID replacing dots with dashes -EXTENSION_ID_SAFE="${EXTENSION_ID//./-}" - -# Define output directory -OUTPUT_DIR="${OUTPUT_DIR:-$EXTENSION_DIR/bin}" - -# Create output and target directories if they don't exist -mkdir -p "$OUTPUT_DIR" - -# Get Git commit hash and build date -COMMIT=$(git rev-parse HEAD) -BUILD_DATE=$(date -u +%Y-%m-%dT%H:%M:%SZ) - -# List of OS and architecture combinations -if [ -n "$EXTENSION_PLATFORM" ]; then - PLATFORMS=("$EXTENSION_PLATFORM") -else - PLATFORMS=( - "windows/amd64" - "windows/arm64" - "darwin/amd64" - "darwin/arm64" - "linux/amd64" - "linux/arm64" - ) -fi - -VERSION_PATH="azureaieval/internal/version" - -# Loop through platforms and build -for PLATFORM in "${PLATFORMS[@]}"; do - OS=$(echo "$PLATFORM" | cut -d'/' -f1) - ARCH=$(echo "$PLATFORM" | cut -d'/' -f2) - - OUTPUT_NAME="$OUTPUT_DIR/$EXTENSION_ID_SAFE-$OS-$ARCH" - - if [ "$OS" = "windows" ]; then - OUTPUT_NAME+='.exe' - fi - - echo "Building for $OS/$ARCH..." - - # Delete the output file if it already exists - [ -f "$OUTPUT_NAME" ] && rm -f "$OUTPUT_NAME" - - # Set environment variables for Go build - GOOS=$OS GOARCH=$ARCH go build \ - -ldflags="-X '$VERSION_PATH.Version=$EXTENSION_VERSION' -X '$VERSION_PATH.Commit=$COMMIT' -X '$VERSION_PATH.BuildDate=$BUILD_DATE'" \ - -o "$OUTPUT_NAME" - - if [ $? -ne 0 ]; then - echo "An error occurred while building for $OS/$ARCH" - exit 1 - fi -done - -echo "Build completed successfully!" -echo "Binaries are located in the $OUTPUT_DIR directory." diff --git a/cli/azd/extensions/azure.ai.dataset/ci-build.ps1 b/cli/azd/extensions/azure.ai.dataset/ci-build.ps1 deleted file mode 100644 index 403bc23b08d..00000000000 --- a/cli/azd/extensions/azure.ai.dataset/ci-build.ps1 +++ /dev/null @@ -1,114 +0,0 @@ -param( - [string] $Version = (Get-Content "$PSScriptRoot/version.txt"), - [string] $SourceVersion = (git rev-parse HEAD), - [switch] $CodeCoverageEnabled, - # Accepted because the shared CI template always passes it. This extension - # has no record/playback mode, so there is no second binary to produce. - [switch] $BuildRecordMode, - [string] $MSYS2Shell, # path to msys2_shell.cmd - [string] $OutputFileName -) -$PSNativeCommandArgumentPassing = 'Legacy' - -# Remove any previously built binaries. -go clean - -if ($LASTEXITCODE) { - Write-Host "Error running go clean" - exit $LASTEXITCODE -} - -# Run `go help build` for detail on these flags. -$buildFlags = @( - # Remove file system paths from the binary. Recorded file names become a - # module path@version, or a plain import path for the standard library. - "-trimpath", - - # Position Independent Executable, for memory-corruption hardening across - # platforms. On Windows this enables ASLR and sets DYNAMICBASE and - # HIGH-ENTROPY-VA in the PE header. - "-buildmode=pie" -) - -if ($CodeCoverageEnabled) { - $buildFlags += "-cover" -} - -# cfi: Control Flow Integrity, cfg: Control Flow Guard, -# osusergo: use the pure Go user lookup. -$tagsFlag = "-tags=cfi,cfg,osusergo" - -# -s: omit the symbol table, -w: omit DWARF, -X: set a variable at link time. -$ldFlag = "-ldflags=-s -w " + - "-X 'azureaieval/internal/version.Version=$Version' " + - "-X 'azureaieval/internal/version.Commit=$SourceVersion' " + - "-X 'azureaieval/internal/version.BuildDate=$(Get-Date -Format o)' " - -if ($IsWindows) { - Write-Host "Building for Windows" -} -elseif ($IsLinux) { - Write-Host "Building for linux" - - # Disable cgo for the x64 Linux build. This also links statically, which - # widens compatibility with older Linux distributions. - if ($env:GOARCH -ne "arm64") { - $env:CGO_ENABLED = "0" - } -} -elseif ($IsMacOS) { - Write-Host "Building for macOS" -} - -$outputFlag = "-o=$OutputFileName" - -$buildFlags += @( - $tagsFlag, - $ldFlag, - $outputFlag -) - -function PrintFlags() { - param( - [string] $flags - ) - - # Format the flags so they can be pasted straight into pwsh. - $i = 0 - foreach ($buildFlag in $buildFlags) { - # Quote values so characters such as ',' survive a repaste. Not needed - # for the direct invocation below. - $argWithValue = $buildFlag.Split('=', 2) - if ($argWithValue.Length -eq 2 -and !$argWithValue[1].StartsWith("`"")) { - $buildFlag = "$($argWithValue[0])=`"$($argWithValue[1])`"" - } - - if ($i -eq $buildFlags.Length - 1) { - Write-Host " $buildFlag" - } - else { - Write-Host " $buildFlag ``" - } - $i++ - } -} - -$oldGOEXPERIMENT = $env:GOEXPERIMENT -# Opt into per-iteration loop variables, which is what most readers expect and -# what the Go team intends to make the default. -$env:GOEXPERIMENT = "loopvar" - -try { - Write-Host "Running: go build ``" - PrintFlags -flags $buildFlags - go build @buildFlags - if ($LASTEXITCODE) { - Write-Host "Error running go build" - exit $LASTEXITCODE - } - - Write-Host "go build succeeded" -} -finally { - $env:GOEXPERIMENT = $oldGOEXPERIMENT -} diff --git a/cli/azd/extensions/azure.ai.dataset/ci-test.ps1 b/cli/azd/extensions/azure.ai.dataset/ci-test.ps1 deleted file mode 100644 index 6175585d318..00000000000 --- a/cli/azd/extensions/azure.ai.dataset/ci-test.ps1 +++ /dev/null @@ -1,56 +0,0 @@ -# Runs the unit tests and writes a JUnit report. -# -# The pipeline publishes **/junitTestReport.xml from the extension directory, -# so the report has to be written under that name for results to show up in the -# build. gotestsum produces it; the go test fallback does not, so the fallback -# only runs when gotestsum is unavailable. -# -# The live integration tests are excluded: they carry the `live` build tag, so -# an untagged run does not compile them, and they additionally require -# AZURE_AI_EVAL_E2E_LIVE and a project endpoint. They are still type-checked -# below, so a change that breaks them cannot reach main unnoticed. -# -# TODO before the first release: PR CI runs this script on windows, linux and -# darwin amd64, so the untagged tests are covered on all three. The live and -# hero suites are only type-checked, never executed, and both have only ever -# run on Windows by hand. Run them once on linux, where they assume a path -# separator and shell out to `azd` and to a proxy address. - -$gopath = go env GOPATH -$gotestsumBinary = "gotestsum" -if ($IsWindows) { - $gotestsumBinary += ".exe" -} -$gotestsum = Join-Path $gopath "bin" $gotestsumBinary - -Write-Host "Running unit tests..." - -if (Test-Path $gotestsum) { - & $gotestsum --format testname --junitfile junitTestReport.xml -- ./... -count=1 -} else { - Write-Host "gotestsum not found; falling back to go test (no JUnit report)." -ForegroundColor Yellow - go test ./... -v -count=1 -} - -if ($LASTEXITCODE -ne 0) { - Write-Host "" - Write-Host "Tests failed with exit code: $LASTEXITCODE" -ForegroundColor Red - exit $LASTEXITCODE -} - -# The tagged suites are never run here, so without this nothing compiles them -# and a change that breaks one reaches main silently. Type-checking needs no -# credentials, so it costs a few seconds and runs everywhere the tests do. -Write-Host "" -Write-Host "Type-checking the live and hero suites..." -go vet -tags live,hero ./... - -if ($LASTEXITCODE -ne 0) { - Write-Host "" - Write-Host "The tagged test suites do not compile: $LASTEXITCODE" -ForegroundColor Red - exit $LASTEXITCODE -} - -Write-Host "" -Write-Host "All tests passed!" -ForegroundColor Green -exit 0 diff --git a/cli/azd/extensions/azure.ai.dataset/cspell.yaml b/cli/azd/extensions/azure.ai.dataset/cspell.yaml deleted file mode 100644 index 4d6c8ad651b..00000000000 --- a/cli/azd/extensions/azure.ai.dataset/cspell.yaml +++ /dev/null @@ -1,27 +0,0 @@ -import: ../../.vscode/cspell.yaml -words: - # Go module and package names - - azureaieval - - evalcore - - httptest - - creack - # Service identifiers and API fields - - evalrun - - lookback - - AOAI - # Built-in evaluator names - - ifeval - - groundedness - # Repository names - - foundrysdk - # Terms - - inlines - - negotiables - - parseable - - retargeted - - subsetting - - undeployed - - undoable - - unpassed - - unscored - - Unparseable diff --git a/cli/azd/extensions/azure.ai.dataset/extension.yaml b/cli/azd/extensions/azure.ai.dataset/extension.yaml deleted file mode 100644 index 39e5b02d77c..00000000000 --- a/cli/azd/extensions/azure.ai.dataset/extension.yaml +++ /dev/null @@ -1,24 +0,0 @@ -# yaml-language-server: $schema=../extension.schema.json -id: azure.ai.dataset -namespace: ai.dataset -displayName: Foundry datasets (Beta) -description: Register and version Foundry datasets from your terminal. (Beta) -usage: azd ai dataset [options] -# NOTE: Make sure version.txt is in sync with this version. -version: 1.0.0-beta.1 -requiredAzdVersion: ">=1.27.1" -language: go -capabilities: - - custom-commands - - metadata -examples: - - name: create - description: Register a dataset from a local file. - usage: azd ai dataset create support-regression --from-file ./data/golden.jsonl - - name: versions list - description: List the versions of a dataset. - usage: azd ai dataset versions list support-regression -tags: - - ai - - foundry - - dataset diff --git a/cli/azd/extensions/azure.ai.dataset/go.mod b/cli/azd/extensions/azure.ai.dataset/go.mod deleted file mode 100644 index 32ac3b07c56..00000000000 --- a/cli/azd/extensions/azure.ai.dataset/go.mod +++ /dev/null @@ -1,105 +0,0 @@ -module azureaidataset - -go 1.26.4 - -require ( - github.com/Azure/azure-sdk-for-go/sdk/azcore v1.21.0 - github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.14.0-beta.3 - github.com/azure/azure-dev/cli/azd v1.28.0 - github.com/fatih/color v1.18.0 - github.com/spf13/cobra v1.10.1 - github.com/spf13/pflag v1.0.10 - github.com/stretchr/testify v1.11.1 -) - -require ( - dario.cat/mergo v1.0.2 // indirect - github.com/AlecAivazis/survey/v2 v2.3.7 // indirect - github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2 // indirect - github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/appservice/armappservice/v2 v2.3.0 // indirect - github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/keyvault/armkeyvault v1.5.0 // indirect - github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armsubscriptions v1.3.0 // indirect - github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azsecrets v1.4.0 // indirect - github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/internal v1.2.0 // indirect - github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0 // indirect - github.com/Masterminds/semver/v3 v3.4.0 // indirect - github.com/adam-lavrik/go-imath v0.0.0-20210910152346-265a42a96f0b // indirect - github.com/alecthomas/chroma/v2 v2.20.0 // indirect - github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect - github.com/aymerick/douceur v0.2.0 // indirect - github.com/bahlo/generic-list-go v0.2.0 // indirect - github.com/blang/semver/v4 v4.0.0 // indirect - github.com/braydonk/yaml v0.9.0 // indirect - github.com/buger/goterm v1.0.4 // indirect - github.com/buger/jsonparser v1.1.2 // indirect - github.com/cespare/xxhash/v2 v2.3.0 // indirect - github.com/charmbracelet/colorprofile v0.3.2 // indirect - github.com/charmbracelet/glamour v0.10.0 // indirect - github.com/charmbracelet/lipgloss v1.1.1-0.20250404203927-76690c660834 // indirect - github.com/charmbracelet/x/ansi v0.10.2 // indirect - github.com/charmbracelet/x/cellbuf v0.0.13 // indirect - github.com/charmbracelet/x/exp/slice v0.0.0-20251008171431-5d3777519489 // indirect - github.com/charmbracelet/x/term v0.2.1 // indirect - github.com/cli/browser v1.3.0 // indirect - github.com/clipperhouse/uax29/v2 v2.2.0 // indirect - github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect - github.com/dlclark/regexp2 v1.11.5 // indirect - github.com/drone/envsubst v1.0.3 // indirect - github.com/go-logr/logr v1.4.3 // indirect - github.com/go-logr/stdr v1.2.2 // indirect - github.com/gofrs/flock v0.12.1 // indirect - github.com/golang-jwt/jwt/v5 v5.3.0 // indirect - github.com/golobby/container/v3 v3.3.2 // indirect - github.com/google/uuid v1.6.0 // indirect - github.com/gorilla/css v1.0.1 // indirect - github.com/inconshreveable/mousetrap v1.1.0 // indirect - github.com/invopop/jsonschema v0.13.0 // indirect - github.com/jmespath-community/go-jmespath v1.1.1 // indirect - github.com/joho/godotenv v1.5.1 // indirect - github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect - github.com/kylelemons/godebug v1.1.0 // indirect - github.com/lucasb-eyer/go-colorful v1.3.0 // indirect - github.com/mailru/easyjson v0.9.1 // indirect - github.com/mark3labs/mcp-go v0.41.1 // indirect - github.com/mattn/go-colorable v0.1.14 // indirect - github.com/mattn/go-isatty v0.0.20 // indirect - github.com/mattn/go-runewidth v0.0.19 // indirect - github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d // indirect - github.com/microcosm-cc/bluemonday v1.0.27 // indirect - github.com/microsoft/ApplicationInsights-Go v0.4.4 // indirect - github.com/microsoft/go-deviceid v1.0.0 // indirect - github.com/muesli/reflow v0.3.0 // indirect - github.com/muesli/termenv v0.16.0 // indirect - github.com/nathan-fiscaletti/consolesize-go v0.0.0-20220204101620-317176b6684d // indirect - github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect - github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect - github.com/rivo/uniseg v0.4.7 // indirect - github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 // indirect - github.com/sethvargo/go-retry v0.3.0 // indirect - github.com/spf13/cast v1.10.0 // indirect - github.com/theckman/yacspin v0.13.12 // indirect - github.com/wk8/go-ordered-map/v2 v2.1.8 // indirect - github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect - github.com/yosida95/uritemplate/v3 v3.0.2 // indirect - github.com/yuin/goldmark v1.7.13 // indirect - github.com/yuin/goldmark-emoji v1.0.6 // indirect - go.opentelemetry.io/auto/sdk v1.2.1 // indirect - go.opentelemetry.io/otel v1.43.0 // indirect - go.opentelemetry.io/otel/metric v1.43.0 // indirect - go.opentelemetry.io/otel/sdk v1.43.0 // indirect - go.opentelemetry.io/otel/trace v1.43.0 // indirect - go.uber.org/atomic v1.11.0 // indirect - go.uber.org/multierr v1.11.0 // indirect - golang.org/x/crypto v0.53.0 // indirect - golang.org/x/exp v0.0.0-20250911091902-df9299821621 // indirect - golang.org/x/net v0.56.0 // indirect - golang.org/x/sync v0.21.0 // indirect - golang.org/x/sys v0.46.0 // indirect - golang.org/x/term v0.44.0 // indirect - golang.org/x/text v0.38.0 // indirect - golang.org/x/time v0.9.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20260401024825-9d38bb4040a9 // indirect - google.golang.org/grpc v1.80.0 // indirect - google.golang.org/protobuf v1.36.11 // indirect - gopkg.in/yaml.v3 v3.0.1 // indirect -) diff --git a/cli/azd/extensions/azure.ai.dataset/go.sum b/cli/azd/extensions/azure.ai.dataset/go.sum deleted file mode 100644 index 1753277fdf9..00000000000 --- a/cli/azd/extensions/azure.ai.dataset/go.sum +++ /dev/null @@ -1,316 +0,0 @@ -code.cloudfoundry.org/clock v0.0.0-20180518195852-02e53af36e6c/go.mod h1:QD9Lzhd/ux6eNQVUDVRJX/RKTigpewimNYBi7ivZKY8= -dario.cat/mergo v1.0.2 h1:85+piFYR1tMbRrLcDwR18y4UKJ3aH1Tbzi24VRW1TK8= -dario.cat/mergo v1.0.2/go.mod h1:E/hbnu0NxMFBjpMIE34DRGLWqDy0g5FuKDhCb31ngxA= -github.com/AlecAivazis/survey/v2 v2.3.7 h1:6I/u8FvytdGsgonrYsVn2t8t4QiRnh6QSTqkkhIiSjQ= -github.com/AlecAivazis/survey/v2 v2.3.7/go.mod h1:xUTIdE4KCOIjsBAE1JYsUPoCqYdZ1reCfTwbto0Fduo= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.21.0 h1:fou+2+WFTib47nS+nz/ozhEBnvU96bKHy6LjRsY4E28= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.21.0/go.mod h1:t76Ruy8AHvUAC8GfMWJMa0ElSbuIcO03NLpynfbgsPA= -github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.14.0-beta.3 h1:0g4UTtvRA9goC37cmD9ZHdW6CCNJR4cOXBnHz0r4ubM= -github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.14.0-beta.3/go.mod h1:fEiHi0sbYqbo3shUkIF1SNxm8GyeEJl+Poc/djOvbdE= -github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.3.2 h1:yz1bePFlP5Vws5+8ez6T3HWXPmwOK7Yvq8QxDBD3SKY= -github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.3.2/go.mod h1:Pa9ZNPuoNu/GztvBSKk9J1cDJW6vk/n0zLtV4mgd8N8= -github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2 h1:9iefClla7iYpfYWdzPCRDozdmndjTm8DXdpCzPajMgA= -github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2/go.mod h1:XtLgD3ZD34DAaVIIAyG3objl5DynM3CQ/vMcbBNJZGI= -github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/appservice/armappservice/v2 v2.3.0 h1:JI8PcWOImyvIUEZ0Bbmfe05FOlWkMi2KhjG+cAKaUms= -github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/appservice/armappservice/v2 v2.3.0/go.mod h1:nJLFPGJkyKfDDyJiPuHIXsCi/gpJkm07EvRgiX7SGlI= -github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/internal/v2 v2.0.0 h1:PTFGRSlMKCQelWwxUyYVEUqseBJVemLyqWJjvMyt0do= -github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/internal/v2 v2.0.0/go.mod h1:LRr2FzBTQlONPPa5HREE5+RjSCTXl7BwOvYOaWTqCaI= -github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/internal/v3 v3.1.0 h1:2qsIIvxVT+uE6yrNldntJKlLRgxGbZ85kgtz5SNBhMw= -github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/internal/v3 v3.1.0/go.mod h1:AW8VEadnhw9xox+VaVd9sP7NjzOAnaZBLRH6Tq3cJ38= -github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/keyvault/armkeyvault v1.5.0 h1:nnQ9vXH039UrEFxi08pPuZBE7VfqSJt343uJLw0rhWI= -github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/keyvault/armkeyvault v1.5.0/go.mod h1:4YIVtzMFVsPwBvitCDX7J9sqthSj43QD1sP6fYc1egc= -github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.2.0 h1:Dd+RhdJn0OTtVGaeDLZpcumkIVCtA/3/Fo42+eoYvVM= -github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.2.0/go.mod h1:5kakwfW5CjC9KK+Q4wjXAg+ShuIm2mBMua0ZFj2C8PE= -github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armsubscriptions v1.3.0 h1:wxQx2Bt4xzPIKvW59WQf1tJNx/ZZKPfN+EhPX3Z6CYY= -github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armsubscriptions v1.3.0/go.mod h1:TpiwjwnW/khS0LKs4vW5UmmT9OWcxaveS8U7+tlknzo= -github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azsecrets v1.4.0 h1:/g8S6wk65vfC6m3FIxJ+i5QDyN9JWwXI8Hb0Img10hU= -github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azsecrets v1.4.0/go.mod h1:gpl+q95AzZlKVI3xSoseF9QPrypk0hQqBiJYeB/cR/I= -github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/internal v1.2.0 h1:nCYfgcSyHZXJI8J0IWE5MsCGlb2xp9fJiXyxWgmOFg4= -github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/internal v1.2.0/go.mod h1:ucUjca2JtSZboY8IoUqyQyuuXvwbMBVwFOm0vdQPNhA= -github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1 h1:WJTmL004Abzc5wDB5VtZG2PJk5ndYDgVacGqfirKxjM= -github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1/go.mod h1:tCcJZ0uHAmvjsVYzEFivsRTN00oz5BEsRgQHu5JZ9WE= -github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0 h1:XRzhVemXdgvJqCH0sFfrBUTnUJSBrBf7++ypk+twtRs= -github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0/go.mod h1:HKpQxkWaGLJ+D/5H8QRpyQXA1eKjxkFlOMwck5+33Jk= -github.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1Xbatp0= -github.com/Masterminds/semver/v3 v3.4.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= -github.com/Netflix/go-expect v0.0.0-20220104043353-73e0943537d2 h1:+vx7roKuyA63nhn5WAunQHLTznkw5W8b1Xc0dNjp83s= -github.com/Netflix/go-expect v0.0.0-20220104043353-73e0943537d2/go.mod h1:HBCaDeC1lPdgDeDbhX8XFpy1jqjK0IBG8W5K+xYqA0w= -github.com/adam-lavrik/go-imath v0.0.0-20210910152346-265a42a96f0b h1:g9SuFmxM/WucQFKTMSP+irxyf5m0RiUJreBDhGI6jSA= -github.com/adam-lavrik/go-imath v0.0.0-20210910152346-265a42a96f0b/go.mod h1:XjvqMUpGd3Xn9Jtzk/4GEBCSoBX0eB2RyriXgne0IdM= -github.com/alecthomas/assert/v2 v2.11.0 h1:2Q9r3ki8+JYXvGsDyBXwH3LcJ+WK5D0gc5E8vS6K3D0= -github.com/alecthomas/assert/v2 v2.11.0/go.mod h1:Bze95FyfUr7x34QZrjL+XP+0qgp/zg8yS+TtBj1WA3k= -github.com/alecthomas/chroma/v2 v2.20.0 h1:sfIHpxPyR07/Oylvmcai3X/exDlE8+FA820NTz+9sGw= -github.com/alecthomas/chroma/v2 v2.20.0/go.mod h1:e7tViK0xh/Nf4BYHl00ycY6rV7b8iXBksI9E359yNmA= -github.com/alecthomas/repr v0.5.1 h1:E3G4t2QbHTSNpPKBgMTln5KLkZHLOcU7r37J4pXBuIg= -github.com/alecthomas/repr v0.5.1/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4= -github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k= -github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8= -github.com/aymanbagabas/go-udiff v0.2.0 h1:TK0fH4MteXUDspT88n8CKzvK0X9O2xu9yQjWpi6yML8= -github.com/aymanbagabas/go-udiff v0.2.0/go.mod h1:RE4Ex0qsGkTAJoQdQQCA0uG+nAzJO/pI/QwceO5fgrA= -github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= -github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4= -github.com/azure/azure-dev/cli/azd v1.28.0 h1:mqqyV85m7A1XfWJFjV/Ut0QoIEImFeF++1Ruq/cRp0s= -github.com/azure/azure-dev/cli/azd v1.28.0/go.mod h1:Ge7QaU9PoJM7i6J0xArDoQCf2tUn6O7OIKkoItxFTA8= -github.com/bahlo/generic-list-go v0.2.0 h1:5sz/EEAK+ls5wF+NeqDpk5+iNdMDXrh3z3nPnH1Wvgk= -github.com/bahlo/generic-list-go v0.2.0/go.mod h1:2KvAjgMlE5NNynlg/5iLrrCCZ2+5xWbdbCW3pNTGyYg= -github.com/benbjohnson/clock v1.3.5 h1:VvXlSJBzZpA/zum6Sj74hxwYI2DIxRWuNIoXAzHZz5o= -github.com/benbjohnson/clock v1.3.5/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= -github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= -github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= -github.com/bradleyjkemp/cupaloy/v2 v2.8.0 h1:any4BmKE+jGIaMpnU8YgH/I2LPiLBufr6oMMlVBbn9M= -github.com/bradleyjkemp/cupaloy/v2 v2.8.0/go.mod h1:bm7JXdkRd4BHJk9HpwqAI8BoAY1lps46Enkdqw6aRX0= -github.com/braydonk/yaml v0.9.0 h1:ewGMrVmEVpsm3VwXQDR388sLg5+aQ8Yihp6/hc4m+h4= -github.com/braydonk/yaml v0.9.0/go.mod h1:hcm3h581tudlirk8XEUPDBAimBPbmnL0Y45hCRl47N4= -github.com/buger/goterm v1.0.4 h1:Z9YvGmOih81P0FbVtEYTFF6YsSgxSUKEhf/f9bTMXbY= -github.com/buger/goterm v1.0.4/go.mod h1:HiFWV3xnkolgrBV3mY8m0X0Pumt4zg4QhbdOzQtB8tE= -github.com/buger/jsonparser v1.1.2 h1:frqHqw7otoVbk5M8LlE/L7HTnIq2v9RX6EJ48i9AxJk= -github.com/buger/jsonparser v1.1.2/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= -github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= -github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/charmbracelet/colorprofile v0.3.2 h1:9J27WdztfJQVAQKX2WOlSSRB+5gaKqqITmrvb1uTIiI= -github.com/charmbracelet/colorprofile v0.3.2/go.mod h1:mTD5XzNeWHj8oqHb+S1bssQb7vIHbepiebQ2kPKVKbI= -github.com/charmbracelet/glamour v0.10.0 h1:MtZvfwsYCx8jEPFJm3rIBFIMZUfUJ765oX8V6kXldcY= -github.com/charmbracelet/glamour v0.10.0/go.mod h1:f+uf+I/ChNmqo087elLnVdCiVgjSKWuXa/l6NU2ndYk= -github.com/charmbracelet/lipgloss v1.1.1-0.20250404203927-76690c660834 h1:ZR7e0ro+SZZiIZD7msJyA+NjkCNNavuiPBLgerbOziE= -github.com/charmbracelet/lipgloss v1.1.1-0.20250404203927-76690c660834/go.mod h1:aKC/t2arECF6rNOnaKaVU6y4t4ZeHQzqfxedE/VkVhA= -github.com/charmbracelet/x/ansi v0.10.2 h1:ith2ArZS0CJG30cIUfID1LXN7ZFXRCww6RUvAPA+Pzw= -github.com/charmbracelet/x/ansi v0.10.2/go.mod h1:HbLdJjQH4UH4AqA2HpRWuWNluRE6zxJH/yteYEYCFa8= -github.com/charmbracelet/x/cellbuf v0.0.13 h1:/KBBKHuVRbq1lYx5BzEHBAFBP8VcQzJejZ/IA3iR28k= -github.com/charmbracelet/x/cellbuf v0.0.13/go.mod h1:xe0nKWGd3eJgtqZRaN9RjMtK7xUYchjzPr7q6kcvCCs= -github.com/charmbracelet/x/exp/golden v0.0.0-20240806155701-69247e0abc2a h1:G99klV19u0QnhiizODirwVksQB91TJKV/UaTnACcG30= -github.com/charmbracelet/x/exp/golden v0.0.0-20240806155701-69247e0abc2a/go.mod h1:wDlXFlCrmJ8J+swcL/MnGUuYnqgQdW9rhSD61oNMb6U= -github.com/charmbracelet/x/exp/slice v0.0.0-20251008171431-5d3777519489 h1:a5q2sWiet6kgqucSGjYN1jhT2cn4bMKUwprtm2IGRto= -github.com/charmbracelet/x/exp/slice v0.0.0-20251008171431-5d3777519489/go.mod h1:vqEfX6xzqW1pKKZUUiFOKg0OQ7bCh54Q2vR/tserrRA= -github.com/charmbracelet/x/term v0.2.1 h1:AQeHeLZ1OqSXhrAWpYUtZyX1T3zVxfpZuEQMIQaGIAQ= -github.com/charmbracelet/x/term v0.2.1/go.mod h1:oQ4enTYFV7QN4m0i9mzHrViD7TQKvNEEkHUMCmsxdUg= -github.com/cli/browser v1.3.0 h1:LejqCrpWr+1pRqmEPDGnTZOjsMe7sehifLynZJuqJpo= -github.com/cli/browser v1.3.0/go.mod h1:HH8s+fOAxjhQoBUAsKuPCbqUuxZDhQ2/aD+SzsEfBTk= -github.com/clipperhouse/uax29/v2 v2.2.0 h1:ChwIKnQN3kcZteTXMgb1wztSgaU+ZemkgWdohwgs8tY= -github.com/clipperhouse/uax29/v2 v2.2.0/go.mod h1:EFJ2TJMRUaplDxHKj1qAEhCtQPW2tJSwu5BF98AuoVM= -github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= -github.com/creack/pty v1.1.17 h1:QeVUsEDNrLBW4tMgZHvxy18sKtr6VI492kBhUfhDJNI= -github.com/creack/pty v1.1.17/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= -github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= -github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dlclark/regexp2 v1.11.5 h1:Q/sSnsKerHeCkc/jSTNq1oCm7KiVgUMZRDUoRu0JQZQ= -github.com/dlclark/regexp2 v1.11.5/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= -github.com/drone/envsubst v1.0.3 h1:PCIBwNDYjs50AsLZPYdfhSATKaRg/FJmDc2D6+C2x8g= -github.com/drone/envsubst v1.0.3/go.mod h1:N2jZmlMufstn1KEqvbHjw40h1KyTmnVzHcSc9bFiJ2g= -github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM= -github.com/fatih/color v1.18.0/go.mod h1:4FelSpRwEGDpQ12mAdzqdOukCy4u8WUtOY6lkT/6HfU= -github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= -github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= -github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= -github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= -github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= -github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= -github.com/gofrs/flock v0.12.1 h1:MTLVXXHf8ekldpJk3AKicLij9MdwOWkZ+a/jHHZby9E= -github.com/gofrs/flock v0.12.1/go.mod h1:9zxTsyu5xtJ9DK+1tFZyibEV7y3uwDxPPfbxeeHCoD0= -github.com/gofrs/uuid v3.3.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= -github.com/golang-jwt/jwt/v5 v5.3.0 h1:pv4AsKCKKZuqlgs5sUmn4x8UlGa0kEVt/puTpKx9vvo= -github.com/golang-jwt/jwt/v5 v5.3.0/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= -github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= -github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= -github.com/golobby/container/v3 v3.3.2 h1:7u+RgNnsdVlhGoS8gY4EXAG601vpMMzLZlYqSp77Quw= -github.com/golobby/container/v3 v3.3.2/go.mod h1:RDdKpnKpV1Of11PFBe7Dxc2C1k2KaLE4FD47FflAmj0= -github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= -github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= -github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= -github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= -github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/gorilla/css v1.0.1 h1:ntNaBIghp6JmvWnxbZKANoLyuXTPZ4cAMlo6RyhlbO8= -github.com/gorilla/css v1.0.1/go.mod h1:BvnYkspnSzMmwRK+b8/xgNPLiIuNZr6vbZBTPQ2A3b0= -github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM= -github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg= -github.com/hinshun/vt10x v0.0.0-20220119200601-820417d04eec h1:qv2VnGeEQHchGaZ/u7lxST/RaJw+cv273q79D81Xbog= -github.com/hinshun/vt10x v0.0.0-20220119200601-820417d04eec/go.mod h1:Q48J4R4DvxnHolD5P8pOtXigYlRuPLGl6moFx3ulM68= -github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= -github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= -github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= -github.com/invopop/jsonschema v0.13.0 h1:KvpoAJWEjR3uD9Kbm2HWJmqsEaHt8lBUpd0qHcIi21E= -github.com/invopop/jsonschema v0.13.0/go.mod h1:ffZ5Km5SWWRAIN6wbDXItl95euhFz2uON45H2qjYt+0= -github.com/jmespath-community/go-jmespath v1.1.1 h1:bFikPhsi/FdmlZhVgSCd2jj1e7G/rw+zyQfyg5UF+L4= -github.com/jmespath-community/go-jmespath v1.1.1/go.mod h1:4gOyFJsR/Gk+05RgTKYrifT7tBPWD8Lubtb5jRrfy9I= -github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= -github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= -github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs= -github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8= -github.com/keybase/go-keychain v0.0.1 h1:way+bWYa6lDppZoZcgMbYsvC7GxljxrskdNInRtuthU= -github.com/keybase/go-keychain v0.0.1/go.mod h1:PdEILRW3i9D8JcdM+FmY6RwkHGnhHxXwkPPMeUgOK1k= -github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= -github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= -github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= -github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= -github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= -github.com/lucasb-eyer/go-colorful v1.3.0 h1:2/yBRLdWBZKrf7gB40FoiKfAWYQ0lqNcbuQwVHXptag= -github.com/lucasb-eyer/go-colorful v1.3.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= -github.com/mailru/easyjson v0.9.1 h1:LbtsOm5WAswyWbvTEOqhypdPeZzHavpZx96/n553mR8= -github.com/mailru/easyjson v0.9.1/go.mod h1:1+xMtQp2MRNVL/V1bOzuP3aP8VNwRW55fQUto+XFtTU= -github.com/mark3labs/mcp-go v0.41.1 h1:w78eWfiQam2i8ICL7AL0WFiq7KHNJQ6UB53ZVtH4KGA= -github.com/mark3labs/mcp-go v0.41.1/go.mod h1:T7tUa2jO6MavG+3P25Oy/jR7iCeJPHImCZHRymCn39g= -github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= -github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE= -github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= -github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= -github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= -github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= -github.com/mattn/go-runewidth v0.0.12/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk= -github.com/mattn/go-runewidth v0.0.19 h1:v++JhqYnZuu5jSKrk9RbgF5v4CGUjqRfBm05byFGLdw= -github.com/mattn/go-runewidth v0.0.19/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhgLpndooCuJAs= -github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= -github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d h1:5PJl274Y63IEHC+7izoQE9x6ikvDFZS2mDVS3drnohI= -github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= -github.com/microcosm-cc/bluemonday v1.0.27 h1:MpEUotklkwCSLeH+Qdx1VJgNqLlpY2KXwXFM08ygZfk= -github.com/microcosm-cc/bluemonday v1.0.27/go.mod h1:jFi9vgW+H7c3V0lb6nR74Ib/DIB5OBs92Dimizgw2cA= -github.com/microsoft/ApplicationInsights-Go v0.4.4 h1:G4+H9WNs6ygSCe6sUyxRc2U81TI5Es90b2t/MwX5KqY= -github.com/microsoft/ApplicationInsights-Go v0.4.4/go.mod h1:fKRUseBqkw6bDiXTs3ESTiU/4YTIHsQS4W3fP2ieF4U= -github.com/microsoft/go-deviceid v1.0.0 h1:i5AQ654Xk9kfvwJeKQm3w2+eT1+ImBDVEpAR0AjpP40= -github.com/microsoft/go-deviceid v1.0.0/go.mod h1:KY13FeVdHkzD8gy+6T8+kVmD/7RMpTaWW75K+T4uZWg= -github.com/muesli/reflow v0.3.0 h1:IFsN6K9NfGtjeggFP+68I4chLZV2yIKsXJFNZ+eWh6s= -github.com/muesli/reflow v0.3.0/go.mod h1:pbwTDkVPibjO2kyvBQRBxTWEEGDGq0FlB1BIKtnHY/8= -github.com/muesli/termenv v0.16.0 h1:S5AlUN9dENB57rsbnkPyfdGuWIlkmzJjbFf0Tf5FWUc= -github.com/muesli/termenv v0.16.0/go.mod h1:ZRfOIKPFDYQoDFF4Olj7/QJbW60Ol/kL1pU3VfY/Cnk= -github.com/nathan-fiscaletti/consolesize-go v0.0.0-20220204101620-317176b6684d h1:NqRhLdNVlozULwM1B3VaHhcXYSgrOAv8V5BE65om+1Q= -github.com/nathan-fiscaletti/consolesize-go v0.0.0-20220204101620-317176b6684d/go.mod h1:cxIIfNMTwff8f/ZvRouvWYF6wOoO7nj99neWSx2q/Es= -github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= -github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ= -github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU= -github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= -github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= -github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= -github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= -github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= -github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= -github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= -github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 h1:KRzFb2m7YtdldCEkzs6KqmJw4nqEVZGK7IN2kJkjTuQ= -github.com/santhosh-tekuri/jsonschema/v6 v6.0.2/go.mod h1:JXeL+ps8p7/KNMjDQk3TCwPpBy0wYklyWTfbkIzdIFU= -github.com/sethvargo/go-retry v0.3.0 h1:EEt31A35QhrcRZtrYFDTBg91cqZVnFL2navjDrah2SE= -github.com/sethvargo/go-retry v0.3.0/go.mod h1:mNX17F0C/HguQMyMyJxcnU471gOZGxCLyYaFyAZraas= -github.com/spf13/cast v1.10.0 h1:h2x0u2shc1QuLHfxi+cTJvs30+ZAHOGRic8uyGTDWxY= -github.com/spf13/cast v1.10.0/go.mod h1:jNfB8QC9IA6ZuY2ZjDp0KtFO2LZZlg4S/7bzP6qqeHo= -github.com/spf13/cobra v1.10.1 h1:lJeBwCfmrnXthfAupyUTzJ/J4Nc1RsHC/mSRU2dll/s= -github.com/spf13/cobra v1.10.1/go.mod h1:7SmJGaTHFVBY0jW4NXGluQoLvhqFQM+6XSKD+P4XaB0= -github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk= -github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= -github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= -github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= -github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= -github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= -github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= -github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= -github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= -github.com/tedsuo/ifrit v0.0.0-20180802180643-bea94bb476cc/go.mod h1:eyZnKCc955uh98WQvzOm0dgAeLnf2O0Rz0LPoC5ze+0= -github.com/theckman/yacspin v0.13.12 h1:CdZ57+n0U6JMuh2xqjnjRq5Haj6v1ner2djtLQRzJr4= -github.com/theckman/yacspin v0.13.12/go.mod h1:Rd2+oG2LmQi5f3zC3yeZAOl245z8QOvrH4OPOJNZxLg= -github.com/wk8/go-ordered-map/v2 v2.1.8 h1:5h/BUHu93oj4gIdvHHHGsScSTMijfx5PeYkE/fJgbpc= -github.com/wk8/go-ordered-map/v2 v2.1.8/go.mod h1:5nJHM5DyteebpVlHnWMV0rPz6Zp7+xBAnxjb1X5vnTw= -github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavMF/ppJZNG9ZpyihvCd0w101no= -github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJuqunuUZ/Dhy/avygyECGrLceyNeo4LiM= -github.com/yosida95/uritemplate/v3 v3.0.2 h1:Ed3Oyj9yrmi9087+NczuL5BwkIc4wvTb5zIM+UJPGz4= -github.com/yosida95/uritemplate/v3 v3.0.2/go.mod h1:ILOh0sOhIJR3+L/8afwt/kE++YT040gmv5BQTMR2HP4= -github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= -github.com/yuin/goldmark v1.7.13 h1:GPddIs617DnBLFFVJFgpo1aBfe/4xcvMc3SB5t/D0pA= -github.com/yuin/goldmark v1.7.13/go.mod h1:ip/1k0VRfGynBgxOz0yCqHrbZXhcjxyuS66Brc7iBKg= -github.com/yuin/goldmark-emoji v1.0.6 h1:QWfF2FYaXwL74tfGOW5izeiZepUDroDJfWubQI9HTHs= -github.com/yuin/goldmark-emoji v1.0.6/go.mod h1:ukxJDKFpdFb5x0a5HqbdlcKtebh086iJpI31LTKmWuA= -go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= -go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= -go.opentelemetry.io/otel v1.43.0 h1:mYIM03dnh5zfN7HautFE4ieIig9amkNANT+xcVxAj9I= -go.opentelemetry.io/otel v1.43.0/go.mod h1:JuG+u74mvjvcm8vj8pI5XiHy1zDeoCS2LB1spIq7Ay0= -go.opentelemetry.io/otel/metric v1.43.0 h1:d7638QeInOnuwOONPp4JAOGfbCEpYb+K6DVWvdxGzgM= -go.opentelemetry.io/otel/metric v1.43.0/go.mod h1:RDnPtIxvqlgO8GRW18W6Z/4P462ldprJtfxHxyKd2PY= -go.opentelemetry.io/otel/sdk v1.43.0 h1:pi5mE86i5rTeLXqoF/hhiBtUNcrAGHLKQdhg4h4V9Dg= -go.opentelemetry.io/otel/sdk v1.43.0/go.mod h1:P+IkVU3iWukmiit/Yf9AWvpyRDlUeBaRg6Y+C58QHzg= -go.opentelemetry.io/otel/sdk/metric v1.43.0 h1:S88dyqXjJkuBNLeMcVPRFXpRw2fuwdvfCGLEo89fDkw= -go.opentelemetry.io/otel/sdk/metric v1.43.0/go.mod h1:C/RJtwSEJ5hzTiUz5pXF1kILHStzb9zFlIEe85bhj6A= -go.opentelemetry.io/otel/trace v1.43.0 h1:BkNrHpup+4k4w+ZZ86CZoHHEkohws8AY+WTX09nk+3A= -go.opentelemetry.io/otel/trace v1.43.0/go.mod h1:/QJhyVBUUswCphDVxq+8mld+AvhXZLhe+8WVFxiFff0= -go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= -go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= -go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= -go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.53.0 h1:QZ4Muo8THX6CizN2vPPd5fBGHyogrdK9fG4wLPFUsto= -golang.org/x/crypto v0.53.0/go.mod h1:DNLU434OwVakk9PzuwV8w62mAJpRJL3vsgcfp4Qnsio= -golang.org/x/exp v0.0.0-20250911091902-df9299821621 h1:2id6c1/gto0kaHYyrixvknJ8tUK/Qs5IsmBtrc+FtgU= -golang.org/x/exp v0.0.0-20250911091902-df9299821621/go.mod h1:TwQYMMnGpvZyc+JpB/UAuTNIsVJifOlSkrZkhcvpVUk= -golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.56.0 h1:Rw8j/hFzGvJUZwNBXnAtf5sVDVt+65SK2C7IxCxZt5o= -golang.org/x/net v0.56.0/go.mod h1:D3Ku6r+V6JROoZK144D2XfMHFcMq/0zSfLelVTCFKec= -golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM= -golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= -golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210331175145-43e1dd70ce54/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw= -golang.org/x/sys v0.46.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= -golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.44.0 h1:0rLvDRCtNj0gZkyIXhCyOb2OAzEhLVqc4B+hrsBhrmc= -golang.org/x/term v0.44.0/go.mod h1:7ze4MdzUzLXpSAoFP1H0bOI9aXDqveSvatT5vKcFh2Y= -golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.38.0 h1:sXmwo9DwP3OK9EZ7PqAdaooSGozfl/3a6/xJcbzPRhE= -golang.org/x/text v0.38.0/go.mod h1:YXZt3QhHUKYT53r2lLKFIVi6Ao1jdzrTR/KQ09qyxF4= -golang.org/x/time v0.9.0 h1:EsRrnYcQiGH+5FfbgvV4AP7qEZstoyrHB0DzarOQ4ZY= -golang.org/x/time v0.9.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= -golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= -gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260401024825-9d38bb4040a9 h1:m8qni9SQFH0tJc1X0vmnpw/0t+AImlSvp30sEupozUg= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260401024825-9d38bb4040a9/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= -google.golang.org/grpc v1.80.0 h1:Xr6m2WmWZLETvUNvIUmeD5OAagMw3FiKmMlTdViWsHM= -google.golang.org/grpc v1.80.0/go.mod h1:ho/dLnxwi3EDJA4Zghp7k2Ec1+c2jqup0bFkw07bwF4= -google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= -google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= -gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= -gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= -gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= -gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/cli/azd/extensions/azure.ai.dataset/internal/cmd/apiversions.go b/cli/azd/extensions/azure.ai.dataset/internal/cmd/apiversions.go deleted file mode 100644 index 16ad4a772c4..00000000000 --- a/cli/azd/extensions/azure.ai.dataset/internal/cmd/apiversions.go +++ /dev/null @@ -1,13 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package cmd - -// API versions used by the Foundry data plane. -const ( - // ProjectEndpointAPIVersion covers datasets on the project endpoint. - ProjectEndpointAPIVersion = "2025-11-15-preview" - - // DataGenerationAPIVersion covers dataset generation jobs. - DataGenerationAPIVersion = "v1" -) diff --git a/cli/azd/extensions/azure.ai.dataset/internal/cmd/artifacts.go b/cli/azd/extensions/azure.ai.dataset/internal/cmd/artifacts.go deleted file mode 100644 index c8a101b5a36..00000000000 --- a/cli/azd/extensions/azure.ai.dataset/internal/cmd/artifacts.go +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package cmd - -import "fmt" - -// envKeyDatasetVersion caches the version resolved at the last publish, so a -// later read does not have to list every version to find the newest. -const envKeyDatasetVersion = "EVAL_DATASET_VERSION" - -// checkAssetExistence enforces the one difference between create and update. -func checkAssetExistence(verb, kind, name string, exists bool) error { - switch { - case verb == "create" && exists: - return fmt.Errorf( - "%s %q already exists: use `update` to publish a new version", kind, name) - case verb == "update" && !exists: - return fmt.Errorf( - "%s %q does not exist: use `create` to register it", kind, name) - } - return nil -} diff --git a/cli/azd/extensions/azure.ai.dataset/internal/cmd/context.go b/cli/azd/extensions/azure.ai.dataset/internal/cmd/context.go deleted file mode 100644 index 2e6b6380ebb..00000000000 --- a/cli/azd/extensions/azure.ai.dataset/internal/cmd/context.go +++ /dev/null @@ -1,140 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package cmd - -import ( - "context" - "errors" - "fmt" - "log" - "strings" - - "azureaidataset/internal/foundry/projectctx" - "azureaidataset/internal/pkg/dataset_api" - - "github.com/Azure/azure-sdk-for-go/sdk/azcore" - "github.com/Azure/azure-sdk-for-go/sdk/azidentity" - "github.com/azure/azure-dev/cli/azd/pkg/azdext" -) - -// projectEndpointEnvKey is the azd environment key holding the Foundry project -// endpoint the data-plane clients target. -const projectEndpointEnvKey = "FOUNDRY_PROJECT_ENDPOINT" - -// datasetContext carries everything the commands need to reach the data plane. -type datasetContext struct { - azdClient *azdext.AzdClient - endpoint string - envName string - cred azcore.TokenCredential - - datasetClient *dataset_api.DatasetClient -} - -// newDatasetContext resolves the project endpoint and builds the data-plane -// clients. The resolution order is projectctx's, so that every Foundry -// extension answers the same question the same way: -// -// 1. --project-endpoint -// 2. the active azd environment (FOUNDRY_PROJECT_ENDPOINT, then AZURE_AI_PROJECT_ENDPOINT) -// 3. global config: extensions.ai-agents.project.context.endpoint -// 4. the host environment variables of the same two names -// 5. otherwise an error naming how to set one -func newDatasetContext(ctx context.Context, endpointFlag string) (*datasetContext, error) { - azdClient, err := azdext.NewAzdClient() - if err != nil { - return nil, fmt.Errorf("connecting to azd: %w", err) - } - - dc := &datasetContext{azdClient: azdClient} - - // The environment name is resolved regardless of where the endpoint came - // from: it is what cached version numbers are read from and written to. - _, envName := lookupEndpointFromAzd(ctx, azdClient) - dc.envName = envName - - resolved, err := projectctx.Resolve(ctx, projectctx.ResolveOpts{FlagValue: endpointFlag}) - if err != nil { - return nil, err - } - dc.endpoint = strings.TrimSuffix(resolved.Endpoint, "/") - log.Printf("[endpoint] resolved from %s", resolved.Source) - - cred, err := azidentity.NewAzureDeveloperCLICredential( - &azidentity.AzureDeveloperCLICredentialOptions{}, - ) - if err != nil { - return nil, fmt.Errorf("creating Azure credential: %w", err) - } - dc.cred = cred - - dc.datasetClient = dataset_api.NewDatasetClient(dc.endpoint, cred) - - return dc, nil -} - -// lookupEndpointFromAzd reads the endpoint from the active azd environment, -// returning empty strings when azd has no current environment. -func lookupEndpointFromAzd(ctx context.Context, azdClient *azdext.AzdClient) (endpoint, envName string) { - envResp, err := azdClient.Environment().GetCurrent(ctx, &azdext.EmptyRequest{}) - if err != nil || envResp == nil || envResp.Environment == nil { - return "", "" - } - val, err := azdClient.Environment().GetValue(ctx, &azdext.GetEnvRequest{ - EnvName: envResp.Environment.Name, - Key: projectEndpointEnvKey, - }) - if err != nil || val == nil || val.Value == "" { - return "", envResp.Environment.Name - } - return val.Value, envResp.Environment.Name -} - -// errNoAzdEnvironment reports that there is no azd environment to persist into. -// -// These commands work standalone against the data plane, so running outside a -// project is ordinary rather than a problem worth reporting. -var errNoAzdEnvironment = errors.New("no active azd environment") - -// setEnvValue persists a value into the active azd environment. -func (dc *datasetContext) setEnvValue(ctx context.Context, key, value string) error { - if dc.envName == "" { - envResp, err := dc.azdClient.Environment().GetCurrent(ctx, &azdext.EmptyRequest{}) - if err != nil || envResp == nil || envResp.Environment == nil { - return fmt.Errorf("%w to write %s into", errNoAzdEnvironment, key) - } - dc.envName = envResp.Environment.Name - } - _, err := dc.azdClient.Environment().SetValue(ctx, &azdext.SetEnvRequest{ - EnvName: dc.envName, - Key: key, - Value: value, - }) - if err != nil { - return fmt.Errorf("writing %s to the azd environment: %w", key, err) - } - return nil -} - -// getEnvValue reads a value from the active azd environment, returning empty -// when it is unset. -func (dc *datasetContext) getEnvValue(ctx context.Context, key string) string { - if dc.envName == "" { - return "" - } - val, err := dc.azdClient.Environment().GetValue(ctx, &azdext.GetEnvRequest{ - EnvName: dc.envName, - Key: key, - }) - if err != nil || val == nil { - return "" - } - return val.Value -} - -func (dc *datasetContext) Close() { - if dc.azdClient != nil { - dc.azdClient.Close() - } -} diff --git a/cli/azd/extensions/azure.ai.dataset/internal/cmd/dataset.go b/cli/azd/extensions/azure.ai.dataset/internal/cmd/dataset.go deleted file mode 100644 index c96f52e3f45..00000000000 --- a/cli/azd/extensions/azure.ai.dataset/internal/cmd/dataset.go +++ /dev/null @@ -1,321 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package cmd - -import ( - "errors" - "fmt" - "os" - "path/filepath" - "strings" - - "azureaidataset/internal/pkg/dataset_api" - - "github.com/spf13/cobra" -) - -// firstDatasetVersion is the version the service assigns to a dataset's first -// publish, and so the one that exists for every dataset that exists at all. -const firstDatasetVersion = "1" - -// newDatasetCreateCommand builds `dataset create `, which registers a -// dataset that does not exist yet. -func newDatasetCreateCommand() *cobra.Command { - return newDatasetWriteCommand("create", "Register a dataset, publishing its first version.") -} - -// newDatasetUpdateCommand builds `dataset update `, which publishes a -// further version of one that does. -func newDatasetUpdateCommand() *cobra.Command { - return newDatasetWriteCommand("update", "Publish a new version of a dataset.") -} - -// newDatasetWriteCommand builds create and update. Both run the same upload, -// and the existence check is the only thing that separates them: a version is -// brought into being by startPendingUpload, which neither knows nor cares -// whether the name was already in use. -func newDatasetWriteCommand(verb, short string) *cobra.Command { - var ( - fromFile string - version string - endpointFlg string - ) - - cmd := &cobra.Command{ - Use: verb + " ", - Short: short, - Args: cobra.ExactArgs(1), - RunE: func(cmd *cobra.Command, args []string) error { - name := args[0] - if fromFile == "" { - return requireFlag("from-file") - } - - localDir, err := datasetUploadDir(fromFile) - if err != nil { - return err - } - - ctx := cmd.Context() - ec, err := newDatasetContext(ctx, endpointFlg) - if err != nil { - return err - } - defer ec.Close() - - existing, err := ec.datasetClient.ListDatasetVersions( - ctx, name, ProjectEndpointAPIVersion, - ) - exists := err == nil && existing != nil && len(existing.Value) > 0 - if !exists { - // The version listing lags a publish, so a `create` followed by - // an `update` was told the dataset it had just made does not - // exist. A direct read of the first version settles it: point - // reads go consistent immediately. - if _, err := ec.datasetClient.GetDataset( - ctx, name, firstDatasetVersion, ProjectEndpointAPIVersion, - ); err == nil { - exists = true - } - } - if err := checkAssetExistence(verb, "dataset", name, exists); err != nil { - return err - } - - ds, err := ec.datasetClient.UploadNextVersion( - ctx, name, version, localDir, ProjectEndpointAPIVersion, - ) - if err != nil { - return fmt.Errorf("registering dataset %q: %w", name, err) - } - - if err := ec.setEnvValue(ctx, envKeyDatasetVersion, ds.Version); err != nil { - // Persisting is a convenience, so this never fails the command. - // It goes to stdout because azd does not surface an extension's - // stderr, and is skipped outside a project, where having nowhere - // to persist is expected rather than notable. - if !errors.Is(err, errNoAzdEnvironment) && !isJSON(cmd) { - fmt.Fprintf(cmd.OutOrStdout(), "warning: %v\n", err) - } - } - - if isJSON(cmd) { - return emitJSON(cmd.OutOrStdout(), ds) - } - fmt.Fprintf(cmd.OutOrStdout(), "Registered dataset %s version %s\n", ds.Name, ds.Version) - return nil - }, - } - - cmd.Flags().StringVar(&fromFile, "from-file", "", - "Path to a .jsonl file, or a directory containing one.") - cmd.Flags().StringVar(&version, "version", "", - "Current version to increment from. Omit to increment from the latest registered version.") - cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") - return cmd -} - -// datasetUploadDir resolves what was named into the directory the upload scans. -func datasetUploadDir(path string) (string, error) { - info, err := os.Stat(path) - if err != nil { - return "", fmt.Errorf("reading --from-file %q: %w", path, err) - } - if info.IsDir() { - return path, nil - } - if !strings.EqualFold(filepath.Ext(path), ".jsonl") { - return "", fmt.Errorf( - "--from-file must be a .jsonl file or a directory containing one, got %q", path) - } - return filepath.Dir(path), nil -} - -func newDatasetListCommand() *cobra.Command { - var endpointFlg string - - cmd := &cobra.Command{ - Use: "list", - Short: "List the project's datasets.", - Args: cobra.NoArgs, - RunE: func(cmd *cobra.Command, args []string) error { - ctx := cmd.Context() - ec, err := newDatasetContext(ctx, endpointFlg) - if err != nil { - return err - } - defer ec.Close() - - list, err := ec.datasetClient.ListDatasets(ctx, ProjectEndpointAPIVersion) - if err != nil { - return fmt.Errorf("listing datasets: %w", err) - } - return renderDatasets(cmd, list) - }, - } - - cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") - return cmd -} - -// newDatasetVersionsCommand groups the version listing, so that `list` means -// the assets rather than the history of one of them. -func newDatasetVersionsCommand() *cobra.Command { - cmd := &cobra.Command{ - Use: "versions", - Short: "Inspect the versions of one dataset.", - } - cmd.AddCommand(newDatasetVersionsListCommand()) - return cmd -} - -func newDatasetVersionsListCommand() *cobra.Command { - var endpointFlg string - - cmd := &cobra.Command{ - Use: "list ", - Short: "List the versions of a dataset.", - Args: cobra.ExactArgs(1), - RunE: func(cmd *cobra.Command, args []string) error { - name := args[0] - - ctx := cmd.Context() - ec, err := newDatasetContext(ctx, endpointFlg) - if err != nil { - return err - } - defer ec.Close() - - list, err := ec.datasetClient.ListDatasetVersions(ctx, name, ProjectEndpointAPIVersion) - if err != nil { - return fmt.Errorf("listing versions of dataset %q: %w", name, err) - } - return renderDatasets(cmd, list) - }, - } - - cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") - return cmd -} - -func renderDatasets(cmd *cobra.Command, list *dataset_api.DatasetList) error { - if isJSON(cmd) { - return emitJSONList(cmd.OutOrStdout(), list.Value) - } - rows := make([][]string, 0, len(list.Value)) - for _, d := range list.Value { - rows = append(rows, []string{d.Name, d.Version, d.Format}) - } - if len(rows) == 0 { - fmt.Fprintln(cmd.OutOrStdout(), "No datasets found.") - return nil - } - return emitTable(cmd.OutOrStdout(), []string{"NAME", "VERSION", "FORMAT"}, rows) -} - -func newDatasetShowCommand() *cobra.Command { - var ( - version string - endpointFlg string - ) - - cmd := &cobra.Command{ - Use: "show ", - Short: "Show a dataset version.", - Args: cobra.ExactArgs(1), - RunE: func(cmd *cobra.Command, args []string) error { - name := args[0] - - ctx := cmd.Context() - ec, err := newDatasetContext(ctx, endpointFlg) - if err != nil { - return err - } - defer ec.Close() - - if version == "" { - list, err := ec.datasetClient.ListDatasetVersions(ctx, name, ProjectEndpointAPIVersion) - if err != nil { - return fmt.Errorf("resolving the latest version of %q: %w", name, err) - } - if len(list.Value) == 0 { - return fmt.Errorf("dataset %q has no versions", name) - } - version = dataset_api.LatestVersion(list.Value) - } - - ds, err := ec.datasetClient.GetDataset(ctx, name, version, ProjectEndpointAPIVersion) - if err != nil { - if dataset_api.IsNotFound(err) { - return fmt.Errorf( - "no dataset %q at version %q in this project; "+ - "`azd ai dataset list` shows the ones there are", name, version) - } - return fmt.Errorf("reading dataset %q version %q: %w", name, version, err) - } - - if isJSON(cmd) { - return emitJSON(cmd.OutOrStdout(), ds) - } - return emitDetail(cmd.OutOrStdout(), []field{ - {"Name", ds.Name}, - {"Version", ds.Version}, - {"Format", ds.Format}, - {"URI", ds.ResolvedBlobURI()}, - }) - }, - } - - cmd.Flags().StringVar(&version, "version", "", "Version to show. Omit for the latest.") - cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") - return cmd -} - -func newDatasetDeleteCommand() *cobra.Command { - var ( - version string - endpointFlg string - ) - - cmd := &cobra.Command{ - Use: "delete ", - Short: "Delete a dataset version.", - Args: cobra.ExactArgs(1), - RunE: func(cmd *cobra.Command, args []string) error { - name := args[0] - if version == "" { - return requireFlag("version") - } - - ctx := cmd.Context() - ec, err := newDatasetContext(ctx, endpointFlg) - if err != nil { - return err - } - defer ec.Close() - - if err := ec.datasetClient.DeleteDatasetVersion( - ctx, name, version, ProjectEndpointAPIVersion, - ); err != nil { - if dataset_api.IsNotFound(err) { - return fmt.Errorf( - "no dataset %q at version %q in this project", name, version) - } - return fmt.Errorf("deleting dataset %q version %q: %w", name, version, err) - } - - if isJSON(cmd) { - return emitJSON(cmd.OutOrStdout(), map[string]string{ - "name": name, "version": version, "status": "deleted", - }) - } - fmt.Fprintf(cmd.OutOrStdout(), "Deleted dataset %s version %s\n", name, version) - return nil - }, - } - - cmd.Flags().StringVar(&version, "version", "", "Version to delete.") - cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") - return cmd -} diff --git a/cli/azd/extensions/azure.ai.dataset/internal/cmd/debug.go b/cli/azd/extensions/azure.ai.dataset/internal/cmd/debug.go deleted file mode 100644 index 48fc0802aaf..00000000000 --- a/cli/azd/extensions/azure.ai.dataset/internal/cmd/debug.go +++ /dev/null @@ -1,65 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package cmd - -import ( - "fmt" - "io" - "log" - "os" - "strconv" - "time" - - azcorelog "github.com/Azure/azure-sdk-for-go/sdk/azcore/log" - "github.com/spf13/pflag" -) - -// setupDebugLogging silences the standard logger unless debug mode is on. -// -// The data-plane clients trace every request through log.Printf, which Go -// writes to stderr by default. Without this the CLI interleaves raw HTTP traces -// with its own output on every command. Returns a cleanup function the caller -// should defer. -func setupDebugLogging(flags *pflag.FlagSet) func() { - if !isDebug(flags) { - log.SetOutput(io.Discard) - azcorelog.SetListener(nil) - return func() {} - } - - logFileName := fmt.Sprintf("azd-ai-eval-%s.log", time.Now().Format("2006-01-02")) - - //nolint:gosec // the name is generated locally from the date, not user input - logFile, err := os.OpenFile(logFileName, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0600) - - var w io.Writer - var closeFile func() - if err != nil { - w = os.Stderr - closeFile = func() {} - } else { - w = logFile - closeFile = func() { logFile.Close() } //nolint:gosec // best-effort cleanup - } - - log.SetOutput(w) - azcorelog.SetListener(func(event azcorelog.Event, msg string) { - fmt.Fprintf(w, "[%s] %s: %s\n", time.Now().Format(time.RFC3339), event, msg) - }) - - return func() { - log.SetOutput(io.Discard) - azcorelog.SetListener(nil) - closeFile() - } -} - -// isDebug reports whether --debug or AZD_EXT_DEBUG is set. -func isDebug(flags *pflag.FlagSet) bool { - if debugFlag, err := flags.GetBool("debug"); err == nil && debugFlag { - return true - } - debug, _ := strconv.ParseBool(os.Getenv("AZD_EXT_DEBUG")) - return debug -} diff --git a/cli/azd/extensions/azure.ai.dataset/internal/cmd/helpers_test.go b/cli/azd/extensions/azure.ai.dataset/internal/cmd/helpers_test.go deleted file mode 100644 index 8c226d8b5e4..00000000000 --- a/cli/azd/extensions/azure.ai.dataset/internal/cmd/helpers_test.go +++ /dev/null @@ -1,26 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package cmd - -import ( - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -// The one difference between create and update, and the only thing stopping a -// create from silently publishing version 2 of someone else's dataset. -func TestCheckAssetExistence(t *testing.T) { - assert.NoError(t, checkAssetExistence("create", "dataset", "x", false)) - assert.NoError(t, checkAssetExistence("update", "dataset", "x", true)) - - err := checkAssetExistence("create", "dataset", "x", true) - require.Error(t, err) - assert.Contains(t, err.Error(), "update", "the error has to name the verb that works") - - err = checkAssetExistence("update", "dataset", "x", false) - require.Error(t, err) - assert.Contains(t, err.Error(), "create") -} diff --git a/cli/azd/extensions/azure.ai.dataset/internal/cmd/output.go b/cli/azd/extensions/azure.ai.dataset/internal/cmd/output.go deleted file mode 100644 index 8cb9eb1f4ce..00000000000 --- a/cli/azd/extensions/azure.ai.dataset/internal/cmd/output.go +++ /dev/null @@ -1,121 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package cmd - -import ( - "encoding/json" - "fmt" - "io" - "strings" - "text/tabwriter" - - "github.com/spf13/cobra" -) - -const outputJSON = "json" - -// Progress markers from the azd style guide, so the extension's lines sit -// alongside core's without a second vocabulary. -const ( - doneMark = "(✓) Done:" // finished successfully - skippedMark = "(-) Skipped:" // intentionally not done, not a failure - failedMark = "(x) Failed:" // the step did not complete -) - -// outputFormat reads the inherited -o/--output flag. -func outputFormat(cmd *cobra.Command) string { - if cmd == nil { - return "" - } - v, err := cmd.Flags().GetString("output") - if err != nil { - return "" - } - return strings.ToLower(v) -} - -// isJSON reports whether the command should emit machine-readable output. -func isJSON(cmd *cobra.Command) bool { - return outputFormat(cmd) == outputJSON -} - -// emitJSON writes v as indented JSON. -func emitJSON(w io.Writer, v any) error { - enc := json.NewEncoder(w) - enc.SetIndent("", " ") - return enc.Encode(v) -} - -// emitJSONList writes items as a JSON array. -// -// List commands emit a bare array rather than the envelope the service replied -// with. The envelopes disagree with each other — the OpenAI-shaped APIs wrap -// results in `data`, the ARM-shaped ones in `value` — so passing them through -// would make a caller's parsing depend on which service happens to back a given -// command. They also carry paging fields that this extension does not follow, -// which would suggest there is more to fetch when there is not. -// -// A nil slice encodes as `null`, so it is normalized to an empty array: a -// caller iterating the result should see no elements, not a type error. -func emitJSONList[T any](w io.Writer, items []T) error { - if items == nil { - items = []T{} - } - return emitJSON(w, items) -} - -// emitTable writes a list view: uppercase headers over a rule, tab-aligned. -// -// The rule is what separates the header from the data at a glance, and it is -// what `azure.ai.skills` prints, so a reader moving between the Foundry -// extensions sees one table. -func emitTable(w io.Writer, headers []string, rows [][]string) error { - tw := tabwriter.NewWriter(w, 0, 0, 3, ' ', 0) - if _, err := fmt.Fprintln(tw, strings.Join(headers, "\t")); err != nil { - return err - } - rule := make([]string, len(headers)) - for i, h := range headers { - rule[i] = strings.Repeat("-", len(h)) - } - if _, err := fmt.Fprintln(tw, strings.Join(rule, "\t")); err != nil { - return err - } - for _, row := range rows { - if _, err := fmt.Fprintln(tw, strings.Join(row, "\t")); err != nil { - return err - } - } - return tw.Flush() -} - -// field is one row of a detail view. -type field struct { - Key string // Title Case, per the azd style guide - Value string -} - -// emitDetail writes a two-column key/value view, the shape `show` uses. -// -// Empty values are dropped rather than printed blank: a detail view is read to -// learn what a thing is, and a column of empty keys says only that the writer -// did not know which fields this kind has. -func emitDetail(w io.Writer, fields []field) error { - tw := tabwriter.NewWriter(w, 0, 0, 3, ' ', 0) - for _, f := range fields { - if f.Value == "" { - continue - } - if _, err := fmt.Fprintf(tw, "%s\t%s\n", f.Key, f.Value); err != nil { - return err - } - } - return tw.Flush() -} - -// requireFlag returns an error naming the missing flag, used when --no-prompt -// prevents asking for a required value. -func requireFlag(name string) error { - return fmt.Errorf("--%s is required (running with --no-prompt)", name) -} diff --git a/cli/azd/extensions/azure.ai.dataset/internal/cmd/output_test.go b/cli/azd/extensions/azure.ai.dataset/internal/cmd/output_test.go deleted file mode 100644 index 3e7fec6ab20..00000000000 --- a/cli/azd/extensions/azure.ai.dataset/internal/cmd/output_test.go +++ /dev/null @@ -1,177 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package cmd - -import ( - "bytes" - "encoding/json" - "os" - "path/filepath" - "strings" - "testing" - - "azureaidataset/internal/pkg/dataset_api" - - "github.com/spf13/cobra" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -// commandWithOutput builds a command carrying the -o flag the azd SDK root -// supplies at runtime, on the same flag set the production code reads. -func commandWithOutput(t *testing.T, value string) *cobra.Command { - t.Helper() - cmd := &cobra.Command{Use: "x"} - cmd.Flags().StringP("output", "o", "", "") - require.NoError(t, cmd.Flags().Set("output", value)) - cmd.SetOut(&bytes.Buffer{}) - return cmd -} - -// The flag selects machine-readable output, and a caller who types JSON in -// caps means the same thing as one who does not. -func TestOutputFormatAndIsJSON(t *testing.T) { - assert.True(t, isJSON(commandWithOutput(t, "json"))) - assert.True(t, isJSON(commandWithOutput(t, "JSON")), "the format is matched without regard to case") - assert.False(t, isJSON(commandWithOutput(t, "table"))) - assert.False(t, isJSON(commandWithOutput(t, ""))) - - assert.False(t, isJSON(nil), "a command with no flags is not JSON output") - assert.Empty(t, outputFormat(nil)) - - // A command that never declared -o must not panic on being asked. - assert.Empty(t, outputFormat(&cobra.Command{Use: "bare"})) -} - -// A nil slice encodes as null, which a caller iterating the result reads as a -// type error rather than as an empty list. -func TestEmitJSONListNormalizesNil(t *testing.T) { - var buf bytes.Buffer - var none []dataset_api.Dataset - require.NoError(t, emitJSONList(&buf, none)) - assert.Equal(t, "[]", strings.TrimSpace(buf.String())) - - buf.Reset() - require.NoError(t, emitJSONList(&buf, []dataset_api.Dataset{{Name: "a", Version: "1.0"}})) - var round []map[string]any - require.NoError(t, json.Unmarshal(buf.Bytes(), &round)) - require.Len(t, round, 1) - assert.Equal(t, "a", round[0]["name"]) -} - -// The list emits a bare array, not the envelope the service replied with: the -// envelopes disagree with each other and carry paging this extension does not -// follow. -func TestEmitJSONListDropsTheEnvelope(t *testing.T) { - cmd := commandWithOutput(t, "json") - var buf bytes.Buffer - cmd.SetOut(&buf) - - require.NoError(t, renderDatasets(cmd, &dataset_api.DatasetList{ - Value: []dataset_api.Dataset{{Name: "a", Version: "1.0"}}, - NextLink: "https://example/page2", - })) - - assert.True(t, strings.HasPrefix(strings.TrimSpace(buf.String()), "["), - "a list answers with an array") - assert.NotContains(t, buf.String(), "nextLink", - "paging this extension does not follow must not suggest there is more to fetch") -} - -// A list view is uppercase headers over a rule. The rule is what separates the -// header from the data at a glance, and it is what the sibling extensions print. -func TestRenderDatasetsTable(t *testing.T) { - cmd := commandWithOutput(t, "") - var buf bytes.Buffer - cmd.SetOut(&buf) - - require.NoError(t, renderDatasets(cmd, &dataset_api.DatasetList{Value: []dataset_api.Dataset{ - {Name: "golden", Version: "2.0", Format: "jsonl"}, - {Name: "smoke", Version: "1.0", Format: "jsonl"}, - }})) - - lines := strings.Split(strings.TrimRight(buf.String(), "\n"), "\n") - require.Len(t, lines, 4, "a header, its rule, and one line per dataset") - assert.Contains(t, lines[0], "NAME") - assert.Contains(t, lines[0], "VERSION") - assert.Contains(t, lines[0], "FORMAT") - assert.True(t, strings.HasPrefix(strings.TrimSpace(lines[1]), "----"), - "the rule under the header is the convention, got %q", lines[1]) - assert.Contains(t, lines[2], "golden") - assert.Contains(t, lines[3], "smoke") -} - -// An empty project has to say so. A bare header over nothing reads as output -// that got cut off. -func TestRenderDatasetsSaysWhenThereAreNone(t *testing.T) { - cmd := commandWithOutput(t, "") - var buf bytes.Buffer - cmd.SetOut(&buf) - - require.NoError(t, renderDatasets(cmd, &dataset_api.DatasetList{})) - assert.Contains(t, buf.String(), "No datasets found.") - assert.NotContains(t, buf.String(), "NAME") -} - -// A detail view is Title Case key/value, the shape `show` uses, and a blank -// value is dropped rather than printed as an empty column. -func TestEmitDetail(t *testing.T) { - var buf bytes.Buffer - require.NoError(t, emitDetail(&buf, []field{ - {"Name", "golden"}, - {"Version", "2.0"}, - {"Description", ""}, - {"Format", "jsonl"}, - })) - - lines := strings.Split(strings.TrimRight(buf.String(), "\n"), "\n") - require.Len(t, lines, 3, "the empty Description is dropped") - assert.True(t, strings.HasPrefix(lines[0], "Name")) - assert.Contains(t, lines[0], "golden") - assert.NotContains(t, buf.String(), "Description") -} - -// `show` returns one thing, so the spec's output conventions make it a detail -// view rather than the raw JSON it would otherwise be easiest to print. -func TestShowUsesADetailView(t *testing.T) { - body, err := os.ReadFile(filepath.Join(".", "dataset.go")) - require.NoError(t, err) - assert.Contains(t, string(body), "emitDetail", - "dataset show returns one thing, so dataset.go renders it as a detail view") -} - -// The message has to name the flag that would have supplied the value, since -// --no-prompt means nobody is there to be asked. -func TestRequireFlag(t *testing.T) { - err := requireFlag("name") - require.Error(t, err) - assert.Contains(t, err.Error(), "--name") - assert.Contains(t, err.Error(), "--no-prompt") -} - -// --from-file takes either the file or the directory holding it, because both -// are what a caller has to hand. Anything else is worth refusing by name. -func TestDatasetUploadDir(t *testing.T) { - dir := t.TempDir() - rows := filepath.Join(dir, "rows.jsonl") - require.NoError(t, os.WriteFile(rows, []byte("{\"query\":\"q\"}\n"), 0o600)) - - got, err := datasetUploadDir(rows) - require.NoError(t, err) - assert.Equal(t, dir, got, "a file resolves to the directory the upload scans") - - got, err = datasetUploadDir(dir) - require.NoError(t, err) - assert.Equal(t, dir, got) - - notJSONL := filepath.Join(dir, "rows.csv") - require.NoError(t, os.WriteFile(notJSONL, []byte("a,b\n"), 0o600)) - _, err = datasetUploadDir(notJSONL) - require.Error(t, err) - assert.Contains(t, err.Error(), ".jsonl") - - _, err = datasetUploadDir(filepath.Join(dir, "missing.jsonl")) - require.Error(t, err) - assert.Contains(t, err.Error(), "--from-file") -} diff --git a/cli/azd/extensions/azure.ai.dataset/internal/cmd/root.go b/cli/azd/extensions/azure.ai.dataset/internal/cmd/root.go deleted file mode 100644 index c53f715bb58..00000000000 --- a/cli/azd/extensions/azure.ai.dataset/internal/cmd/root.go +++ /dev/null @@ -1,62 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package cmd - -import ( - "fmt" - - "github.com/azure/azure-dev/cli/azd/pkg/azdext" - "github.com/fatih/color" - "github.com/spf13/cobra" -) - -// NewRootCommand builds the `azd ai dataset` command tree. -func NewRootCommand() *cobra.Command { - rootCmd, _ := azdext.NewExtensionRootCommand(azdext.ExtensionCommandOptions{ - Name: "dataset", - Use: "dataset [options]", - Short: fmt.Sprintf( - "Register and version Foundry datasets from your terminal. %s", - color.YellowString("(Beta)"), - ), - }) - rootCmd.SilenceUsage = true - rootCmd.SilenceErrors = true - rootCmd.CompletionOptions.DisableDefaultCmd = true - - // The data-plane clients trace requests through the standard logger, which - // Go writes to stderr, so it has to be silenced unless debug was asked for. - // The SDK's own hook is chained rather than replaced, and cobra ignores - // PersistentPreRun entirely once PersistentPreRunE is set. - sdkPreRun := rootCmd.PersistentPreRunE - rootCmd.PersistentPreRunE = func(cmd *cobra.Command, args []string) error { - if sdkPreRun != nil { - if err := sdkPreRun(cmd, args); err != nil { - return err - } - } - setupDebugLogging(cmd.Flags()) - return nil - } - - // Generation stays with `azure.ai.evaluations`: it writes the `datasets:` - // entry in evals/eval.yaml, which is that extension's file. - rootCmd.AddCommand( - newDatasetCreateCommand(), - newDatasetUpdateCommand(), - newDatasetListCommand(), - newDatasetShowCommand(), - newDatasetDeleteCommand(), - newDatasetVersionsCommand(), - ) - - // The manifest declares the `metadata` capability, which azd uses to - // discover this extension's command tree. Without the command registered, - // that discovery fails with "unknown command". - rootCmd.AddCommand(azdext.NewMetadataCommand("1.0", "azure.ai.dataset", func() *cobra.Command { - return rootCmd - })) - - return rootCmd -} diff --git a/cli/azd/extensions/azure.ai.dataset/internal/cmd/surface_test.go b/cli/azd/extensions/azure.ai.dataset/internal/cmd/surface_test.go deleted file mode 100644 index e8318e4580c..00000000000 --- a/cli/azd/extensions/azure.ai.dataset/internal/cmd/surface_test.go +++ /dev/null @@ -1,231 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package cmd - -import ( - "io/fs" - "os" - "path/filepath" - "regexp" - "strings" - "testing" - - "github.com/spf13/cobra" - "github.com/spf13/pflag" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -// walk visits every command in the tree, skipping the ones azd contributes. -func walk(t *testing.T, cmd *cobra.Command, path []string, visit func(string, *cobra.Command)) { - t.Helper() - for _, child := range cmd.Commands() { - name := strings.Fields(child.Use)[0] - switch name { - case "help", "completion", "listen", "metadata": - continue - } - full := append(append([]string{}, path...), name) - visit(strings.Join(full, " "), child) - walk(t, child, full, visit) - } -} - -// The command tree is the spec's `azd ai dataset` table. The CRUD groups moved -// here from azure.ai.evaluations; `generate` deliberately did not, because it -// writes the `datasets:` entry in evals/eval.yaml, which that extension owns. -func TestCommandTreeMatchesTheSpec(t *testing.T) { - want := []string{ - "create", - "delete", - "list", - "show", - "update", - "versions", - "versions list", - } - - var got []string - walk(t, NewRootCommand(), nil, func(path string, _ *cobra.Command) { - got = append(got, path) - }) - - assert.ElementsMatch(t, want, got, - "the command tree changed; update the spec's command table with it") -} - -// Flag names are shared vocabulary across the Foundry extensions. A command -// that invents its own spelling for something the others already name is the -// kind of difference nobody notices until a user types the one they learned -// somewhere else. -func TestFlagVocabularyIsShared(t *testing.T) { - forbidden := map[string]string{ - "--out-file": "--output-file", - "--out-dir": "--output-dir", - "--file": "--from-file", - "--out": "--output-file", - "--dir": "--output-dir", - } - - walk(t, NewRootCommand(), nil, func(path string, cmd *cobra.Command) { - cmd.LocalFlags().VisitAll(func(f *pflag.Flag) { - if want, bad := forbidden["--"+f.Name]; bad { - t.Errorf("%s declares --%s; use %s", path, f.Name, want) - } - }) - }) -} - -// `-o json` and `--no-prompt` come from the azd extension SDK's root command, -// so every command inherits them — until one declares a flag by the same name, -// which silently shadows the global. -func TestNoCommandShadowsAGlobalFlag(t *testing.T) { - global := []string{"output", "no-prompt", "environment", "cwd", "debug"} - - walk(t, NewRootCommand(), nil, func(path string, cmd *cobra.Command) { - for _, name := range global { - assert.Nilf(t, cmd.LocalFlags().Lookup(name), - "%s declares its own --%s, which shadows the global one", path, name) - } - }) -} - -// Every command here reaches the service, so the shared Foundry resolver has to -// be reachable from all of them. -func TestServiceCommandsTakeProjectEndpoint(t *testing.T) { - walk(t, NewRootCommand(), nil, func(path string, cmd *cobra.Command) { - if cmd.RunE == nil { - return - } - assert.NotNil(t, cmd.Flags().Lookup("project-endpoint"), - "%s reaches the service, so it must accept --project-endpoint", path) - }) -} - -// Generation stays with `azure.ai.evaluations`, so nothing here may grow a -// `--from`: a second generate would be a second place for the catalog write to -// go missing. -func TestNoGenerationCommandLandsHere(t *testing.T) { - walk(t, NewRootCommand(), nil, func(path string, cmd *cobra.Command) { - assert.Nilf(t, cmd.Flags().Lookup("from"), - "%s offers --from; generation belongs to azure.ai.evaluations", path) - assert.NotEqualf(t, "generate", cmd.Name(), - "%s is a generation command; it belongs to azure.ai.evaluations", path) - }) -} - -// Messages that tell a user what to run next have to name a command that -// exists. -// -// In the extension these commands moved from, three suggestions pointed at -// `azd ai dataset ...` while that namespace was served by nobody, and the -// check there matched on the wrong prefix so none of them failed. This -// extension's namespace is `ai.dataset`; anything it suggests under -// `azd ai dataset` has to resolve here, and a suggestion under another -// namespace is one it cannot make. -func TestSuggestedCommandsExist(t *testing.T) { - pattern := regexp.MustCompile("azd ai ([a-z][a-z0-9-]*(?: [a-z][a-z0-9-]*)*)") - - err := filepath.WalkDir("../..", func(path string, d fs.DirEntry, err error) error { - if err != nil { - return err - } - if d.IsDir() || !strings.HasSuffix(path, ".go") || strings.HasSuffix(path, "_test.go") { - return nil - } - body, err := os.ReadFile(path) - if err != nil { - return err - } - for _, line := range strings.Split(string(body), "\n") { - if strings.HasPrefix(strings.TrimSpace(line), "//") { - continue - } - for _, m := range pattern.FindAllStringSubmatch(line, -1) { - words := strings.Fields(m[1]) - if len(words) == 0 { - continue - } - // A suggestion pointing at a sibling extension is that - // extension's contract, not this one's, and cannot be resolved - // from here. Listed rather than wildcarded so a typo in a - // namespace still fails. - if siblingNamespaces[words[0]] { - continue - } - assert.Equalf(t, "dataset", words[0], - "%s suggests `azd ai %s`, which is neither this extension's "+ - "namespace nor a sibling it knows about", path, m[1]) - words = words[1:] - - // Trim trailing prose: "job show" is a command, "job show and - // then" is a sentence that begins with one. - for len(words) > 0 { - if resolved, _, e := NewRootCommand().Find(words); e == nil { - if strings.Fields(resolved.Use)[0] == words[len(words)-1] { - break - } - } - words = words[:len(words)-1] - } - assert.NotEmptyf(t, words, - "%s suggests `azd ai %s`, which is not a command", path, m[1]) - } - } - return nil - }) - require.NoError(t, err) -} - -// A message pointing at `azd ai eval dataset ...` is almost always the copy -// these commands came from rather than a deliberate cross-extension pointer. -// `dataset` is this extension's own namespace, so telling a user to run the -// eval extension's version of a command it serves itself sends them somewhere -// they may not have installed. -// -// Nothing here may suggest one any more: `generate` was the only command with a -// reason to, and it stayed with azure.ai.evaluations. -func TestNoStaleEvalDatasetSuggestions(t *testing.T) { - allowed := map[string]bool{} - - err := filepath.WalkDir("../..", func(path string, d fs.DirEntry, err error) error { - if err != nil { - return err - } - if d.IsDir() || !strings.HasSuffix(path, ".go") || strings.HasSuffix(path, "_test.go") { - return nil - } - body, err := os.ReadFile(path) - if err != nil { - return err - } - pattern := regexp.MustCompile(`azd ai eval dataset [a-z][a-z0-9-]*`) - for i, line := range strings.Split(string(body), "\n") { - if strings.HasPrefix(strings.TrimSpace(line), "//") { - continue - } - for _, m := range pattern.FindAllString(line, -1) { - assert.Truef(t, allowed[m], - "%s:%d suggests `%s`; this extension serves that command as "+ - "`azd ai dataset ...`", path, i+1, m) - } - } - return nil - }) - require.NoError(t, err) -} - -// siblingNamespaces are the other Foundry extensions this one points users at. -var siblingNamespaces = map[string]bool{ - "project": true, // `azd ai project set` owns the shared endpoint context -} - -func find(t *testing.T, path string) *cobra.Command { - t.Helper() - cmd, _, err := NewRootCommand().Find(strings.Fields(path)) - require.NoError(t, err, "no such command: %s", path) - require.Equal(t, strings.Fields(path)[len(strings.Fields(path))-1], - strings.Fields(cmd.Use)[0], "resolved the wrong command for %s", path) - return cmd -} diff --git a/cli/azd/extensions/azure.ai.dataset/internal/cmd/table_test.go b/cli/azd/extensions/azure.ai.dataset/internal/cmd/table_test.go deleted file mode 100644 index 92b18ee230a..00000000000 --- a/cli/azd/extensions/azure.ai.dataset/internal/cmd/table_test.go +++ /dev/null @@ -1,70 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package cmd - -import ( - "bytes" - "strings" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -// A list view is uppercase headers over a rule, per the spec's output -// conventions and the sibling extension it cites. The rule is what separates -// the header from the data at a glance, and every `list` command was printing -// the header straight onto the first row. -func TestEmitTableWritesTheRule(t *testing.T) { - var buf bytes.Buffer - require.NoError(t, emitTable(&buf, - []string{"NAME", "VERSION"}, - [][]string{{"support-regression", "3"}, {"nightly", "1"}})) - - lines := strings.Split(strings.TrimRight(buf.String(), "\n"), "\n") - require.Len(t, lines, 4, "a header, its rule, and one line per row") - - assert.Contains(t, lines[0], "NAME") - assert.Contains(t, lines[0], "VERSION") - - // Dashes as wide as the header they sit under, which is what makes the - // rule line up once tabwriter has padded the columns. - assert.Contains(t, lines[1], strings.Repeat("-", len("NAME"))) - assert.Contains(t, lines[1], strings.Repeat("-", len("VERSION"))) - assert.Empty(t, strings.Trim(lines[1], "- "), - "the rule carries nothing but dashes and padding") - - assert.Contains(t, lines[2], "support-regression") - assert.Contains(t, lines[3], "nightly") -} - -// The columns line up: the rule is padded to the same widths as the header, so -// a wide value in the first row does not leave the rule short. -func TestEmitTableRuleAlignsWithTheHeader(t *testing.T) { - var buf bytes.Buffer - require.NoError(t, emitTable(&buf, - []string{"NAME", "STATUS"}, - [][]string{{"a-very-much-longer-value-than-the-header", "completed"}})) - - lines := strings.Split(strings.TrimRight(buf.String(), "\n"), "\n") - require.Len(t, lines, 3) - - // tabwriter pads every line in a column to the same width, so the header - // and its rule start their second column at the same offset. - assert.Equal(t, - strings.Index(lines[0], "STATUS"), - strings.Index(lines[1], "------"), - "the rule has to sit under the header it belongs to") -} - -// A listing with nothing in it still prints the header and rule: a caller -// seeing no output cannot tell an empty list from a command that failed to -// render. -func TestEmitTableWithNoRows(t *testing.T) { - var buf bytes.Buffer - require.NoError(t, emitTable(&buf, []string{"NAME", "VERSION"}, nil)) - - lines := strings.Split(strings.TrimRight(buf.String(), "\n"), "\n") - assert.Len(t, lines, 2) -} diff --git a/cli/azd/extensions/azure.ai.dataset/internal/exterrors/codes.go b/cli/azd/extensions/azure.ai.dataset/internal/exterrors/codes.go deleted file mode 100644 index 58fc300148c..00000000000 --- a/cli/azd/extensions/azure.ai.dataset/internal/exterrors/codes.go +++ /dev/null @@ -1,71 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package exterrors - -// Error codes for user cancellation. -const ( - CodeCancelled = "cancelled" -) - -// Error codes for validation failures (user input, manifests, flags). -const ( - CodeInvalidParameter = "invalid_parameter" - CodeInvalidPositionalArg = "invalid_positional_arg" -) - -// Error codes for dependency failures (missing resources, services, env values). -const ( - CodeAzdClientFailed = "azd_client_failed" - CodeMissingProjectEndpoint = "missing_project_endpoint" -) - -// Error codes for auth failures. -const ( - CodeNotLoggedIn = "not_logged_in" - CodeLoginExpired = "login_expired" - CodeAuthFailed = "auth_failed" -) - -// Error codes for toolbox operations. -const ( - CodeToolboxNotFound = "toolbox_not_found" - CodeToolboxVersionNotFound = "toolbox_version_not_found" - CodeInvalidToolboxName = "invalid_toolbox_name" - CodeMissingUpdateField = "missing_update_field" - CodeDefaultVersionDelete = "default_version_delete" - CodeOnlyVersionDelete = "only_version_delete" - CodeMissingForceFlag = "missing_force_flag" - CodeUnsupportedConnectionCategory = "unsupported_connection_category" - CodeMissingIndex = "missing_index" - CodeUnsupportedIndexFlag = "unsupported_index_flag" - CodeMissingInstanceName = "missing_instance_name" - CodeUnsupportedInstanceNameFlag = "unsupported_instance_name_flag" - CodeInvalidSkillName = "invalid_skill_name" - CodeInvalidSkillSpec = "invalid_skill_spec" - CodeDuplicateSkill = "duplicate_skill" - CodeSkillNotInToolbox = "skill_not_in_toolbox" - CodeSkillAlreadyAttached = "skill_already_attached" - CodeDuplicateConnection = "duplicate_connection" - CodeDuplicateToolName = "duplicate_tool_name" - CodeMissingToolType = "missing_tool_type" - CodeConnectionNotFound = "connection_not_found" - CodeConnectionNotInToolbox = "connection_not_in_toolbox" - CodeConnectionMissingTarget = "connection_missing_target" - CodeLastToolRemoval = "last_tool_removal" - CodePendingToolboxStoreFailed = "pending_toolbox_store_failed" -) - -// Operation names for [ServiceFromAzure] errors. -// These are prefixed to the Azure error code (e.g., "get_toolbox.NotFound"). -const ( - OpCreateToolboxVersion = "create_toolbox_version" - OpGetToolbox = "get_toolbox" - OpDeleteToolbox = "delete_toolbox" - OpDeleteToolboxVersion = "delete_toolbox_version" - OpSetDefaultVersion = "set_default_version" - OpListToolboxes = "list_toolboxes" - OpGetToolboxVersion = "get_toolbox_version" - OpListToolboxVersions = "list_toolbox_versions" - OpResolveProjectConnection = "resolve_project_connection" -) diff --git a/cli/azd/extensions/azure.ai.dataset/internal/exterrors/errors.go b/cli/azd/extensions/azure.ai.dataset/internal/exterrors/errors.go deleted file mode 100644 index 4e3c02b6731..00000000000 --- a/cli/azd/extensions/azure.ai.dataset/internal/exterrors/errors.go +++ /dev/null @@ -1,155 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -// Package exterrors provides structured error helpers for the azure.ai.toolboxes -// extension. -// -// Use plain Go errors until the current code can confidently choose a final -// category, code, and suggestion. At that point, create a structured error with -// one of the helpers in this package or with [ServiceFromAzure] for Azure SDK -// failures. -// -// Once an error is structured, usually return it unchanged. Avoid wrapping a -// structured error with [fmt.Errorf] and %w for extra context: azd serializes -// the structured error's own message and metadata, not the outer wrapper text. -package exterrors - -import ( - "context" - "errors" - "fmt" - - "github.com/Azure/azure-sdk-for-go/sdk/azcore" - "github.com/azure/azure-dev/cli/azd/pkg/azdext" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/status" -) - -// --------------------------------------------------------------------------- -// Structured error factories -// --------------------------------------------------------------------------- - -// Validation returns a validation [azdext.LocalError] for user input / flag errors. -func Validation(code, message, suggestion string) error { - return &azdext.LocalError{ - Message: message, - Code: code, - Category: azdext.LocalErrorCategoryValidation, - Suggestion: suggestion, - } -} - -// Dependency returns a dependency [azdext.LocalError] for missing resources or services. -func Dependency(code, message, suggestion string) error { - return &azdext.LocalError{ - Message: message, - Code: code, - Category: azdext.LocalErrorCategoryDependency, - Suggestion: suggestion, - } -} - -// Auth returns an auth [azdext.LocalError] for authentication/authorization failures. -func Auth(code, message, suggestion string) error { - return &azdext.LocalError{ - Message: message, - Code: code, - Category: azdext.LocalErrorCategoryAuth, - Suggestion: suggestion, - } -} - -// User returns a user-action [azdext.LocalError] (e.g. cancellation). No suggestion. -func User(code, message string) error { - return &azdext.LocalError{ - Message: message, - Code: code, - Category: azdext.LocalErrorCategoryUser, - } -} - -// Internal returns an internal [azdext.LocalError] for unexpected extension failures. -func Internal(code, message string) error { - return &azdext.LocalError{ - Message: message, - Code: code, - Category: azdext.LocalErrorCategoryInternal, - } -} - -// Cancelled returns a user cancellation error. -func Cancelled(message string) error { - return User(CodeCancelled, message) -} - -// --------------------------------------------------------------------------- -// Azure error converters -// --------------------------------------------------------------------------- - -// ServiceFromAzure wraps an [azcore.ResponseError] into an [azdext.ServiceError] -// with operation context. If the error is not an azcore.ResponseError, it -// returns a generic internal [azdext.LocalError]. -func ServiceFromAzure(err error, operation string) error { - var respErr *azcore.ResponseError - if errors.As(err, &respErr) { - serviceName := "" - if respErr.RawResponse != nil && respErr.RawResponse.Request != nil { - serviceName = respErr.RawResponse.Request.Host - } - code := respErr.ErrorCode - if code == "" { - code = fmt.Sprintf("%d", respErr.StatusCode) - } - return &azdext.ServiceError{ - Message: fmt.Sprintf("%s: %s", operation, respErr.Error()), - ErrorCode: fmt.Sprintf("%s.%s", operation, code), - StatusCode: respErr.StatusCode, - ServiceName: serviceName, - } - } - if IsCancellation(err) { - return Cancelled(fmt.Sprintf("%s was cancelled", operation)) - } - return Internal(operation, fmt.Sprintf("%s: %s", operation, err.Error())) -} - -// FromPrompt wraps a gRPC error from an azd host Prompt call into a structured -// error. Auth errors (Unauthenticated) are classified as Auth errors with a -// re-auth suggestion; cancellations as User cancellations; other errors are -// returned wrapped with the provided context message. -func FromPrompt(err error, contextMsg string) error { - if err == nil { - return nil - } - - if IsCancellation(err) { - return Cancelled(contextMsg) - } - - st, ok := status.FromError(err) - if ok && st.Code() == codes.Unauthenticated { - return Auth( - CodeAuthFailed, - fmt.Sprintf("%s: %s", contextMsg, st.Message()), - "run `azd auth login` to authenticate", - ) - } - - return fmt.Errorf("%s: %w", contextMsg, err) -} - -// --------------------------------------------------------------------------- -// Helpers -// --------------------------------------------------------------------------- - -// IsCancellation reports whether err represents user cancellation -// ([context.Canceled] or gRPC [codes.Canceled]). -func IsCancellation(err error) bool { - if errors.Is(err, context.Canceled) { - return true - } - if st, ok := status.FromError(err); ok && st.Code() == codes.Canceled { - return true - } - return false -} diff --git a/cli/azd/extensions/azure.ai.dataset/internal/foundry/projectctx/resolver.go b/cli/azd/extensions/azure.ai.dataset/internal/foundry/projectctx/resolver.go deleted file mode 100644 index 3c6c3129a50..00000000000 --- a/cli/azd/extensions/azure.ai.dataset/internal/foundry/projectctx/resolver.go +++ /dev/null @@ -1,152 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package projectctx - -import ( - "context" - "errors" - "os" - - "github.com/azure/azure-dev/cli/azd/pkg/azdext" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/status" -) - -// ReadAzdHostedSourcesFunc is a package-level seam so tests can stub the -// daemon-backed lookup without spinning up a real azd gRPC server. -var ReadAzdHostedSourcesFunc = readAzdHostedSources - -// readAzdHostedSources dials the azd daemon (if reachable) and reads both the -// active environment's project endpoint and the global-config project context -// in a single client lifetime. The active-env read prefers -// FOUNDRY_PROJECT_ENDPOINT and falls back to AZURE_AI_PROJECT_ENDPOINT (the key -// `azd ai agent init` / `azd add` persist). Errors talking to the daemon are -// returned only for non-Unavailable cases on the config read — Unavailable is -// treated as "no daemon" and the caller falls through to subsequent levels. -func readAzdHostedSources(ctx context.Context) (AzdHostedSources, error) { - var out AzdHostedSources - - azdClient, err := azdext.NewAzdClient() - if err != nil { - // No azd client at all => no hosted sources, not an error. - return out, nil - } - defer azdClient.Close() - - if envResp, err := azdClient.Environment().GetCurrent( - ctx, &azdext.EmptyRequest{}, - ); err == nil { - for _, key := range []string{foundryEnvKey, azureAiEnvKey} { - envVal, valErr := azdClient.Environment().GetValue(ctx, &azdext.GetEnvRequest{ - EnvName: envResp.Environment.Name, - Key: key, - }) - if valErr == nil && envVal.Value != "" { - out.EnvValue = envVal.Value - out.EnvName = envResp.Environment.Name - break - } - } - } - - state, found, cfgErr := getProjectContext(ctx, azdClient) - if cfgErr != nil { - // A gRPC Unavailable code means the azd daemon is not reachable; - // treat it the same as azdClient creation failing and fall through. - // Any other error (e.g. parse failure) is a hard error. - if !containsGRPCCode(cfgErr, codes.Unavailable) { - return out, cfgErr - } - } else { - out.CfgState = state - out.CfgFound = found - } - - return out, nil -} - -// containsGRPCCode walks the error chain looking for a gRPC status with the -// specified code. fmt.Errorf("%w", ...) wraps errors without forwarding the -// GRPCStatus() method, so we must unwrap manually. -// -// Note: only follows errors.Unwrap chains; errors.Join multi-wraps are not traversed. -func containsGRPCCode(err error, code codes.Code) bool { - for ; err != nil; err = errors.Unwrap(err) { - if st, ok := status.FromError(err); ok && st.Code() == code { - return true - } - } - return false -} - -// Resolve resolves a Foundry project endpoint using the 5-level cascade: -// -// 1. --project-endpoint flag -// 2. Active azd env value (FOUNDRY_PROJECT_ENDPOINT, then AZURE_AI_PROJECT_ENDPOINT) -// 3. Global config: extensions.ai-agents.project.context.endpoint (read-only; -// owned by azure.ai.agents) -// 4. Host environment variable (FOUNDRY_PROJECT_ENDPOINT, then AZURE_AI_PROJECT_ENDPOINT) -// 5. Structured error with actionable suggestion -// -// Invalid values at any level produce a hard validation error (no silent fallback). -func Resolve(ctx context.Context, opts ResolveOpts) (*Resolved, error) { - // Level 1: explicit flag. - if opts.FlagValue != "" { - normalized, _, err := Validate(opts.FlagValue) - if err != nil { - return nil, err - } - return &Resolved{Endpoint: normalized, Source: SourceFlag}, nil - } - - // Levels 2 + 3: azd-hosted sources (active env, then global config). - sources, err := ReadAzdHostedSourcesFunc(ctx) - if err != nil { - return nil, err - } - - // Level 2: active azd environment's FOUNDRY_PROJECT_ENDPOINT (with the - // AZURE_AI_PROJECT_ENDPOINT fallback applied in readAzdHostedSources). - if sources.EnvValue != "" { - normalized, _, err := Validate(sources.EnvValue) - if err != nil { - return nil, err - } - return &Resolved{ - Endpoint: normalized, - Source: SourceAzdEnv, - AzdEnvName: sources.EnvName, - }, nil - } - - // Level 3: global config (~/.azd/config.json). - if sources.CfgFound && sources.CfgState.Endpoint != "" { - normalized, _, err := Validate(sources.CfgState.Endpoint) - if err != nil { - return nil, err - } - return &Resolved{ - Endpoint: normalized, - Source: SourceGlobalConfig, - SetAt: sources.CfgState.SetAt, - }, nil - } - - // Level 4: host environment variable (FOUNDRY_PROJECT_ENDPOINT, then the - // AZURE_AI_PROJECT_ENDPOINT fallback). - for _, key := range []string{foundryEnvKey, azureAiEnvKey} { - envVal := os.Getenv(key) - if envVal == "" { - continue - } - normalized, _, err := Validate(envVal) - if err != nil { - return nil, err - } - return &Resolved{Endpoint: normalized, Source: SourceFoundryEnv}, nil - } - - // Level 5: structured error. - return nil, NoEndpointError() -} diff --git a/cli/azd/extensions/azure.ai.dataset/internal/foundry/projectctx/resolver_test.go b/cli/azd/extensions/azure.ai.dataset/internal/foundry/projectctx/resolver_test.go deleted file mode 100644 index f21bbbd60fe..00000000000 --- a/cli/azd/extensions/azure.ai.dataset/internal/foundry/projectctx/resolver_test.go +++ /dev/null @@ -1,259 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package projectctx - -import ( - "context" - "errors" - "testing" - - "azureaidataset/internal/exterrors" - - "github.com/azure/azure-dev/cli/azd/pkg/azdext" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -// withHostedSources installs a stub for ReadAzdHostedSourcesFunc for the -// duration of the test and restores the production value on cleanup. Tests -// using this MUST NOT run in parallel because the seam is a package-level var. -func withHostedSources(t *testing.T, sources AzdHostedSources, err error) { - t.Helper() - orig := ReadAzdHostedSourcesFunc - ReadAzdHostedSourcesFunc = func(context.Context) (AzdHostedSources, error) { - return sources, err - } - t.Cleanup(func() { ReadAzdHostedSourcesFunc = orig }) -} - -// isolateFromAzdDaemon installs an empty hosted-sources stub and clears -// AZD_SERVER so any code path that bypasses the seam cannot reach a real -// daemon. After calling this, the resolver only sees the flag and the -// FOUNDRY_PROJECT_ENDPOINT / AZURE_AI_PROJECT_ENDPOINT host env vars. -func isolateFromAzdDaemon(t *testing.T) { - t.Helper() - t.Setenv("AZD_SERVER", "") - withHostedSources(t, AzdHostedSources{}, nil) -} - -func TestResolve_FlagWins(t *testing.T) { - // Even with FOUNDRY_PROJECT_ENDPOINT and azd-hosted sources set, the flag wins. - t.Setenv("FOUNDRY_PROJECT_ENDPOINT", "https://env.services.ai.azure.com/api/projects/env-proj") - withHostedSources(t, AzdHostedSources{ - EnvValue: "https://azdenv.services.ai.azure.com/api/projects/p", - EnvName: "dev", - }, nil) - - result, err := Resolve(t.Context(), ResolveOpts{ - FlagValue: "https://flag.services.ai.azure.com/api/projects/flag-proj", - }) - require.NoError(t, err) - assert.Equal(t, "https://flag.services.ai.azure.com/api/projects/flag-proj", result.Endpoint) - assert.Equal(t, SourceFlag, result.Source) -} - -func TestResolve_AzdEnvWinsOverConfigAndFoundryEnv(t *testing.T) { - // EnvValue here stands in for whichever active-env key readAzdHostedSources - // resolved (FOUNDRY_PROJECT_ENDPOINT, or the AZURE_AI_PROJECT_ENDPOINT - // fallback); either way level 2 wins over global config and the host env. - t.Setenv("FOUNDRY_PROJECT_ENDPOINT", "https://foundry.services.ai.azure.com/api/projects/p") - withHostedSources(t, AzdHostedSources{ - EnvValue: " HTTPS://Azdenv.Services.AI.Azure.com/api/projects/p/ ", - EnvName: "dev", - CfgState: State{ - Endpoint: "https://cfg.services.ai.azure.com/api/projects/p", - SetAt: "2025-01-01T00:00:00Z", - }, - CfgFound: true, - }, nil) - - result, err := Resolve(t.Context(), ResolveOpts{}) - require.NoError(t, err) - assert.Equal(t, "https://azdenv.services.ai.azure.com/api/projects/p", result.Endpoint) - assert.Equal(t, SourceAzdEnv, result.Source) - assert.Equal(t, "dev", result.AzdEnvName) -} - -func TestResolve_AzdEnvInvalidIsHardError(t *testing.T) { - // Level 2 invalid values are hard errors (no silent fallback to lower levels). - t.Setenv("FOUNDRY_PROJECT_ENDPOINT", "https://foundry.services.ai.azure.com/api/projects/p") - withHostedSources(t, AzdHostedSources{ - EnvValue: "http://not-https.services.ai.azure.com/api/projects/p", - EnvName: "dev", - }, nil) - - _, err := Resolve(t.Context(), ResolveOpts{}) - require.Error(t, err) - - var localErr *azdext.LocalError - require.ErrorAs(t, err, &localErr) - assert.Contains(t, localErr.Message, "https") -} - -func TestResolve_GlobalConfigWinsOverFoundryEnv(t *testing.T) { - t.Setenv("FOUNDRY_PROJECT_ENDPOINT", "https://foundry.services.ai.azure.com/api/projects/p") - withHostedSources(t, AzdHostedSources{ - CfgState: State{ - Endpoint: " HTTPS://Cfg.Services.AI.Azure.com/api/projects/p/ ", - SetAt: "2025-01-02T03:04:05Z", - }, - CfgFound: true, - }, nil) - - result, err := Resolve(t.Context(), ResolveOpts{}) - require.NoError(t, err) - assert.Equal(t, "https://cfg.services.ai.azure.com/api/projects/p", result.Endpoint) - assert.Equal(t, SourceGlobalConfig, result.Source) - assert.Equal(t, "2025-01-02T03:04:05Z", result.SetAt) -} - -func TestResolve_GlobalConfigInvalidIsHardError(t *testing.T) { - t.Setenv("FOUNDRY_PROJECT_ENDPOINT", "https://foundry.services.ai.azure.com/api/projects/p") - withHostedSources(t, AzdHostedSources{ - CfgState: State{ - Endpoint: "http://not-https.services.ai.azure.com/api/projects/p", - SetAt: "2025-01-02T03:04:05Z", - }, - CfgFound: true, - }, nil) - - _, err := Resolve(t.Context(), ResolveOpts{}) - require.Error(t, err) - - var localErr *azdext.LocalError - require.ErrorAs(t, err, &localErr) - assert.Contains(t, localErr.Message, "https") -} - -func TestResolve_HostedSourcesErrorPropagates(t *testing.T) { - // Non-recoverable errors from the hosted-source lookup must be surfaced - // and must not silently fall through to level 4. - t.Setenv("FOUNDRY_PROJECT_ENDPOINT", "https://foundry.services.ai.azure.com/api/projects/p") - sentinel := errors.New("boom") - withHostedSources(t, AzdHostedSources{}, sentinel) - - _, err := Resolve(t.Context(), ResolveOpts{}) - require.ErrorIs(t, err, sentinel) -} - -func TestResolve_FoundryEnvFallback(t *testing.T) { - isolateFromAzdDaemon(t) - t.Setenv("FOUNDRY_PROJECT_ENDPOINT", "https://env.services.ai.azure.com/api/projects/env-proj") - - result, err := Resolve(t.Context(), ResolveOpts{}) - require.NoError(t, err) - assert.Equal(t, "https://env.services.ai.azure.com/api/projects/env-proj", result.Endpoint) - assert.Equal(t, SourceFoundryEnv, result.Source) -} - -func TestResolve_AzureAiHostEnvFallback(t *testing.T) { - // When FOUNDRY_PROJECT_ENDPOINT is unset, the resolver falls back to the - // AZURE_AI_PROJECT_ENDPOINT host env var (the key azd ai agent init / azd - // add persist). See https://github.com/Azure/azure-dev/issues/8688. - isolateFromAzdDaemon(t) - t.Setenv("FOUNDRY_PROJECT_ENDPOINT", "") - t.Setenv("AZURE_AI_PROJECT_ENDPOINT", "https://azureai.services.ai.azure.com/api/projects/p") - - result, err := Resolve(t.Context(), ResolveOpts{}) - require.NoError(t, err) - assert.Equal(t, "https://azureai.services.ai.azure.com/api/projects/p", result.Endpoint) - assert.Equal(t, SourceFoundryEnv, result.Source) -} - -func TestResolve_FoundryHostEnvWinsOverAzureAi(t *testing.T) { - // With both host env vars set, FOUNDRY_PROJECT_ENDPOINT takes precedence. - isolateFromAzdDaemon(t) - t.Setenv("FOUNDRY_PROJECT_ENDPOINT", "https://foundry.services.ai.azure.com/api/projects/f") - t.Setenv("AZURE_AI_PROJECT_ENDPOINT", "https://azureai.services.ai.azure.com/api/projects/a") - - result, err := Resolve(t.Context(), ResolveOpts{}) - require.NoError(t, err) - assert.Equal(t, "https://foundry.services.ai.azure.com/api/projects/f", result.Endpoint) - assert.Equal(t, SourceFoundryEnv, result.Source) -} - -func TestResolve_FoundryEnvNormalized(t *testing.T) { - isolateFromAzdDaemon(t) - t.Setenv("FOUNDRY_PROJECT_ENDPOINT", " https://X.SERVICES.AI.AZURE.COM/api/projects/p/ ") - - result, err := Resolve(t.Context(), ResolveOpts{}) - require.NoError(t, err) - assert.Equal(t, "https://x.services.ai.azure.com/api/projects/p", result.Endpoint) - assert.Equal(t, SourceFoundryEnv, result.Source) -} - -func TestResolve_InvalidFlagRejected(t *testing.T) { - isolateFromAzdDaemon(t) - - _, err := Resolve(t.Context(), ResolveOpts{ - FlagValue: "http://not-https.services.ai.azure.com/api/projects/p", - }) - require.Error(t, err) - - var localErr *azdext.LocalError - require.ErrorAs(t, err, &localErr) - assert.Contains(t, localErr.Message, "https") -} - -func TestResolve_InvalidFoundryEnvRejected(t *testing.T) { - isolateFromAzdDaemon(t) - t.Setenv("FOUNDRY_PROJECT_ENDPOINT", "http://bad.services.ai.azure.com/api/projects/p") - - _, err := Resolve(t.Context(), ResolveOpts{}) - require.Error(t, err) - - var localErr *azdext.LocalError - require.ErrorAs(t, err, &localErr) - assert.Contains(t, localErr.Message, "https") -} - -func TestResolve_InvalidAzureAiHostEnvRejected(t *testing.T) { - // An invalid AZURE_AI_PROJECT_ENDPOINT fallback is a hard error, not a - // silent skip to level 5. - isolateFromAzdDaemon(t) - t.Setenv("FOUNDRY_PROJECT_ENDPOINT", "") - t.Setenv("AZURE_AI_PROJECT_ENDPOINT", "http://not-https.services.ai.azure.com/api/projects/p") - - _, err := Resolve(t.Context(), ResolveOpts{}) - require.Error(t, err) - - var localErr *azdext.LocalError - require.ErrorAs(t, err, &localErr) - assert.Contains(t, localErr.Message, "https") -} - -func TestResolve_NothingResolvable(t *testing.T) { - isolateFromAzdDaemon(t) - t.Setenv("FOUNDRY_PROJECT_ENDPOINT", "") - t.Setenv("AZURE_AI_PROJECT_ENDPOINT", "") - - _, err := Resolve(t.Context(), ResolveOpts{}) - require.Error(t, err) - - var localErr *azdext.LocalError - require.ErrorAs(t, err, &localErr) - assert.Equal(t, exterrors.CodeMissingProjectEndpoint, localErr.Code) - assert.Equal(t, azdext.LocalErrorCategoryDependency, localErr.Category) -} - -func TestResolve_CfgFoundButEndpointEmptyFallsThrough(t *testing.T) { - // CfgFound=true with Endpoint="" must not short-circuit; the resolver - // should continue to level 4 (FOUNDRY_PROJECT_ENDPOINT). - t.Setenv("FOUNDRY_PROJECT_ENDPOINT", "https://env.services.ai.azure.com/api/projects/p") - withHostedSources(t, AzdHostedSources{ - CfgState: State{Endpoint: "", SetAt: "2025-01-01T00:00:00Z"}, - CfgFound: true, - }, nil) - - result, err := Resolve(t.Context(), ResolveOpts{}) - require.NoError(t, err) - assert.Equal(t, "https://env.services.ai.azure.com/api/projects/p", result.Endpoint) - assert.Equal(t, SourceFoundryEnv, result.Source) -} - -func TestContainsGRPCCode_NonGRPCErrorReturnsFalse(t *testing.T) { - t.Parallel() - assert.False(t, containsGRPCCode(errors.New("plain"), 0)) - assert.False(t, containsGRPCCode(nil, 0)) -} diff --git a/cli/azd/extensions/azure.ai.dataset/internal/foundry/projectctx/store.go b/cli/azd/extensions/azure.ai.dataset/internal/foundry/projectctx/store.go deleted file mode 100644 index 433e9ab8d41..00000000000 --- a/cli/azd/extensions/azure.ai.dataset/internal/foundry/projectctx/store.go +++ /dev/null @@ -1,40 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package projectctx - -import ( - "context" - "fmt" - - "github.com/azure/azure-dev/cli/azd/pkg/azdext" -) - -// projectContextConfigPath is the read-only UserConfig path for the persisted -// project context owned by azure.ai.agents. The toolboxes extension reads this -// key but never writes it (§ 6 of the design spec). -const projectContextConfigPath = "extensions.ai-agents.project.context" - -// getProjectContext reads the persisted project context from global config. -// Returns (state, true, nil) when present, (zero, false, nil) when absent. -func getProjectContext( - ctx context.Context, azdClient *azdext.AzdClient, -) (State, bool, error) { - ch, err := azdext.NewConfigHelper(azdClient) - if err != nil { - return State{}, false, fmt.Errorf("getProjectContext: %w", err) - } - - var state State - found, err := ch.GetUserJSON(ctx, projectContextConfigPath, &state) - if err != nil { - return State{}, false, - fmt.Errorf("getProjectContext: failed to read config: %w", err) - } - - if !found || state.Endpoint == "" { - return State{}, false, nil - } - - return state, true, nil -} diff --git a/cli/azd/extensions/azure.ai.dataset/internal/foundry/projectctx/types.go b/cli/azd/extensions/azure.ai.dataset/internal/foundry/projectctx/types.go deleted file mode 100644 index 93bf43f3780..00000000000 --- a/cli/azd/extensions/azure.ai.dataset/internal/foundry/projectctx/types.go +++ /dev/null @@ -1,81 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -// Package projectctx encapsulates the Foundry project endpoint cascade and -// validation shared by every Foundry-extension command tree. -// -// This is the toolboxes-extension copy of the agent_context.go / project_endpoint.go / -// project_context_store.go logic in azure.ai.agents (see § 3.2 of the toolbox -// design spec). Semantics match the agents original verbatim; identifiers are -// exported because they cross the package boundary in this layout. -package projectctx - -const ( - // foundryEnvKey is the canonical project-endpoint key. It is read both from - // the active azd environment (level 2) and as a host environment variable - // (level 4). - foundryEnvKey = "FOUNDRY_PROJECT_ENDPOINT" - // azureAiEnvKey is the legacy/sibling project-endpoint key written by - // `azd ai agent init` and `azd add` (Bicep output). It is read as a fallback - // after foundryEnvKey at both the active-azd-env and host-env levels so the - // hosted-agent + toolbox workflow resolves without an extra manual step. - // See https://github.com/Azure/azure-dev/issues/8688. - azureAiEnvKey = "AZURE_AI_PROJECT_ENDPOINT" -) - -// EndpointSource identifies where a resolved project endpoint came from. -type EndpointSource string - -const ( - // SourceFlag means the endpoint came from the --project-endpoint flag. - SourceFlag EndpointSource = "flag" - // SourceAzdEnv means the endpoint came from the active azd environment's - // FOUNDRY_PROJECT_ENDPOINT (or, as a fallback, AZURE_AI_PROJECT_ENDPOINT) value. - SourceAzdEnv EndpointSource = "azdEnv" - // SourceGlobalConfig means the endpoint came from ~/.azd/config.json - // (extensions.ai-agents.project.context.endpoint — owned by azure.ai.agents - // and shared read-only with sibling extensions). - SourceGlobalConfig EndpointSource = "globalConfig" - // SourceFoundryEnv means the endpoint came from the FOUNDRY_PROJECT_ENDPOINT - // (or, as a fallback, AZURE_AI_PROJECT_ENDPOINT) host environment variable. - SourceFoundryEnv EndpointSource = "foundryEnv" -) - -// ResolveOpts controls the 5-level endpoint resolution cascade. -type ResolveOpts struct { - // FlagValue is the value of the --project-endpoint flag (level 1). - // Empty means the flag was not provided. - FlagValue string -} - -// Resolved holds the result of Resolve. -type Resolved struct { - Endpoint string - Source EndpointSource - AzdEnvName string - SetAt string // RFC3339 timestamp; only meaningful when Source == SourceGlobalConfig -} - -// AzdHostedSources holds the values the resolver reads from azd-managed -// sources (active env + ~/.azd/config.json). Returned as a single struct so -// tests can stub the whole lookup via ReadAzdHostedSourcesFunc. -type AzdHostedSources struct { - // EnvValue is the active-azd-env project endpoint: FOUNDRY_PROJECT_ENDPOINT - // if set, otherwise AZURE_AI_PROJECT_ENDPOINT, otherwise "" (not set / no - // active env / no azd client available). - EnvValue string - // EnvName is the active azd env name. Only meaningful when EnvValue != "". - EnvName string - // CfgState is the project context persisted in global config. - CfgState State - // CfgFound indicates whether a non-empty endpoint was found in global config. - CfgFound bool -} - -// State is the JSON shape stored at extensions.ai-agents.project.context in -// ~/.azd/config.json. This key is owned by azure.ai.agents; the toolboxes -// extension reads it but never writes it. -type State struct { - Endpoint string `json:"endpoint"` - SetAt string `json:"setAt"` -} diff --git a/cli/azd/extensions/azure.ai.dataset/internal/foundry/projectctx/validator.go b/cli/azd/extensions/azure.ai.dataset/internal/foundry/projectctx/validator.go deleted file mode 100644 index 47e17e6163d..00000000000 --- a/cli/azd/extensions/azure.ai.dataset/internal/foundry/projectctx/validator.go +++ /dev/null @@ -1,113 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package projectctx - -import ( - "fmt" - "net/url" - "strings" - - "azureaidataset/internal/exterrors" -) - -// foundryHostSuffixes is the authoritative list of accepted Foundry host suffixes. -var foundryHostSuffixes = []string{ - ".services.ai.azure.com", -} - -// projectEndpointPathPrefix is the expected path prefix for Foundry project endpoints. -const projectEndpointPathPrefix = "/api/projects/" - -// isFoundryHost reports whether the hostname ends with a recognized Foundry suffix. -func isFoundryHost(hostname string) bool { - h := strings.ToLower(hostname) - for _, suffix := range foundryHostSuffixes { - if strings.HasSuffix(h, suffix) { - return true - } - } - return false -} - -// Validate validates and normalizes a Foundry project endpoint URL. -// -// The URL must be an absolute https:// URL whose host ends with a recognized -// Foundry suffix. Whitespace is trimmed, trailing slashes are stripped, and -// the result is returned in normalized form. -// -// The second return value is true when the path does not look like -// /api/projects/ — callers may use this as a non-fatal warning. -func Validate(raw string) (normalized string, pathWarning bool, err error) { - raw = strings.TrimSpace(raw) - if raw == "" { - return "", false, exterrors.Validation( - exterrors.CodeInvalidParameter, - "project endpoint must not be empty", - "provide a Foundry project endpoint URL "+ - "(e.g. https://.services.ai.azure.com/api/projects/)", - ) - } - - u, parseErr := url.Parse(raw) - if parseErr != nil { - return "", false, exterrors.Validation( - exterrors.CodeInvalidParameter, - fmt.Sprintf("invalid project endpoint URL: %v", parseErr), - "provide a valid https:// Foundry project endpoint URL", - ) - } - - if !strings.EqualFold(u.Scheme, "https") { - return "", false, exterrors.Validation( - exterrors.CodeInvalidParameter, - "project endpoint must use https", - "provide an https:// URL", - ) - } - - host := u.Hostname() - if host == "" || !isFoundryHost(host) { - return "", false, exterrors.Validation( - exterrors.CodeInvalidParameter, - fmt.Sprintf( - "project endpoint host %q is not a recognized Foundry host (*%s)", - host, foundryHostSuffixes[0], - ), - "the host must end with "+foundryHostSuffixes[0], - ) - } - - if u.Port() != "" { - return "", false, exterrors.Validation( - exterrors.CodeInvalidParameter, - fmt.Sprintf("project endpoint host %q must not include a port", u.Host), - "remove the explicit port from the URL", - ) - } - - // Normalize: lowercase host, strip trailing slash. - path := strings.TrimRight(u.EscapedPath(), "/") - normalized = fmt.Sprintf("https://%s%s", strings.ToLower(host), path) - - // Warn when the path does not look like /api/projects/. - if !strings.HasPrefix(path, projectEndpointPathPrefix) || - strings.TrimPrefix(path, projectEndpointPathPrefix) == "" { - pathWarning = true - } - - return normalized, pathWarning, nil -} - -// NoEndpointError returns the structured dependency error used when no project -// endpoint could be resolved from any source. -func NoEndpointError() error { - return exterrors.Dependency( - exterrors.CodeMissingProjectEndpoint, - "no Foundry project endpoint resolved", - "persist a workspace default with `azd ai project set `, "+ - "or set FOUNDRY_PROJECT_ENDPOINT (or AZURE_AI_PROJECT_ENDPOINT) "+ - "in the active azd environment, "+ - "or export FOUNDRY_PROJECT_ENDPOINT (or AZURE_AI_PROJECT_ENDPOINT) in your shell", - ) -} diff --git a/cli/azd/extensions/azure.ai.dataset/internal/pkg/dataset_api/download_test.go b/cli/azd/extensions/azure.ai.dataset/internal/pkg/dataset_api/download_test.go deleted file mode 100644 index 9ac6f21e92b..00000000000 --- a/cli/azd/extensions/azure.ai.dataset/internal/pkg/dataset_api/download_test.go +++ /dev/null @@ -1,47 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package dataset_api - -import ( - "testing" - - "github.com/stretchr/testify/require" -) - -// A dataset's URI points at either the blob or the container holding it, -// depending on how it was created, and nothing in the payload says which: -// isSingleFile is true either way. Uploaded datasets end in the file name; -// generated ones end in the container. Downloading a container returns 409. -func TestLooksLikeBlobURI(t *testing.T) { - uploaded := "https://acct.blob.core.windows.net:443/container-guid/azd-smoke-golden.jsonl" - generated := "https://acct.blob.core.windows.net/asayedahme-420d0b21-956c-513b-bb18-f60bfbf5e724" - - require.True(t, looksLikeBlobURI(uploaded), "an uploaded dataset names its file") - require.False(t, looksLikeBlobURI(generated), "a generated dataset names its container") -} - -// A SAS token on the URI must not change the answer. -func TestLooksLikeBlobURIIgnoresQuery(t *testing.T) { - require.True(t, looksLikeBlobURI( - "https://acct.blob.core.windows.net/c/data.jsonl?sv=2021&sig=abc")) - require.False(t, looksLikeBlobURI( - "https://acct.blob.core.windows.net/c?sv=2021&sig=abc")) - require.False(t, looksLikeBlobURI("https://acct.blob.core.windows.net/c/")) -} - -// An evaluation dataset is JSONL, so that is preferred when a container holds -// more than one file. -func TestPickDatasetBlobPrefersJSONL(t *testing.T) { - require.Equal(t, "data.jsonl", - pickDatasetBlob([]string{"_meta.json", "data.jsonl", "readme.txt"})) - require.Equal(t, "data.JSONL", - pickDatasetBlob([]string{"data.JSONL"}), "the extension match is case-insensitive") -} - -// With nothing recognisable, any real file beats returning nothing. -func TestPickDatasetBlobFallsBackToAnyFile(t *testing.T) { - require.Equal(t, "data.csv", pickDatasetBlob([]string{"data.csv"})) - require.Empty(t, pickDatasetBlob([]string{"folder/"})) - require.Empty(t, pickDatasetBlob(nil)) -} diff --git a/cli/azd/extensions/azure.ai.dataset/internal/pkg/dataset_api/download_wire_test.go b/cli/azd/extensions/azure.ai.dataset/internal/pkg/dataset_api/download_wire_test.go deleted file mode 100644 index a8b23386e34..00000000000 --- a/cli/azd/extensions/azure.ai.dataset/internal/pkg/dataset_api/download_wire_test.go +++ /dev/null @@ -1,255 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package dataset_api - -import ( - "context" - "encoding/json" - "net/http" - "net/http/httptest" - "net/url" - "strings" - "sync" - "testing" - - "github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -const testAPIVersion = "2025-11-15-preview" - -// blobListing is the shape Azure Blob Storage answers a container list with. -func blobListing(names ...string) string { - var b strings.Builder - b.WriteString(``) - for _, n := range names { - b.WriteString("" + n + "") - } - b.WriteString(``) - return b.String() -} - -// storageServer stands in for both the dataset API and blob storage, recording -// what each leg of a download was asked for. -type storageServer struct { - mu sync.Mutex - - // credential is the sasUri handed back for a download, relative to the - // server's own address. - credentialPath string - // blobs maps a container-relative blob name to its content. - blobs map[string]string - // directBlobStatus is the status a direct GET of credentialPath answers. - directBlobStatus int - - gotListQuery url.Values - gotBlobPaths []string - gotAPIVer []string -} - -func (s *storageServer) start(t *testing.T) (*DatasetClient, *httptest.Server) { - t.Helper() - var srv *httptest.Server - srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - s.mu.Lock() - defer s.mu.Unlock() - - if v := r.URL.Query().Get("api-version"); v != "" { - s.gotAPIVer = append(s.gotAPIVer, v) - } - - switch { - case strings.HasSuffix(r.URL.Path, "/credentials"): - w.Header().Set("Content-Type", "application/json") - require.NoError(t, json.NewEncoder(w).Encode(map[string]any{ - "blobReferenceForConsumption": map[string]any{ - "credential": map[string]any{"sasUri": srv.URL + s.credentialPath + "?sig=secret"}, - }, - })) - - case r.URL.Query().Get("comp") == "list": - s.gotListQuery = r.URL.Query() - names := make([]string, 0, len(s.blobs)) - for n := range s.blobs { - names = append(names, n) - } - w.Header().Set("Content-Type", "application/xml") - _, _ = w.Write([]byte(blobListing(names...))) - - // A direct read of the credential URI itself, keyed under "". - case r.URL.Path == s.credentialPath: - s.gotBlobPaths = append(s.gotBlobPaths, r.URL.Path) - if s.directBlobStatus != 0 { - w.WriteHeader(s.directBlobStatus) - return - } - _, _ = w.Write([]byte(s.blobs[""])) - - default: - s.gotBlobPaths = append(s.gotBlobPaths, r.URL.Path) - body, ok := s.blobs[strings.TrimPrefix(r.URL.Path, s.credentialPath+"/")] - if !ok { - w.WriteHeader(http.StatusNotFound) - return - } - _, _ = w.Write([]byte(body)) - } - })) - t.Cleanup(srv.Close) - - client := NewDatasetClientFromPipeline( - srv.URL, runtime.NewPipeline("test", "v1", runtime.PipelineOptions{}, nil)) - return client, srv -} - -// A dataset that was uploaded names its own file, so it reads in one hop and -// the container must never be listed. -func TestDownloadDatasetContentReadsABlobURIDirectly(t *testing.T) { - server := &storageServer{ - credentialPath: "/c/rows.jsonl", - blobs: map[string]string{"": `{"query":"direct"}`}, - } - client, _ := server.start(t) - - data, err := client.DownloadDatasetContent(context.Background(), "ds", "1.0", testAPIVersion) - require.NoError(t, err) - assert.Equal(t, `{"query":"direct"}`, string(data)) - assert.Nil(t, server.gotListQuery, "a blob URI needs no container listing") -} - -// A generated dataset names the container it was written into, and nothing in -// the payload says so: isSingleFile is true either way. Reading the container -// directly returns a 409, so the blob inside has to be found first. -func TestDownloadDatasetContentListsAContainerURI(t *testing.T) { - server := &storageServer{ - credentialPath: "/generated-container", - blobs: map[string]string{ - "_meta.json": `{"ignored":true}`, - "data.jsonl": `{"query":"from the container"}`, - }, - } - client, _ := server.start(t) - - data, err := client.DownloadDatasetContent(context.Background(), "ds", "1.0", testAPIVersion) - require.NoError(t, err) - assert.Equal(t, `{"query":"from the container"}`, string(data), - "the JSONL is chosen over the metadata sitting beside it") - require.NotNil(t, server.gotListQuery) - assert.Equal(t, "container", server.gotListQuery.Get("restype")) - assert.Equal(t, "secret", server.gotListQuery.Get("sig"), - "the listing must keep the SAS token, or storage answers 403") -} - -// A URI can name a file and still be a container — the extension is a guess, -// not a fact. When the direct read fails the listing is the fallback, so the -// download succeeds rather than surfacing the first status. -func TestDownloadDatasetContentFallsBackWhenTheBlobReadFails(t *testing.T) { - server := &storageServer{ - credentialPath: "/c/looks.jsonl", - directBlobStatus: http.StatusConflict, - blobs: map[string]string{"real.jsonl": `{"query":"found by listing"}`}, - } - client, _ := server.start(t) - - data, err := client.DownloadDatasetContent(context.Background(), "ds", "1.0", testAPIVersion) - require.NoError(t, err, "a 409 on the direct read is the container case, not a failure") - assert.Equal(t, `{"query":"found by listing"}`, string(data)) - assert.NotNil(t, server.gotListQuery) -} - -// An empty container is a dataset with nothing to read, and saying so beats -// returning empty content that looks like a dataset with no rows. -func TestDownloadDatasetContentReportsAnEmptyContainer(t *testing.T) { - server := &storageServer{credentialPath: "/empty", blobs: map[string]string{}} - client, _ := server.start(t) - - _, err := client.DownloadDatasetContent(context.Background(), "ds", "1.0", testAPIVersion) - require.Error(t, err) - assert.Contains(t, err.Error(), "no downloadable file") -} - -// The URI carries no SAS of its own, so a credential that resolves to nothing -// has to be reported here rather than as an unauthorized read later. -func TestDownloadDatasetContentRequiresADownloadURI(t *testing.T) { - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.Header().Set("Content-Type", "application/json") - _, _ = w.Write([]byte(`{}`)) - })) - t.Cleanup(srv.Close) - - client := NewDatasetClientFromPipeline( - srv.URL, runtime.NewPipeline("test", "v1", runtime.PipelineOptions{}, nil)) - - _, err := client.DownloadDatasetContent(context.Background(), "ds", "1.0", testAPIVersion) - require.Error(t, err) - assert.Contains(t, err.Error(), "no download URI") -} - -// The blob name is appended to the container path, and the SAS token stays on -// the query where storage expects it. -func TestDownloadBlobKeepsTheSASToken(t *testing.T) { - var gotPath, gotSig string - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - gotPath, gotSig = r.URL.Path, r.URL.Query().Get("sig") - _, _ = w.Write([]byte("rows")) - })) - t.Cleanup(srv.Close) - - client := NewDatasetClientFromPipeline( - srv.URL, runtime.NewPipeline("test", "v1", runtime.PipelineOptions{}, nil)) - - data, err := client.DownloadBlob(context.Background(), srv.URL+"/container?sig=secret", "data.jsonl") - require.NoError(t, err) - assert.Equal(t, "rows", string(data)) - assert.Equal(t, "/container/data.jsonl", gotPath) - assert.Equal(t, "secret", gotSig) -} - -// A storage failure has to name the blob, since the container holds several -// and the status alone does not say which one was refused. -func TestDownloadBlobReportsTheStatusAndName(t *testing.T) { - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(http.StatusForbidden) - })) - t.Cleanup(srv.Close) - - client := NewDatasetClientFromPipeline( - srv.URL, runtime.NewPipeline("test", "v1", runtime.PipelineOptions{}, nil)) - - _, err := client.DownloadBlob(context.Background(), srv.URL+"/c", "data.jsonl") - require.Error(t, err) - assert.Contains(t, err.Error(), "403") - assert.Contains(t, err.Error(), "data.jsonl") -} - -// A malformed URI is the caller's mistake, and it is worth catching before a -// request goes out against a half-parsed address. -func TestBlobOperationsRejectAnUnparseableURI(t *testing.T) { - client := NewDatasetClientFromPipeline( - "https://example", runtime.NewPipeline("test", "v1", runtime.PipelineOptions{}, nil)) - - _, err := client.DownloadBlob(context.Background(), "://nope", "x.jsonl") - require.Error(t, err) - - _, err = client.ListContainerBlobs(context.Background(), "://nope") - require.Error(t, err) - - err = client.UploadBlob(context.Background(), "://nope", "x.jsonl", []byte("{}")) - require.Error(t, err) -} - -// Storage answers a listing in XML, and a shape that does not parse yields no -// names rather than a panic. -func TestParseBlobNames(t *testing.T) { - assert.Equal(t, []string{"a.jsonl", "b.json"}, - parseBlobNames(blobListing("a.jsonl", "b.json"))) - assert.Empty(t, parseBlobNames(blobListing())) - assert.Empty(t, parseBlobNames("not xml at all"), - "an unreadable listing is an empty one, not a crash") - assert.Empty(t, parseBlobNames( - ``), - "a nameless blob cannot be downloaded, so it is not offered") -} diff --git a/cli/azd/extensions/azure.ai.dataset/internal/pkg/dataset_api/list.go b/cli/azd/extensions/azure.ai.dataset/internal/pkg/dataset_api/list.go deleted file mode 100644 index 19b548c3e05..00000000000 --- a/cli/azd/extensions/azure.ai.dataset/internal/pkg/dataset_api/list.go +++ /dev/null @@ -1,103 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package dataset_api - -import ( - "context" - "fmt" - "net/http" - "net/url" - "strconv" - "strings" -) - -// DatasetList is the paged response returned when listing datasets or the -// versions of one dataset. -type DatasetList struct { - Value []Dataset `json:"value"` - NextLink string `json:"nextLink,omitempty"` -} - -// ListDatasets returns the datasets registered on the project. -func (c *DatasetClient) ListDatasets(ctx context.Context, apiVersion string) (*DatasetList, error) { - return doRequestTyped[DatasetList](c, ctx, http.MethodGet, pathDatasets, nil, nil, apiVersion) -} - -// ListDatasetVersions returns every version of a single dataset. -func (c *DatasetClient) ListDatasetVersions( - ctx context.Context, - name string, - apiVersion string, -) (*DatasetList, error) { - path := fmt.Sprintf("%s/%s/versions", pathDatasets, url.PathEscape(name)) - return doRequestTyped[DatasetList](c, ctx, http.MethodGet, path, nil, nil, apiVersion) -} - -// DeleteDatasetVersion removes a single dataset version. -func (c *DatasetClient) DeleteDatasetVersion( - ctx context.Context, - name string, - version string, - apiVersion string, -) error { - path := fmt.Sprintf( - "%s/%s/versions/%s", - pathDatasets, url.PathEscape(name), url.PathEscape(version), - ) - _, err := c.doRequest(ctx, http.MethodDelete, path, nil, nil, apiVersion) - return err -} - -// VersionOrder returns a sortable value for a version string, matching the -// decimal convention NextVersion produces ("1.0", "2.0"). Unparseable versions -// sort lowest. -func VersionOrder(version string) float64 { - v := strings.TrimSpace(version) - if v == "" { - return -1 - } - if f, err := strconv.ParseFloat(v, 64); err == nil { - return f - } - // Fall back to trailing digits, e.g. "v3" -> 3. - i := len(v) - for i > 0 && v[i-1] >= '0' && v[i-1] <= '9' { - i-- - } - if i == len(v) { - return -1 - } - if n, err := strconv.Atoi(v[i:]); err == nil { - return float64(n) - } - return -1 -} - -// VersionGreater reports whether a is a strictly newer version than b. -// -// Both must be orderable; when either is not, the answer is false so an -// unparseable version never triggers a drift failure on its own. -func VersionGreater(a, b string) bool { - orderA, orderB := VersionOrder(a), VersionOrder(b) - if orderA < 0 || orderB < 0 { - return false - } - return orderA > orderB -} - -// LatestVersion returns the highest version in the list, falling back to the -// last entry when none of the versions can be ordered. -func LatestVersion(datasets []Dataset) string { - best := "" - bestOrder := -2.0 - for _, d := range datasets { - if o := VersionOrder(d.Version); o > bestOrder { - bestOrder, best = o, d.Version - } - } - if best == "" && len(datasets) > 0 { - return datasets[len(datasets)-1].Version - } - return best -} diff --git a/cli/azd/extensions/azure.ai.dataset/internal/pkg/dataset_api/models.go b/cli/azd/extensions/azure.ai.dataset/internal/pkg/dataset_api/models.go deleted file mode 100644 index 13fe6da2d9d..00000000000 --- a/cli/azd/extensions/azure.ai.dataset/internal/pkg/dataset_api/models.go +++ /dev/null @@ -1,210 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package dataset_api - -import ( - "fmt" - "math" - "os" - "path/filepath" - "strconv" - "strings" -) - -// CreateDatasetRequest is the request body for creating (uploading) a dataset. -type CreateDatasetRequest struct { - Name string `json:"name"` - Version string `json:"version"` - Format string `json:"format"` - Content string `json:"content"` -} - -// Dataset is the response for dataset operations. -// -// The field spelling is not consistent across the surface: the live -// project-endpoint GET returns camelCase (dataUri, isSingleFile), while other -// paths have used snake_case (data_uri, blob_uri, content_uri). Both spellings -// are accepted here because binding only one silently yields an empty URI, -// which then fails much later at download time. -type Dataset struct { - ID string `json:"id,omitempty"` - Name string `json:"name"` - Version string `json:"version"` - Type string `json:"type,omitempty"` - Format string `json:"format,omitempty"` - - // camelCase spellings (project endpoint). - DataURICamel string `json:"dataUri,omitempty"` - BlobURICamel string `json:"blobUri,omitempty"` - ContentURICamel string `json:"contentUri,omitempty"` - IsSingleFile bool `json:"isSingleFile,omitempty"` - ConnectionName string `json:"connectionName,omitempty"` - - // snake_case spellings. - BlobURI string `json:"blob_uri,omitempty"` - DataURI string `json:"data_uri,omitempty"` - ContentURI string `json:"content_uri,omitempty"` -} - -// ResolvedBlobURI returns the first URI the service supplied, across both -// spellings. An empty result means the dataset carries no downloadable URI and -// the caller must fetch a credential instead. -func (d *Dataset) ResolvedBlobURI() string { - for _, candidate := range []string{ - d.BlobURI, d.BlobURICamel, - d.DataURI, d.DataURICamel, - d.ContentURI, d.ContentURICamel, - } { - if candidate != "" { - return candidate - } - } - return "" -} - -// DatasetCredential is the response for dataset credential (SAS token) requests. -// The API returns a nested structure with blobReference and blobReferenceForConsumption. -type DatasetCredential struct { - // Flat fields (legacy format). - BlobURI string `json:"blob_uri,omitempty"` - SAS string `json:"sas,omitempty"` - SASUri string `json:"sas_uri,omitempty"` - - // Nested fields (current API format). - BlobReference *BlobReference `json:"blobReference,omitempty"` - BlobReferenceConsumption *BlobReference `json:"blobReferenceForConsumption,omitempty"` -} - -// BlobReference represents a blob storage reference with credentials. -type BlobReference struct { - BlobURI string `json:"blobUri,omitempty"` - StorageAccountARM string `json:"storageAccountArmId,omitempty"` - Credential *BlobCredential `json:"credential,omitempty"` -} - -// BlobCredential holds SAS credential details for blob access. -type BlobCredential struct { - Type string `json:"type,omitempty"` - SASUri string `json:"sasUri,omitempty"` - SASPath string `json:"sas,omitempty"` -} - -// ResolvedDownloadURI returns the URL to download the dataset. -// Prefers blobReferenceForConsumption.credential.sasUri (current API), -// then blobReference.credential.sasUri, then flat sas_uri, then blob_uri + sas. -func (c *DatasetCredential) ResolvedDownloadURI() string { - // Current API format: nested blob references. - if c.BlobReferenceConsumption != nil && c.BlobReferenceConsumption.Credential != nil { - if uri := c.BlobReferenceConsumption.Credential.SASUri; uri != "" { - return uri - } - } - if c.BlobReference != nil && c.BlobReference.Credential != nil { - if uri := c.BlobReference.Credential.SASUri; uri != "" { - return uri - } - } - // Legacy flat format. - if c.SASUri != "" { - return c.SASUri - } - if c.BlobURI != "" && c.SAS != "" { - return c.BlobURI + "?" + c.SAS - } - return c.BlobURI -} - -// PendingUploadResponse is returned by the startPendingUpload endpoint. -// It contains a SAS URI for uploading blob data and the blob container URI. -type PendingUploadResponse struct { - BlobReference *BlobReference `json:"blobReference,omitempty"` - BlobReferenceConsumption *BlobReference `json:"blobReferenceForConsumption,omitempty"` - PendingUploadID *string `json:"pendingUploadId,omitempty"` - PendingUploadType string `json:"pendingUploadType,omitempty"` - Version string `json:"version,omitempty"` -} - -// ResolvedUploadURI returns the SAS URI for uploading blobs. -func (p *PendingUploadResponse) ResolvedUploadURI() string { - if p.BlobReference != nil && p.BlobReference.Credential != nil { - if uri := p.BlobReference.Credential.SASUri; uri != "" { - return uri - } - } - return "" -} - -// ResolvedBlobURI returns the blob container URI (without SAS) for the finalize request. -func (p *PendingUploadResponse) ResolvedBlobURI() string { - if p.BlobReference != nil { - return p.BlobReference.BlobURI - } - return "" -} - -// FinalizeDatasetRequest is the request body for finalizing a dataset version -// after blob upload. -type FinalizeDatasetRequest struct { - Name string `json:"name"` - Version string `json:"version"` - Description string `json:"description"` - Type string `json:"type"` - IsReference bool `json:"isReference"` - DataURI string `json:"dataUri"` -} - -// NextVersion computes the next dataset version string. -// -// Rules: -// 1. Empty → "1.0" -// 2. Parsable as a decimal number → increment by 1, format as "N.0" -// 3. Ends with trailing digits → increment the trailing numeric part -// 4. Otherwise → append ".1" -func NextVersion(current string) string { - current = strings.TrimSpace(current) - if current == "" { - return "1.0" - } - - // Try parsing as a decimal number (e.g. "1", "1.0", "2.0"). - if f, err := strconv.ParseFloat(current, 64); err == nil { - return strconv.FormatFloat(math.Floor(f)+1, 'f', 1, 64) - } - - // Find trailing digits and increment them. - i := len(current) - 1 - for i >= 0 && current[i] >= '0' && current[i] <= '9' { - i-- - } - if i < len(current)-1 { - prefix := current[:i+1] - n, err := strconv.Atoi(current[i+1:]) - if err == nil { - return prefix + strconv.Itoa(n+1) - } - } - - return current + ".1" -} - -// ReadFirstJSONLFile finds and reads the first .jsonl file in a directory. -func ReadFirstJSONLFile(dir string) (string, error) { - entries, err := os.ReadDir(dir) - if err != nil { - return "", fmt.Errorf("reading directory: %w", err) - } - for _, e := range entries { - if e.IsDir() { - continue - } - if filepath.Ext(e.Name()) == ".jsonl" { - data, err := os.ReadFile(filepath.Join(dir, e.Name())) //nolint:gosec // local artifact path - if err != nil { - return "", fmt.Errorf("reading %s: %w", e.Name(), err) - } - return string(data), nil - } - } - return "", fmt.Errorf("no .jsonl file found in %s", dir) -} diff --git a/cli/azd/extensions/azure.ai.dataset/internal/pkg/dataset_api/operations.go b/cli/azd/extensions/azure.ai.dataset/internal/pkg/dataset_api/operations.go deleted file mode 100644 index 23e59cc9e8b..00000000000 --- a/cli/azd/extensions/azure.ai.dataset/internal/pkg/dataset_api/operations.go +++ /dev/null @@ -1,640 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package dataset_api - -import ( - "bytes" - "context" - "encoding/json" - "encoding/xml" - "errors" - "fmt" - "io" - "log" - "net/http" - "net/url" - "path" - "strings" - - "azureaidataset/internal/version" - - "github.com/Azure/azure-sdk-for-go/sdk/azcore" - "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" - "github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime" - "github.com/Azure/azure-sdk-for-go/sdk/azcore/streaming" - "github.com/azure/azure-dev/cli/azd/pkg/azsdk" -) - -// API path prefix for dataset endpoints. -const pathDatasets = "/datasets" - -// DatasetClient provides methods for dataset upload, download, and metadata retrieval. -type DatasetClient struct { - endpoint string - pipeline runtime.Pipeline -} - -// NewDatasetClient creates a new DatasetClient. -func NewDatasetClient(endpoint string, cred azcore.TokenCredential) *DatasetClient { - userAgent := fmt.Sprintf("azd-ext-azure-ai-evaluations/%s", version.Version) - - clientOptions := &policy.ClientOptions{ - Logging: policy.LogOptions{ - AllowedHeaders: []string{"X-Ms-Correlation-Request-Id", "X-Request-Id"}, - IncludeBody: false, - }, - PerCallPolicies: []policy.Policy{ - runtime.NewBearerTokenPolicy(cred, []string{"https://ai.azure.com/.default"}, nil), - azsdk.NewMsCorrelationPolicy(), - azsdk.NewUserAgentPolicy(userAgent), - }, - } - - pipeline := runtime.NewPipeline( - "azure-ai-datasets", - "v1.0.0", - runtime.PipelineOptions{}, - clientOptions, - ) - - return &DatasetClient{ - endpoint: endpoint, - pipeline: pipeline, - } -} - -// NewDatasetClientFromPipeline creates a DatasetClient with a pre-built pipeline. -// This is intended for tests that need to bypass auth policies. -func NewDatasetClientFromPipeline(endpoint string, pipeline runtime.Pipeline) *DatasetClient { - return &DatasetClient{ - endpoint: endpoint, - pipeline: pipeline, - } -} - -// CreateDataset registers a dataset with inline content (upload). -func (c *DatasetClient) CreateDataset( - ctx context.Context, - request *CreateDatasetRequest, - apiVersion string, -) (*Dataset, error) { - return doRequestTyped[Dataset](c, ctx, http.MethodPost, pathDatasets, nil, request, apiVersion) -} - -// UploadNextVersion registers the next version of a dataset, discovering the -// current one from the service when currentVersion is empty. -// -// Prefer this over UploadNewVersion. That function derives the next version -// from whatever it is handed, so an empty value restarts at 1.0 and the -// service rejects the pending upload with a 409 -// TemporaryDataReferencesForExistingAsset as soon as 1.0 exists. Callers -// almost always mean "the version after whatever is registered", which is what -// this does. -// -// The version listing is eventually consistent — it returns nothing for a -// second or two after a version is created — so an empty listing cannot be -// trusted to mean the dataset is new. A conflict is therefore treated as a -// stale read: the listing is re-read, and when it is still behind, the version -// just refused is taken as proof that it exists and the next one is tried. -// Trusting the listing alone left a second upload issued moments after the -// first reporting a 409 to the user for a publish that should simply have -// added a version. -func (c *DatasetClient) UploadNextVersion( - ctx context.Context, - name string, - currentVersion string, - localDir string, - apiVersion string, -) (*Dataset, error) { - if currentVersion == "" { - currentVersion = c.latestRegisteredVersion(ctx, name, apiVersion) - } - - var err error - for range versionConflictAttempts { - var ds *Dataset - ds, err = c.UploadNewVersion(ctx, name, currentVersion, localDir, apiVersion) - if err == nil || !IsVersionConflict(err) { - return ds, err - } - - // The version derived from currentVersion is taken, so it exists - // whatever the listing says. Prefer the listing when it has caught up - // and moved further ahead; otherwise step past what was just refused. - refused := NextVersion(currentVersion) - currentVersion = refused - if latest := c.latestRegisteredVersion(ctx, name, apiVersion); versionAtLeast(latest, refused) { - currentVersion = latest - } - } - return nil, err -} - -// versionConflictAttempts bounds the walk past versions the listing has not -// caught up with. Each attempt is one refused pending upload, so this is short. -const versionConflictAttempts = 4 - -// versionAtLeast reports whether a is a version at or beyond b. -func versionAtLeast(a, b string) bool { - if a == "" { - return false - } - return LatestVersion([]Dataset{{Version: a}, {Version: b}}) == a -} - -// latestRegisteredVersion returns the newest registered version, or empty when -// the dataset is unknown or the listing has not caught up yet. -func (c *DatasetClient) latestRegisteredVersion( - ctx context.Context, - name string, - apiVersion string, -) string { - list, err := c.ListDatasetVersions(ctx, name, apiVersion) - if err != nil || list == nil || len(list.Value) == 0 { - return "" - } - return LatestVersion(list.Value) -} - -// isVersionConflict reports whether the service refused the upload because the -// target version already exists. -func IsVersionConflict(err error) bool { - var respErr *azcore.ResponseError - if !errors.As(err, &respErr) { - return false - } - return respErr.StatusCode == http.StatusConflict -} - -// IsNotFound reports whether the service answered 404. -func IsNotFound(err error) bool { - var respErr *azcore.ResponseError - if !errors.As(err, &respErr) { - return false - } - return respErr.StatusCode == http.StatusNotFound -} - -// UploadNewVersion reads the first JSONL file from localDir, computes the next -// version from currentVersion, and uploads it as a new dataset version using -// the 3-step pending upload flow: -// 1. startPendingUpload → get SAS URI -// 2. Upload blob to SAS URI -// 3. Finalize dataset version with dataUri -func (c *DatasetClient) UploadNewVersion( - ctx context.Context, - name string, - currentVersion string, - localDir string, - apiVersion string, -) (*Dataset, error) { - return c.UploadVersion(ctx, name, NextVersion(currentVersion), localDir, apiVersion) -} - -// UploadVersion publishes the dataset at exactly this version. -// -// Separate from UploadNewVersion because its parameter is the version to -// count from, not the one to write: passing "1.0" there publishes 2.0. An -// author who declares a version means that version. -func (c *DatasetClient) UploadVersion( - ctx context.Context, - name string, - version string, - localDir string, - apiVersion string, -) (*Dataset, error) { - content, err := ReadFirstJSONLFile(localDir) - if err != nil { - return nil, fmt.Errorf("reading dataset from %s: %w", localDir, err) - } - - newVersion := version - - // Step 1: Start pending upload to get a SAS URI. - pending, err := c.StartPendingUpload(ctx, name, newVersion, apiVersion) - if err != nil { - return nil, fmt.Errorf("starting pending upload: %w", err) - } - - uploadURI := pending.ResolvedUploadURI() - if uploadURI == "" { - return nil, fmt.Errorf("no upload SAS URI returned from startPendingUpload") - } - - // Step 2: Upload the JSONL file to blob storage. - blobName := name + ".jsonl" - if err := c.UploadBlob(ctx, uploadURI, blobName, []byte(content)); err != nil { - return nil, fmt.Errorf("uploading blob: %w", err) - } - - // Step 3: Finalize the dataset version with the full blob URI. - dataURI := strings.TrimSuffix(pending.ResolvedBlobURI(), "/") + "/" + blobName - return c.FinalizeDatasetVersion(ctx, name, newVersion, dataURI, apiVersion) -} - -// StartPendingUpload initiates a pending upload for a dataset version. -// Returns the SAS URI and blob reference for uploading data. -func (c *DatasetClient) StartPendingUpload( - ctx context.Context, - name string, - version string, - apiVersion string, -) (*PendingUploadResponse, error) { - path := fmt.Sprintf( - "%s/%s/versions/%s/startPendingUpload", - pathDatasets, url.PathEscape(name), url.PathEscape(version), - ) - return doRequestTyped[PendingUploadResponse](c, ctx, http.MethodPost, path, nil, json.RawMessage(`{}`), apiVersion) -} - -// UploadBlob uploads data to a container SAS URI as a block blob. -func (c *DatasetClient) UploadBlob(ctx context.Context, containerSASUri, blobName string, data []byte) error { - u, err := url.Parse(containerSASUri) - if err != nil { - return fmt.Errorf("invalid container SAS URI: %w", err) - } - - // Append blob name to the container path. - u.Path = strings.TrimSuffix(u.Path, "/") + "/" + blobName - - req, err := http.NewRequestWithContext(ctx, http.MethodPut, u.String(), bytes.NewReader(data)) - if err != nil { - return fmt.Errorf("failed to create upload request: %w", err) - } - req.Header.Set("x-ms-blob-type", "BlockBlob") - req.Header.Set("Content-Type", "application/octet-stream") - - httpClient := &http.Client{} - resp, err := httpClient.Do(req) - if err != nil { - return fmt.Errorf("failed to upload blob: %w", err) - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusCreated && resp.StatusCode != http.StatusOK { - body, _ := io.ReadAll(resp.Body) - return fmt.Errorf("blob upload failed with status %d: %s", resp.StatusCode, string(body)) - } - - return nil -} - -// FinalizeDatasetVersion completes the dataset version after blob upload -// by sending the metadata (name, version, dataUri) to the API. -func (c *DatasetClient) FinalizeDatasetVersion( - ctx context.Context, - name string, - version string, - dataURI string, - apiVersion string, -) (*Dataset, error) { - path := fmt.Sprintf("%s/%s/versions/%s", pathDatasets, url.PathEscape(name), url.PathEscape(version)) - request := &FinalizeDatasetRequest{ - Name: name, - Version: version, - Type: "uri_file", - DataURI: dataURI, - } - return doRequestTyped[Dataset](c, ctx, http.MethodPut, path, nil, request, apiVersion) -} - -// GetDataset retrieves metadata for a dataset by name and version. -func (c *DatasetClient) GetDataset( - ctx context.Context, - name string, - version string, - apiVersion string, -) (*Dataset, error) { - path := fmt.Sprintf("%s/%s/versions/%s", pathDatasets, url.PathEscape(name), url.PathEscape(version)) - return doRequestTyped[Dataset](c, ctx, http.MethodGet, path, nil, nil, apiVersion) -} - -// GetDatasetCredential retrieves a SAS credential for downloading a dataset from blob storage. -func (c *DatasetClient) GetDatasetCredential( - ctx context.Context, - name string, - version string, - apiVersion string, -) (*DatasetCredential, error) { - path := fmt.Sprintf( - "%s/%s/versions/%s/credentials", - pathDatasets, url.PathEscape(name), url.PathEscape(version), - ) - return doRequestTyped[DatasetCredential](c, ctx, http.MethodPost, path, nil, nil, apiVersion) -} - -// DownloadDatasetContent fetches a dataset version's content, whether its URI -// names a blob or a container. -// -// The two differ by origin, not by any field: a dataset uploaded through -// startPendingUpload gets a URI ending in the file name, while one produced by -// a generation job gets the container it was written into, with isSingleFile -// true either way. Downloading the container directly returns a 409, so the -// blob inside has to be found first. -// -// A credential is always fetched, because the URI on the dataset carries no -// SAS token and an unauthenticated read fails. -func (c *DatasetClient) DownloadDatasetContent( - ctx context.Context, - name string, - version string, - apiVersion string, -) ([]byte, error) { - cred, err := c.GetDatasetCredential(ctx, name, version, apiVersion) - if err != nil { - return nil, fmt.Errorf("reading download credentials for %q: %w", name, err) - } - - sasURI := cred.ResolvedDownloadURI() - if sasURI == "" { - return nil, fmt.Errorf("no download URI returned for dataset %q", name) - } - - // A URI whose last path segment carries a file extension is the blob - // itself; anything else is the container holding it. - if looksLikeBlobURI(sasURI) { - data, err := c.DownloadDataset(ctx, sasURI) - if err == nil { - return data, nil - } - log.Printf("[dataset_api] direct download failed (%v); treating the URI as a container", err) - } - - names, err := c.ListContainerBlobs(ctx, sasURI) - if err != nil { - return nil, fmt.Errorf("listing the content of dataset %q: %w", name, err) - } - blobName := pickDatasetBlob(names) - if blobName == "" { - return nil, fmt.Errorf("dataset %q holds no downloadable file", name) - } - return c.DownloadBlob(ctx, sasURI, blobName) -} - -// looksLikeBlobURI reports whether the URI's final segment names a file. -func looksLikeBlobURI(raw string) bool { - u, err := url.Parse(raw) - if err != nil { - return false - } - last := path.Base(strings.TrimSuffix(u.Path, "/")) - return path.Ext(last) != "" -} - -// pickDatasetBlob chooses the file to read from a container, preferring JSONL -// since that is what an evaluation dataset is. -func pickDatasetBlob(names []string) string { - for _, n := range names { - if strings.EqualFold(path.Ext(n), ".jsonl") { - return n - } - } - for _, n := range names { - if n != "" && !strings.HasSuffix(n, "/") { - return n - } - } - return "" -} - -// DownloadDataset downloads dataset content from blob storage using a SAS-authenticated URL. -// Returns the raw content as bytes. The downloadURL should be the full URL with SAS token -// (e.g., from DatasetCredential.ResolvedDownloadURI()). -func (c *DatasetClient) DownloadDataset(ctx context.Context, downloadURL string) ([]byte, error) { - req, err := runtime.NewRequest(ctx, http.MethodGet, downloadURL) - if err != nil { - return nil, fmt.Errorf("failed to create download request: %w", err) - } - - // Use a plain HTTP client for blob downloads — the SAS token in the URL provides - // authentication, and Azure SDK pipeline policies (bearer token, correlation ID) - // should not be sent to Azure Blob Storage endpoints. - httpClient := &http.Client{} - resp, err := httpClient.Do(req.Raw()) - if err != nil { - return nil, fmt.Errorf("failed to download dataset from blob: %w", err) - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("blob download failed with status %d", resp.StatusCode) - } - - data, err := io.ReadAll(resp.Body) - if err != nil { - return nil, fmt.Errorf("failed to read dataset content: %w", err) - } - - log.Printf("[dataset_api] downloaded %d bytes", len(data)) - return data, nil -} - -// ListContainerBlobs lists blobs in a container using a container-level SAS URI. -// The containerSASUri should include the SAS token (e.g., from credential.sasUri with sr=c). -// Returns a list of blob names found in the container. -func (c *DatasetClient) ListContainerBlobs(ctx context.Context, containerSASUri string) ([]string, error) { - // Parse the container URI and append list query parameters. - u, err := url.Parse(containerSASUri) - if err != nil { - return nil, fmt.Errorf("invalid container SAS URI: %w", err) - } - - q := u.Query() - q.Set("restype", "container") // cspell:ignore restype — Azure Storage API query parameter - q.Set("comp", "list") - u.RawQuery = q.Encode() - - log.Printf("[dataset_api] listing blobs: %s", u.Redacted()) - - req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil) - if err != nil { - return nil, fmt.Errorf("failed to create list request: %w", err) - } - - httpClient := &http.Client{} - resp, err := httpClient.Do(req) - if err != nil { - return nil, fmt.Errorf("failed to list container blobs: %w", err) - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("container list failed with status %d", resp.StatusCode) - } - - body, err := io.ReadAll(resp.Body) - if err != nil { - return nil, fmt.Errorf("failed to read list response: %w", err) - } - - // Parse XML blob listing to extract blob names. - names := parseBlobNames(string(body)) - log.Printf("[dataset_api] found %d blobs in container", len(names)) - return names, nil -} - -// DownloadBlob downloads a single blob from a container using the container SAS URI -// and the blob name. Returns the blob content as bytes. -func (c *DatasetClient) DownloadBlob(ctx context.Context, containerSASUri, blobName string) ([]byte, error) { - u, err := url.Parse(containerSASUri) - if err != nil { - return nil, fmt.Errorf("invalid container SAS URI: %w", err) - } - - // Append blob name to the container path. - u.Path = strings.TrimSuffix(u.Path, "/") + "/" + blobName - - req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil) - if err != nil { - return nil, fmt.Errorf("failed to create blob download request: %w", err) - } - - httpClient := &http.Client{} - resp, err := httpClient.Do(req) - if err != nil { - return nil, fmt.Errorf("failed to download blob: %w", err) - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("blob download failed with status %d for %s", resp.StatusCode, blobName) - } - - data, err := io.ReadAll(resp.Body) - if err != nil { - return nil, fmt.Errorf("failed to read blob content: %w", err) - } - - log.Printf("[dataset_api] downloaded blob %s (%d bytes)", blobName, len(data)) - return data, nil -} - -// parseBlobNames extracts blob names from the Azure Blob Storage XML list response -// using proper XML parsing against the EnumerationResults schema. -func parseBlobNames(xmlBody string) []string { - type blob struct { - Name string `xml:"Name"` - } - type blobs struct { - Blob []blob `xml:"Blob"` - } - type enumerationResults struct { - Blobs blobs `xml:"Blobs"` - } - - var result enumerationResults - if err := xml.Unmarshal([]byte(xmlBody), &result); err != nil { - return nil - } - - names := make([]string, 0, len(result.Blobs.Blob)) - for _, b := range result.Blobs.Blob { - if b.Name != "" { - names = append(names, b.Name) - } - } - return names -} - -// doRequest performs an HTTP request against the dataset API and returns the raw response body. -func (c *DatasetClient) doRequest( - ctx context.Context, - method string, - path string, - query map[string]string, - body any, - apiVersion string, -) ([]byte, error) { - u, err := url.Parse(c.endpoint) - if err != nil { - return nil, fmt.Errorf("invalid endpoint URL: %w", err) - } - - // Callers escape the name and version they interpolate, so the path is set - // as the raw one. Assigning it to u.Path re-escapes the percent signs, and - // a dataset named "my dataset" then addresses one named "my%20dataset". - escapedPath := u.EscapedPath() + path - decodedPath, err := url.PathUnescape(escapedPath) - if err != nil { - return nil, fmt.Errorf("invalid request path %q: %w", escapedPath, err) - } - u.Path, u.RawPath = decodedPath, escapedPath - - q := u.Query() - if apiVersion != "" { - q.Set("api-version", apiVersion) - } - for k, v := range query { - q.Set(k, v) - } - u.RawQuery = q.Encode() - - req, err := runtime.NewRequest(ctx, method, u.String()) - if err != nil { - return nil, fmt.Errorf("failed to create request: %w", err) - } - - log.Printf("[dataset_api] %s %s", method, u.Redacted()) - - if body != nil { - payload, err := json.Marshal(body) - if err != nil { - return nil, fmt.Errorf("failed to marshal request: %w", err) - } - if err := req.SetBody(streaming.NopCloser(bytes.NewReader(payload)), "application/json"); err != nil { - return nil, fmt.Errorf("failed to set request body: %w", err) - } - } - - resp, err := c.pipeline.Do(req) - if err != nil { - return nil, fmt.Errorf("HTTP request failed: %w", err) - } - defer resp.Body.Close() - - respBody, err := io.ReadAll(resp.Body) - if err != nil { - return nil, fmt.Errorf("failed to read response body: %w", err) - } - - log.Printf("[dataset_api] response status: %d", resp.StatusCode) - - // 204 belongs here for the same reason it does in eval_api: a delete that - // removed the version answers No Content, and rejecting that reports every - // successful delete as an error. - if !runtime.HasStatusCode(resp, - http.StatusOK, http.StatusCreated, http.StatusAccepted, http.StatusNoContent) { - resp.Body = io.NopCloser(bytes.NewReader(respBody)) - return nil, runtime.NewResponseError(resp) - } - - return respBody, nil -} - -// doRequestTyped performs an HTTP request and unmarshals the response into T. -func doRequestTyped[T any]( - c *DatasetClient, - ctx context.Context, - method string, - path string, - query map[string]string, - body any, - apiVersion string, -) (*T, error) { - respBody, err := c.doRequest(ctx, method, path, query, body, apiVersion) - if err != nil { - return nil, err - } - - if len(respBody) == 0 { - return new(T), nil - } - - var result T - if err := json.Unmarshal(respBody, &result); err != nil { - return nil, fmt.Errorf("failed to parse response: %w", err) - } - - return &result, nil -} diff --git a/cli/azd/extensions/azure.ai.dataset/internal/pkg/dataset_api/operations_wire_test.go b/cli/azd/extensions/azure.ai.dataset/internal/pkg/dataset_api/operations_wire_test.go deleted file mode 100644 index 00f4bb8ff19..00000000000 --- a/cli/azd/extensions/azure.ai.dataset/internal/pkg/dataset_api/operations_wire_test.go +++ /dev/null @@ -1,205 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package dataset_api - -import ( - "context" - "net/http" - "net/http/httptest" - "testing" - "time" - - "github.com/Azure/azure-sdk-for-go/sdk/azcore" - "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" - "github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -// fakeCredential satisfies the constructor without reaching for a real token. -type fakeCredential struct{} - -func (fakeCredential) GetToken(context.Context, policy.TokenRequestOptions) (azcore.AccessToken, error) { - return azcore.AccessToken{Token: "fake", ExpiresOn: time.Now().Add(time.Hour)}, nil -} - -// recordedCall is one request the client made, as the service saw it. -type recordedCall struct { - method string - path string - rawPath string - apiVersion string -} - -// recordingDatasetClient answers every request with body and status, recording -// what was asked. Retries are off so a deliberate failure is one call. -func recordingDatasetClient(t *testing.T, status int, body string) (*DatasetClient, *[]recordedCall) { - t.Helper() - calls := &[]recordedCall{} - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - *calls = append(*calls, recordedCall{ - method: r.Method, - path: r.URL.Path, - rawPath: r.URL.EscapedPath(), - apiVersion: r.URL.Query().Get("api-version"), - }) - w.Header().Set("Content-Type", "application/json") - w.WriteHeader(status) - if body != "" { - _, _ = w.Write([]byte(body)) - } - })) - t.Cleanup(srv.Close) - - pipeline := runtime.NewPipeline("test", "v1", runtime.PipelineOptions{}, - &policy.ClientOptions{Retry: policy.RetryOptions{MaxRetries: -1}}) - return NewDatasetClientFromPipeline(srv.URL, pipeline), calls -} - -// The paths are the service contract, and a wrong one costs a round trip to -// find out. Each is pinned against the shape the API documents. -func TestDatasetOperationPaths(t *testing.T) { - cases := []struct { - name string - call func(c *DatasetClient) error - wantMethod string - wantPath string - }{ - { - name: "list", - call: func(c *DatasetClient) error { _, err := c.ListDatasets(t.Context(), testAPIVersion); return err }, - wantMethod: http.MethodGet, - wantPath: "/datasets", - }, - { - name: "list versions", - call: func(c *DatasetClient) error { - _, err := c.ListDatasetVersions(t.Context(), "ds", testAPIVersion) - return err - }, - wantMethod: http.MethodGet, - wantPath: "/datasets/ds/versions", - }, - { - name: "get", - call: func(c *DatasetClient) error { - _, err := c.GetDataset(t.Context(), "ds", "1.0", testAPIVersion) - return err - }, - wantMethod: http.MethodGet, - wantPath: "/datasets/ds/versions/1.0", - }, - { - name: "credential", - call: func(c *DatasetClient) error { - _, err := c.GetDatasetCredential(t.Context(), "ds", "1.0", testAPIVersion) - return err - }, - wantMethod: http.MethodPost, - wantPath: "/datasets/ds/versions/1.0/credentials", - }, - { - name: "start pending upload", - call: func(c *DatasetClient) error { - _, err := c.StartPendingUpload(t.Context(), "ds", "1.0", testAPIVersion) - return err - }, - wantMethod: http.MethodPost, - wantPath: "/datasets/ds/versions/1.0/startPendingUpload", - }, - { - name: "finalize", - call: func(c *DatasetClient) error { - _, err := c.FinalizeDatasetVersion(t.Context(), "ds", "1.0", "https://x/y.jsonl", testAPIVersion) - return err - }, - wantMethod: http.MethodPut, - wantPath: "/datasets/ds/versions/1.0", - }, - { - name: "create", - call: func(c *DatasetClient) error { - _, err := c.CreateDataset(t.Context(), &CreateDatasetRequest{Name: "ds"}, testAPIVersion) - return err - }, - wantMethod: http.MethodPost, - wantPath: "/datasets", - }, - { - name: "delete", - call: func(c *DatasetClient) error { return c.DeleteDatasetVersion(t.Context(), "ds", "1.0", testAPIVersion) }, - wantMethod: http.MethodDelete, - wantPath: "/datasets/ds/versions/1.0", - }, - } - - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - client, calls := recordingDatasetClient(t, http.StatusOK, `{"name":"ds","version":"1.0","value":[]}`) - require.NoError(t, tc.call(client)) - require.Len(t, *calls, 1) - assert.Equal(t, tc.wantMethod, (*calls)[0].method) - assert.Equal(t, tc.wantPath, (*calls)[0].path) - assert.Equal(t, testAPIVersion, (*calls)[0].apiVersion, - "the service rejects a request that names no api-version") - }) - } -} - -// A name is caller-supplied and a version can be anything the author wrote, so -// both are escaped rather than pasted into the path. -func TestDatasetPathsEscapeNameAndVersion(t *testing.T) { - client, calls := recordingDatasetClient(t, http.StatusOK, `{}`) - _, err := client.GetDataset(t.Context(), "my dataset/v", "1.0 beta", testAPIVersion) - require.NoError(t, err) - - require.Len(t, *calls, 1) - assert.Equal(t, "/datasets/my%20dataset%2Fv/versions/1.0%20beta", (*calls)[0].rawPath, - "an unescaped slash would address a different resource entirely") -} - -// A delete answers 204 with nothing in it, which must not read as a failure to -// parse a body that was never promised. -func TestDeleteDatasetVersionAcceptsNoContent(t *testing.T) { - client, calls := recordingDatasetClient(t, http.StatusNoContent, "") - require.NoError(t, client.DeleteDatasetVersion(t.Context(), "ds", "1.0", testAPIVersion)) - assert.Len(t, *calls, 1) -} - -// The listing arrives wrapped in a value envelope; reading it flat yields an -// empty list rather than an error, which looks like a project with no datasets. -func TestListDatasetsReadsTheValueEnvelope(t *testing.T) { - client, _ := recordingDatasetClient(t, http.StatusOK, - `{"value":[{"name":"a","version":"1.0"},{"name":"b","version":"2.0"}]}`) - - list, err := client.ListDatasets(t.Context(), testAPIVersion) - require.NoError(t, err) - require.Len(t, list.Value, 2) - assert.Equal(t, "a", list.Value[0].Name) - assert.Equal(t, "2.0", list.Value[1].Version) -} - -// A failure has to surface as one, since the caller otherwise proceeds with a -// zero-valued dataset and fails somewhere further away. -func TestDatasetOperationsSurfaceServiceFailures(t *testing.T) { - client, _ := recordingDatasetClient(t, http.StatusNotFound, `{"error":{"code":"NotFound"}}`) - - _, err := client.GetDataset(t.Context(), "missing", "1.0", testAPIVersion) - require.Error(t, err) - - err = client.DeleteDatasetVersion(t.Context(), "missing", "1.0", testAPIVersion) - require.Error(t, err) - - _, err = client.ListDatasets(t.Context(), testAPIVersion) - require.Error(t, err) -} - -// The constructor has to build a usable client — it wires the auth policies -// the live service needs, and nothing else exercises that path. -func TestNewDatasetClient(t *testing.T) { - client := NewDatasetClient("https://example.services.ai.azure.com/api/projects/p", fakeCredential{}) - require.NotNil(t, client) - assert.Equal(t, "https://example.services.ai.azure.com/api/projects/p", client.endpoint) -} diff --git a/cli/azd/extensions/azure.ai.dataset/internal/pkg/dataset_api/upload_version_test.go b/cli/azd/extensions/azure.ai.dataset/internal/pkg/dataset_api/upload_version_test.go deleted file mode 100644 index 164cc6d415b..00000000000 --- a/cli/azd/extensions/azure.ai.dataset/internal/pkg/dataset_api/upload_version_test.go +++ /dev/null @@ -1,157 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package dataset_api - -import ( - "context" - "encoding/json" - "net/http" - "net/http/httptest" - "os" - "path/filepath" - "strings" - "sync" - "testing" - - "github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -// uploadServer answers the three-step publish, refusing any version in taken -// and reporting whatever the listing is told to report. -type uploadServer struct { - mu sync.Mutex - taken map[string]bool - listing []string - attempts []string -} - -func (s *uploadServer) handler(t *testing.T, base func() string) http.HandlerFunc { - t.Helper() - return func(w http.ResponseWriter, r *http.Request) { - s.mu.Lock() - defer s.mu.Unlock() - w.Header().Set("Content-Type", "application/json") - - switch { - case strings.HasSuffix(r.URL.Path, "/startPendingUpload"): - version := strings.Split(r.URL.Path, "/versions/")[1] - version = strings.TrimSuffix(version, "/startPendingUpload") - s.attempts = append(s.attempts, version) - if s.taken[version] { - w.WriteHeader(http.StatusConflict) - _, _ = w.Write([]byte(`{"error":{"code":"Conflict"}}`)) - return - } - require.NoError(t, json.NewEncoder(w).Encode(map[string]any{ - "blobReference": map[string]any{ - "blobUri": base() + "/c", - "storageAccountArmId": "id", - "credential": map[string]any{"sasUri": base() + "/c?sig=x"}, - }, - })) - - case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/versions"): - values := []map[string]any{} - for _, v := range s.listing { - values = append(values, map[string]any{"name": "ds", "version": v}) - } - require.NoError(t, json.NewEncoder(w).Encode(map[string]any{"value": values})) - - case r.Method == http.MethodPut: - version := r.URL.Path[strings.LastIndex(r.URL.Path, "/")+1:] - s.taken[version] = true - require.NoError(t, json.NewEncoder(w).Encode(map[string]any{ - "name": "ds", "version": version, - })) - - default: - // The blob PUT. - w.WriteHeader(http.StatusCreated) - } - } -} - -// The version listing lags a publish, so a second upload can be told the -// dataset is new and restart at a version that already exists. Trusting the -// listing alone surfaced that 409 to the user for a publish that should simply -// have added a version. -func TestUploadNextVersionWalksPastAStaleListing(t *testing.T) { - server := &uploadServer{taken: map[string]bool{"1.0": true}} - // The listing has not caught up: it still reports nothing at all. - httpServer := func() *httptest.Server { - var s *httptest.Server - s = httptest.NewServer(server.handler(t, func() string { return s.URL })) - return s - }() - t.Cleanup(httpServer.Close) - - client := NewDatasetClientFromPipeline( - httpServer.URL, runtime.NewPipeline("test", "v1", runtime.PipelineOptions{}, nil)) - - dir := t.TempDir() - require.NoError(t, os.WriteFile( - filepath.Join(dir, "rows.jsonl"), []byte("{\"query\":\"q\"}\n"), 0o600)) - - ds, err := client.UploadNextVersion(context.Background(), "ds", "", dir, "2025-11-15-preview") - require.NoError(t, err, "a stale listing must not surface as a conflict") - assert.Equal(t, "2.0", ds.Version) - assert.Equal(t, []string{"1.0", "2.0"}, server.attempts, - "the version just refused is proof it exists, so the next one is tried") -} - -// When the listing has caught up and is further ahead than the refused -// version, it is the better answer: it skips versions somebody else published. -func TestUploadNextVersionPrefersACaughtUpListing(t *testing.T) { - server := &uploadServer{ - taken: map[string]bool{"1.0": true, "2.0": true, "3.0": true}, - listing: []string{"1.0", "2.0", "3.0"}, - } - httpServer := func() *httptest.Server { - var s *httptest.Server - s = httptest.NewServer(server.handler(t, func() string { return s.URL })) - return s - }() - t.Cleanup(httpServer.Close) - - client := NewDatasetClientFromPipeline( - httpServer.URL, runtime.NewPipeline("test", "v1", runtime.PipelineOptions{}, nil)) - - dir := t.TempDir() - require.NoError(t, os.WriteFile( - filepath.Join(dir, "rows.jsonl"), []byte("{\"query\":\"q\"}\n"), 0o600)) - - ds, err := client.UploadNextVersion(context.Background(), "ds", "", dir, "2025-11-15-preview") - require.NoError(t, err) - assert.Equal(t, "4.0", ds.Version) -} - -// A service that refuses everything must end in the conflict rather than -// looping: an unbounded walk would hammer the service on a real failure. -func TestUploadNextVersionGivesUpBounded(t *testing.T) { - server := &uploadServer{taken: map[string]bool{}} - for _, v := range []string{"1.0", "2.0", "3.0", "4.0", "5.0", "6.0"} { - server.taken[v] = true - } - httpServer := func() *httptest.Server { - var s *httptest.Server - s = httptest.NewServer(server.handler(t, func() string { return s.URL })) - return s - }() - t.Cleanup(httpServer.Close) - - client := NewDatasetClientFromPipeline( - httpServer.URL, runtime.NewPipeline("test", "v1", runtime.PipelineOptions{}, nil)) - - dir := t.TempDir() - require.NoError(t, os.WriteFile( - filepath.Join(dir, "rows.jsonl"), []byte("{\"query\":\"q\"}\n"), 0o600)) - - _, err := client.UploadNextVersion(context.Background(), "ds", "", dir, "2025-11-15-preview") - require.Error(t, err) - assert.True(t, IsVersionConflict(err)) - assert.Len(t, server.attempts, versionConflictAttempts) -} diff --git a/cli/azd/extensions/azure.ai.dataset/internal/pkg/dataset_api/uri_test.go b/cli/azd/extensions/azure.ai.dataset/internal/pkg/dataset_api/uri_test.go deleted file mode 100644 index e6cc46c2e3d..00000000000 --- a/cli/azd/extensions/azure.ai.dataset/internal/pkg/dataset_api/uri_test.go +++ /dev/null @@ -1,75 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package dataset_api - -import ( - "encoding/json" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -// The service spells these fields inconsistently, and a URI read from the -// wrong spelling comes back empty rather than wrong — which is how the dataset -// URI went unbound the first time. -func TestDatasetResolvedBlobURI_AcceptsEitherSpelling(t *testing.T) { - cases := map[string]string{ - `{"dataUri":"https://x/y.jsonl"}`: "https://x/y.jsonl", - `{"data_uri":"https://x/y.jsonl"}`: "https://x/y.jsonl", - `{"blobUri":"https://x/b.jsonl"}`: "https://x/b.jsonl", - `{"contentUri":"https://x/c.jsonl"}`: "https://x/c.jsonl", - } - for body, want := range cases { - var ds Dataset - require.NoError(t, json.Unmarshal([]byte(body), &ds), body) - assert.Equal(t, want, ds.ResolvedBlobURI(), body) - } - - var none Dataset - require.NoError(t, json.Unmarshal([]byte(`{"name":"x"}`), &none)) - assert.Empty(t, none.ResolvedBlobURI(), - "no URI means the caller has to fetch a credential, not that the dataset is unreadable") -} - -// An upload needs the SAS-bearing URI to write to and the plain one to -// finalize with. Confusing them fails at different stages, so both are read -// from their own place. -func TestPendingUploadURIs(t *testing.T) { - var p PendingUploadResponse - require.NoError(t, json.Unmarshal([]byte(`{ - "blobReference": { - "blobUri": "https://acct.blob.core.windows.net/container", - "credential": { "sasUri": "https://acct.blob.core.windows.net/container?sig=abc" } - } - }`), &p)) - - assert.Equal(t, "https://acct.blob.core.windows.net/container?sig=abc", p.ResolvedUploadURI(), - "the upload target carries the SAS") - assert.Equal(t, "https://acct.blob.core.windows.net/container", p.ResolvedBlobURI(), - "the finalize URI does not") - - var empty PendingUploadResponse - assert.Empty(t, empty.ResolvedUploadURI()) - assert.Empty(t, empty.ResolvedBlobURI()) -} - -// Credentials arrive in two shapes and the consumption one takes precedence, -// because that is the one scoped for reading. -func TestCredentialResolvedDownloadURI(t *testing.T) { - var c DatasetCredential - require.NoError(t, json.Unmarshal([]byte(`{ - "blobReferenceForConsumption": { "credential": { "sasUri": "https://acct/read?sig=r" } }, - "blobReference": { "credential": { "sasUri": "https://acct/write?sig=w" } } - }`), &c)) - assert.Equal(t, "https://acct/read?sig=r", c.ResolvedDownloadURI()) - - var legacy DatasetCredential - require.NoError(t, json.Unmarshal([]byte(`{"sas_uri":"https://acct/legacy?sig=l"}`), &legacy)) - assert.Equal(t, "https://acct/legacy?sig=l", legacy.ResolvedDownloadURI(), - "the flat spelling is still honoured") - - var none DatasetCredential - assert.Empty(t, none.ResolvedDownloadURI()) -} diff --git a/cli/azd/extensions/azure.ai.dataset/internal/pkg/dataset_api/version_test.go b/cli/azd/extensions/azure.ai.dataset/internal/pkg/dataset_api/version_test.go deleted file mode 100644 index 052a63ae504..00000000000 --- a/cli/azd/extensions/azure.ai.dataset/internal/pkg/dataset_api/version_test.go +++ /dev/null @@ -1,68 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package dataset_api - -import ( - "testing" - - "github.com/stretchr/testify/require" -) - -// Drift detection compares the version on the service with the one recorded at -// the last deploy, so the ordering has to be numeric rather than lexical: -// "10.0" is newer than "9.0" even though it sorts earlier as a string. -func TestVersionGreater(t *testing.T) { - cases := []struct { - a, b string - want bool - }{ - {"2.0", "1.0", true}, - {"1.0", "2.0", false}, - {"1.0", "1.0", false}, - {"10.0", "9.0", true}, - {"9.0", "10.0", false}, - {"v3", "v2", true}, - } - - for _, tc := range cases { - require.Equal(t, tc.want, VersionGreater(tc.a, tc.b), - "VersionGreater(%q, %q)", tc.a, tc.b) - } -} - -// An unorderable version must never trigger a drift failure on its own: the -// deploy would be blocked with no way for the author to reason about it. -func TestVersionGreaterIgnoresUnorderable(t *testing.T) { - require.False(t, VersionGreater("draft", "1.0")) - require.False(t, VersionGreater("1.0", "draft")) - require.False(t, VersionGreater("", "1.0")) - require.False(t, VersionGreater("1.0", "")) -} - -// The two upload entry points read their version argument differently, and the -// difference is the whole point: UploadNewVersion counts from it, UploadVersion -// writes it. Passing "1.0" to the counting one publishes 2.0, which is not what -// an author who wrote version: "1.0" asked for. -func TestNextVersionCountsFromTheArgument(t *testing.T) { - if got := NextVersion("1.0"); got != "2.0" { - t.Fatalf("NextVersion(1.0) = %q, want 2.0", got) - } - if got := NextVersion("1"); got != "2.0" { - t.Fatalf("NextVersion(1) = %q, want 2.0", got) - } - // An unknown current version starts the sequence rather than guessing. - if got := NextVersion(""); got != "1.0" { - t.Fatalf("NextVersion(empty) = %q, want 1.0", got) - } -} - -func TestLatestVersionOrdersNumerically(t *testing.T) { - got := LatestVersion([]Dataset{{Version: "1.0"}, {Version: "10.0"}, {Version: "2.0"}}) - if got != "10.0" { - t.Fatalf("LatestVersion = %q, want 10.0 (numeric, not lexical)", got) - } - if LatestVersion(nil) != "" { - t.Fatal("LatestVersion(nil) should be empty") - } -} diff --git a/cli/azd/extensions/azure.ai.dataset/internal/version/version.go b/cli/azd/extensions/azure.ai.dataset/internal/version/version.go deleted file mode 100644 index e7279d11fba..00000000000 --- a/cli/azd/extensions/azure.ai.dataset/internal/version/version.go +++ /dev/null @@ -1,11 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package version - -var ( - // Populated at build time. - Version = "dev" - Commit = "none" - BuildDate = "unknown" -) diff --git a/cli/azd/extensions/azure.ai.dataset/main.go b/cli/azd/extensions/azure.ai.dataset/main.go deleted file mode 100644 index 9f66d1148be..00000000000 --- a/cli/azd/extensions/azure.ai.dataset/main.go +++ /dev/null @@ -1,14 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package main - -import ( - "azureaidataset/internal/cmd" - - "github.com/azure/azure-dev/cli/azd/pkg/azdext" -) - -func main() { - azdext.Run(cmd.NewRootCommand()) -} diff --git a/cli/azd/extensions/azure.ai.dataset/version.txt b/cli/azd/extensions/azure.ai.dataset/version.txt deleted file mode 100644 index e864273872d..00000000000 --- a/cli/azd/extensions/azure.ai.dataset/version.txt +++ /dev/null @@ -1 +0,0 @@ -1.0.0-beta.1 \ No newline at end of file diff --git a/eng/pipelines/release-ext-azure-ai-dataset.yml b/eng/pipelines/release-ext-azure-ai-dataset.yml deleted file mode 100644 index dbcebc827c1..00000000000 --- a/eng/pipelines/release-ext-azure-ai-dataset.yml +++ /dev/null @@ -1,45 +0,0 @@ -# Continuous deployment trigger -trigger: - branches: - include: - - main - paths: - include: - - cli/azd/extensions/azure.ai.dataset - - /eng/pipelines/templates/stages/release-azd-extension.yml - - /eng/pipelines/templates/jobs/build-azd-extension.yml - - /eng/pipelines/templates/jobs/cross-build-azd-extension.yml - - /eng/pipelines/templates/variables/image.yml - -pr: - paths: - include: - - cli/azd/extensions/azure.ai.dataset - - eng/pipelines/release-ext-azure-ai-dataset.yml - - /eng/pipelines/templates/stages/release-azd-extension.yml - - eng/pipelines/templates/steps/publish-cli.yml - exclude: - - cli/azd/docs/** - -parameters: - - name: PublishToRegistry - displayName: Publish to registry - type: string - # Scheduled (nightly) runs override this in the shared templates; the runtime - # parameter default must be a literal because it renders before variables exist. - default: stable - values: - - stable - - dev - - nightly - -extends: - template: /eng/pipelines/templates/stages/1es-redirect.yml - parameters: - stages: - - template: /eng/pipelines/templates/stages/release-azd-extension.yml - parameters: - AzdExtensionId: azure.ai.dataset - SanitizedExtensionId: azure-ai-dataset - AzdExtensionDirectory: cli/azd/extensions/azure.ai.dataset - PublishToRegistry: ${{ parameters.PublishToRegistry }} From 4c7cf085c0b63f6f77901e9bdb05939ce115cfd9 Mon Sep 17 00:00:00 2001 From: mohessie Date: Sun, 9 Aug 2026 19:07:36 +0300 Subject: [PATCH 138/320] Run the evals init already knows how to write init --source traces wrote source: {type: traces, agent_name} and run start refused it with 'does not name a target agent': nothing in the run path read source:, and the only data source the extension could build was an agent target. The first hero scenario dead-ended one command in. buildRunDataSource now decides in the order the configuration does. A source: block hands gathering to the service and sends nothing local; otherwise rows come from the dataset and target: says what to invoke for each -- including nothing, when the rows already hold both sides. Adds the three data sources that were missing beside the agent one: traces, stored responses, dataset-only, and a model target that was validated as allowed but silently sent as azure_ai_agent. Verified live: a trace-backed eval scored 5 real conversations. --- .../azure.ai.evaluations/internal/cmd/run.go | 66 ++++++++++++++-- .../internal/messages/messages.go | 61 ++++++++++++++ .../internal/pkg/eval_api/models.go | 79 +++++++++++++++++++ 3 files changed, 198 insertions(+), 8 deletions(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go index 9c10fc7546f..c3b7668bbb0 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go @@ -17,6 +17,7 @@ import ( "strings" "time" + "azureaieval/internal/messages" "azureaieval/internal/pkg/dataset_api" "azureaieval/internal/pkg/eval_api" "azureaieval/internal/project" @@ -391,8 +392,12 @@ func (ec *evalContext) reuseDataSourceFromLastRun( return list.Data[0].DataSource, nil } -// buildRunDataSource binds the dataset to the run. The eval carries no -// dataset today, so it is supplied here. +// buildRunDataSource binds the eval's rows to the run. +// +// Three shapes, in the order the configuration decides them. A `source:` block +// hands the gathering to the service and sends nothing local. Otherwise the +// rows come from a dataset, and `target:` says what to invoke for each one — +// including nothing at all, when the rows already hold both sides. func (ec *evalContext) buildRunDataSource( ctx context.Context, group *project.Eval, @@ -400,16 +405,31 @@ func (ec *evalContext) buildRunDataSource( maxSamples int, ) (*eval_api.EvalRunDataSource, error) { if group == nil { - return nil, fmt.Errorf("no eval to run") + return nil, messages.NoEvalToRun() } - if group.Target == nil || group.Target.Name == "" { - return nil, fmt.Errorf("eval %q does not name a target agent", group.Name) + + if group.Source != nil { + switch group.Source.Type { + case project.SourceTypeTraces: + return tracesDataSource(group) + case project.SourceTypeResponses: + return responsesDataSource(group) + } } - ds := eval_api.NewAgentTargetDataSource(group.Target.Name, nil) + var ds *eval_api.EvalRunDataSource + switch { + case group.Target == nil || group.Target.Name == "": + // Nothing to invoke: the dataset is scored as it stands. + ds = eval_api.NewDatasetOnlyDataSource() + case group.Target.Type == project.TargetTypeModel: + ds = eval_api.NewModelTargetDataSource(group.Target.Name) + default: + ds = eval_api.NewAgentTargetDataSource(group.Target.Name, nil) + } if group.Dataset == "" { - return nil, fmt.Errorf("eval %q does not reference a dataset", group.Name) + return nil, messages.EvalHasNoDataset(group.Name) } // A local source is read from disk; anything else is already registered and @@ -431,12 +451,42 @@ func (ec *evalContext) buildRunDataSource( return nil, err } if len(items) == 0 { - return nil, fmt.Errorf("dataset file %q has no rows", localPath) + return nil, messages.DatasetFileEmpty(localPath) } ds.SetFileContent(items) return ds, nil } +// tracesDataSource evaluates conversations the agent already had. +// +// The service reads them from Application Insights, so the agent has to be +// emitting gen_ai.input.messages / gen_ai.output.messages for anything to be +// found. `agent_name` filters the traces; it is not a target, because a trace +// run invokes nothing. +func tracesDataSource(group *project.Eval) (*eval_api.EvalRunDataSource, error) { + agent := group.Source.AgentName + if agent == "" && group.Target != nil { + agent = group.Target.Name + } + if agent == "" { + return nil, messages.TracesNeedAgentName(group.Name) + } + return eval_api.NewTracesDataSource( + agent, + group.Source.LookbackHours, + time.Time{}, + group.Source.MaxTraces, + ), nil +} + +// responsesDataSource evaluates responses the project already stored. +func responsesDataSource(group *project.Eval) (*eval_api.EvalRunDataSource, error) { + if len(group.Source.ResponseIDs) == 0 { + return nil, messages.ResponsesNeedIDs(group.Name) + } + return eval_api.NewResponsesDataSource(group.Source.ResponseIDs, group.Source.MaxTurns), nil +} + // readRegisteredDataset fetches a published dataset's rows, optionally keeping // only the first n. // diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go b/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go new file mode 100644 index 00000000000..2f5d1782c14 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go @@ -0,0 +1,61 @@ +// Package messages holds every string this extension shows a user. +// +// One file, so the whole voice of the CLI can be reviewed in one sitting and a +// wording change never has to be hunted through the command tree. Nothing here +// imports anything from the extension, so any package can use it. +// +// Conventions, so the set stays consistent: +// +// - Errors state what went wrong and, where there is one, the way out. +// Lowercase, no trailing period: azd renders them after "ERROR: ". +// - A name the user chose is quoted with %q; an identifier the service +// assigned is not, because it is already unmistakable. +// - Progress and success lines are sentences with a capital and no period. +// - Nothing here decides *whether* to print. That stays at the call site. +package messages + +import ( + "errors" + "fmt" +) + +// --------------------------------------------------------------------------- +// Running an eval +// --------------------------------------------------------------------------- + +// NoEvalToRun reports a run with nothing resolved to run. +func NoEvalToRun() error { + return errors.New("no eval to run") +} + +// EvalHasNoDataset reports an eval whose rows cannot be located. +// +// Named separately from the traces and responses cases because the way out is +// different: this one is answered by a dataset, not by a source block. +func EvalHasNoDataset(eval string) error { + return fmt.Errorf( + "eval %q references no dataset and declares no source:. Add a dataset: to "+ + "score rows you supply, or a source: to score traces or stored responses", + eval) +} + +// DatasetFileEmpty reports a local dataset file that parsed but held no rows. +func DatasetFileEmpty(path string) error { + return fmt.Errorf("dataset file %q has no rows", path) +} + +// TracesNeedAgentName reports a trace-backed eval that does not say whose +// traces to read. +func TracesNeedAgentName(eval string) error { + return fmt.Errorf( + "eval %q reads traces but does not say whose. Set source.agent_name to the "+ + "agent whose conversations should be evaluated", + eval) +} + +// ResponsesNeedIDs reports a stored-response eval with nothing to retrieve. +func ResponsesNeedIDs(eval string) error { + return fmt.Errorf( + "eval %q evaluates stored responses but lists none. Set source.response_ids", + eval) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/models.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/models.go index 8f6df9d6bf9..1f09adefaed 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/models.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/models.go @@ -8,6 +8,7 @@ import ( "fmt" "sort" "strings" + "time" ) // --------------------------------------------------------------------------- @@ -394,6 +395,84 @@ func NewAgentTargetDataSource(agentName string, agentVersion *string) *EvalRunDa } } +// NewTracesDataSource evaluates an agent's recorded traces instead of a dataset. +// +// The window is a lookback in hours. The service has no start bound on this +// data source: a start_time is accepted and dropped, leaving the default seven +// days in place, so the conversion happens here rather than being left to look +// like it worked. +func NewTracesDataSource(agentName string, lookbackHours int, end time.Time, maxTraces int) *EvalRunDataSource { + ds := &EvalRunDataSource{ + Type: EvalRunDataSourceTypeTraces, + AgentName: agentName, + LookbackHours: lookbackHours, + MaxTraces: maxTraces, + } + if !end.IsZero() { + ds.EndTime = end.Unix() + } + return ds +} + +// NewDatasetOnlyDataSource scores the dataset as it stands, invoking nothing. +// +// Used when an eval declares no target: the rows already hold both sides of the +// exchange, which is how a recorded conversation is evaluated. +func NewDatasetOnlyDataSource() *EvalRunDataSource { + return &EvalRunDataSource{Type: EvalRunDataSourceTypeJSONL} +} + +// NewModelTargetDataSource sends the dataset's questions straight to a model +// deployment, with no agent in front of it. +// +// The model answers as plain text, so an eval evaluating one has to bind its +// response to {{sample.output_text}} rather than the richer output an agent +// produces. +func NewModelTargetDataSource(model string) *EvalRunDataSource { + return &EvalRunDataSource{ + Type: EvalRunDataSourceTypeAgentTarget, + InputMessages: &EvalRunInputMessages{ + Type: "template", + Template: []EvalRunMessageTemplate{ + { + Role: "user", + Content: "{{item.query}}", + Type: "message", + }, + }, + }, + Target: &EvalRunTarget{ + Type: "azure_ai_model", + Model: model, + }, + } +} + +// NewResponsesDataSource evaluates responses the project already stored. +// +// The ids travel as ordinary JSONL rows and a data_mapping points the service +// at the field holding each one, which is how it retrieves the chat history +// behind the response. +func NewResponsesDataSource(responseIDs []string, maxTurns int) *EvalRunDataSource { + rows := make([]map[string]any, 0, len(responseIDs)) + for _, id := range responseIDs { + rows = append(rows, map[string]any{"item": map[string]any{"response_id": id}}) + } + + return &EvalRunDataSource{ + Type: EvalRunDataSourceTypeResponses, + ItemGenerationParams: &ItemGenerationParams{ + Type: "response_retrieval", + MaxNumTurns: maxTurns, + DataMapping: map[string]string{"response_id": "{{item.response_id}}"}, + Source: &EvalRunDataContent{ + Type: EvalRunDataContentTypeFileContent, + Content: rows, + }, + }, + } +} + // SetFileContent sets the data source to use inline file content. // // There is no by-reference counterpart. A run's `file_id` means an uploaded From c2135cd60bfe9e2c276ed0fe03aa0d6d81d5214a Mon Sep 17 00:00:00 2001 From: mohessie Date: Sun, 9 Aug 2026 19:11:22 +0300 Subject: [PATCH 139/320] Cover the run data source, which nothing tested The trace path shipped broken because no test ever built a run data source. Ten cases: each of the four shapes, both refusals a source: block can hit, the model target that used to be sent as an agent, and --max-samples meaning the same thing wherever the rows come from. --- .../internal/cmd/run_datasource_test.go | 213 ++++++++++++++++++ 1 file changed, 213 insertions(+) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_datasource_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_datasource_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_datasource_test.go new file mode 100644 index 00000000000..9dac0df876d --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_datasource_test.go @@ -0,0 +1,213 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "context" + "os" + "path/filepath" + "testing" + + "azureaieval/internal/pkg/eval_api" + "azureaieval/internal/project" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// writeDataset drops a JSONL file beside a config that registers it, and +// returns the config path. An eval's dataset: is a catalog name, not a path, so +// the declaration is what makes the rows reachable. +func writeDataset(t *testing.T, rows string) string { + t.Helper() + dir := t.TempDir() + require.NoError(t, os.MkdirAll(filepath.Join(dir, "datasets"), 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(dir, "datasets", "d.jsonl"), []byte(rows), 0o600)) + configPath := filepath.Join(dir, "eval.yaml") + config := "datasets:\n - name: d\n source: ./datasets/d.jsonl\n" + require.NoError(t, os.WriteFile(configPath, []byte(config), 0o600)) + return configPath +} + +const oneRow = `{"query":"q","ground_truth":"a"}` + "\n" + +// A trace-backed eval is the first hero scenario, and it is the one shape that +// carries no dataset at all: the service gathers the rows itself. It used to be +// refused for "not naming a target agent", which is the whole point of it. +func TestBuildRunDataSource_Traces(t *testing.T) { + ec := &evalContext{} + group := &project.Eval{ + Name: "trace-eval", + Source: &project.SourceDecl{ + Type: project.SourceTypeTraces, + AgentName: "support-agent", + LookbackHours: 24, + MaxTraces: 500, + }, + } + + ds, err := ec.buildRunDataSource(context.Background(), group, "", 0) + + require.NoError(t, err) + assert.Equal(t, eval_api.EvalRunDataSourceTypeTraces, ds.Type) + assert.Equal(t, "support-agent", ds.AgentName) + assert.Equal(t, 24, ds.LookbackHours) + assert.Equal(t, 500, ds.MaxTraces) + // Nothing is invoked and nothing local is sent. + assert.Nil(t, ds.Target) + assert.Nil(t, ds.Source) +} + +// agent_name under source: is a filter, but an eval that names a target and +// leaves the filter off still means "this agent's traces". +func TestBuildRunDataSource_TracesFallsBackToTargetName(t *testing.T) { + ec := &evalContext{} + group := &project.Eval{ + Name: "trace-eval", + Source: &project.SourceDecl{Type: project.SourceTypeTraces}, + Target: &project.Target{Type: project.TargetTypeAgent, Name: "support-agent"}, + } + + ds, err := ec.buildRunDataSource(context.Background(), group, "", 0) + + require.NoError(t, err) + assert.Equal(t, "support-agent", ds.AgentName) +} + +// With neither, the run cannot say whose conversations to read, and saying so +// is more use than letting the service return nothing. +func TestBuildRunDataSource_TracesWithoutAnAgentIsRefused(t *testing.T) { + ec := &evalContext{} + group := &project.Eval{ + Name: "trace-eval", + Source: &project.SourceDecl{Type: project.SourceTypeTraces}, + } + + _, err := ec.buildRunDataSource(context.Background(), group, "", 0) + + require.Error(t, err) + assert.Contains(t, err.Error(), "source.agent_name") +} + +// Stored responses travel as rows carrying ids, with a data_mapping telling the +// service which field holds one. +func TestBuildRunDataSource_Responses(t *testing.T) { + ec := &evalContext{} + group := &project.Eval{ + Name: "replay", + Source: &project.SourceDecl{ + Type: project.SourceTypeResponses, + ResponseIDs: []string{"resp_1", "resp_2"}, + MaxTurns: 3, + }, + } + + ds, err := ec.buildRunDataSource(context.Background(), group, "", 0) + + require.NoError(t, err) + assert.Equal(t, eval_api.EvalRunDataSourceTypeResponses, ds.Type) + require.NotNil(t, ds.ItemGenerationParams) + assert.Equal(t, 3, ds.ItemGenerationParams.MaxNumTurns) + assert.Equal(t, + map[string]string{"response_id": "{{item.response_id}}"}, + ds.ItemGenerationParams.DataMapping) + require.NotNil(t, ds.ItemGenerationParams.Source) + assert.Len(t, ds.ItemGenerationParams.Source.Content, 2) +} + +func TestBuildRunDataSource_ResponsesWithoutIDsIsRefused(t *testing.T) { + ec := &evalContext{} + group := &project.Eval{ + Name: "replay", + Source: &project.SourceDecl{Type: project.SourceTypeResponses}, + } + + _, err := ec.buildRunDataSource(context.Background(), group, "", 0) + + require.Error(t, err) + assert.Contains(t, err.Error(), "source.response_ids") +} + +// No target means the rows already hold both sides of the exchange, so the run +// scores them as they stand rather than invoking anything. +func TestBuildRunDataSource_NoTargetScoresTheDatasetAsItStands(t *testing.T) { + ec := &evalContext{} + configPath := writeDataset(t, oneRow) + group := &project.Eval{Name: "recorded", Dataset: "d"} + + ds, err := ec.buildRunDataSource(context.Background(), group, configPath, 0) + + require.NoError(t, err) + assert.Equal(t, eval_api.EvalRunDataSourceTypeJSONL, ds.Type) + assert.Nil(t, ds.Target) + require.NotNil(t, ds.Source) + assert.Len(t, ds.Source.Content, 1) +} + +// A model target was accepted by config validation and then sent as +// azure_ai_agent, so the run failed against a resource that does not exist. +func TestBuildRunDataSource_ModelTargetIsSentAsAModel(t *testing.T) { + ec := &evalContext{} + configPath := writeDataset(t, oneRow) + group := &project.Eval{ + Name: "model-eval", + Dataset: "d", + Target: &project.Target{Type: project.TargetTypeModel, Name: "gpt-4o-mini"}, + } + + ds, err := ec.buildRunDataSource(context.Background(), group, configPath, 0) + + require.NoError(t, err) + require.NotNil(t, ds.Target) + assert.Equal(t, "azure_ai_model", ds.Target.Type) + assert.Equal(t, "gpt-4o-mini", ds.Target.Model) + assert.Empty(t, ds.Target.Name, "a model target is addressed by deployment, not by agent name") +} + +func TestBuildRunDataSource_AgentTarget(t *testing.T) { + ec := &evalContext{} + configPath := writeDataset(t, oneRow) + group := &project.Eval{ + Name: "agent-eval", + Dataset: "d", + Target: &project.Target{Type: project.TargetTypeAgent, Name: "support-agent"}, + } + + ds, err := ec.buildRunDataSource(context.Background(), group, configPath, 0) + + require.NoError(t, err) + assert.Equal(t, eval_api.EvalRunDataSourceTypeAgentTarget, ds.Type) + require.NotNil(t, ds.Target) + assert.Equal(t, "azure_ai_agent", ds.Target.Type) + assert.Equal(t, "support-agent", ds.Target.Name) +} + +// An eval with neither a dataset nor a source: has no rows from anywhere, and +// the error has to name both ways out rather than only the dataset. +func TestBuildRunDataSource_NoRowsFromAnywhere(t *testing.T) { + ec := &evalContext{} + + _, err := ec.buildRunDataSource(context.Background(), &project.Eval{Name: "empty"}, "", 0) + + require.Error(t, err) + assert.Contains(t, err.Error(), "dataset:") + assert.Contains(t, err.Error(), "source:") +} + +// --max-samples has to mean the same thing wherever the rows come from. +func TestBuildRunDataSource_MaxSamplesCapsLocalRows(t *testing.T) { + ec := &evalContext{} + configPath := writeDataset(t, oneRow+oneRow+oneRow) + group := &project.Eval{ + Name: "capped", + Dataset: "d", + Target: &project.Target{Type: project.TargetTypeAgent, Name: "a"}, + } + + ds, err := ec.buildRunDataSource(context.Background(), group, configPath, 2) + + require.NoError(t, err) + require.NotNil(t, ds.Source) + assert.Len(t, ds.Source.Content, 2) +} From f190ee93fdbf35155d2949a20988812606a87451 Mon Sep 17 00:00:00 2001 From: mohessie Date: Sun, 9 Aug 2026 19:18:45 +0300 Subject: [PATCH 140/320] Take the generation model from the agent, as the spec says The spec makes --generation-model default to the target agent's deployment. It did not: generate refused without the flag, so the documented default was an error message instead. Only the service knows the deployment, so the requirement moves past the client build -- resolvePlan still refuses early when there is no agent to read one from, which keeps a missing model off the authentication round trip. --- .../internal/cmd/generate_commands.go | 35 ++++++++++++++++--- .../internal/cmd/generate_plan_test.go | 18 ++++++++-- .../internal/messages/messages.go | 12 +++++++ .../internal/pkg/eval_api/models.go | 11 ++++++ 4 files changed, 68 insertions(+), 8 deletions(-) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_commands.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_commands.go index a401d9db1fb..87696485faa 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_commands.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_commands.go @@ -4,11 +4,13 @@ package cmd import ( + "context" "fmt" "os" "path/filepath" "strings" + "azureaieval/internal/messages" "azureaieval/internal/project" "github.com/spf13/cobra" @@ -79,15 +81,15 @@ func resolvePlan(f *generateFlags, name string, defaultOutputDir string) (genera BaseDir: f.path, OutputDir: firstNonEmpty(f.outputDir, "./"+defaultOutputDir), } - if plan.Model == "" { - return plan, fmt.Errorf( - "a model deployment is required to generate: pass --generation-model") + if plan.Model == "" && plan.Agent == "" { + return plan, messages.GenerationModelRequired() } return plan, nil } -// prepareGeneration builds the client and settles the one input that needs it: -// the agent's published instructions, which only the service can supply. +// prepareGeneration builds the client and settles the two inputs that need it: +// the agent's published instructions, and its deployment when the caller named +// no model of its own. Only the service can supply either. func prepareGeneration( cmd *cobra.Command, f *generateFlags, @@ -106,9 +108,32 @@ func prepareGeneration( ec.Close() return nil, plan, err } + + if plan.Model == "" { + plan.Model = ec.agentDeployment(ctx, plan.Agent) + } + if plan.Model == "" { + ec.Close() + return nil, plan, messages.GenerationModelRequired() + } return ec, plan, nil } +// agentDeployment reads the deployment the target agent answers with. +// +// Best effort: a caller who named no model gets one error naming the flag, not +// two errors about an agent they never mentioned. +func (ec *evalContext) agentDeployment(ctx context.Context, agentName string) string { + if agentName == "" { + return "" + } + agent, err := ec.evalClient.GetAgent(ctx, agentName, ProjectEndpointAPIVersion) + if err != nil { + return "" + } + return agent.Model() +} + // declaredTarget reads the agent from the evaluation configuration, which is // where the target is already declared, so `generate` does not need it // repeated. Best effort: generation runs from the instruction alone when there diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_plan_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_plan_test.go index 275e194044d..26a96aad84e 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_plan_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_plan_test.go @@ -70,12 +70,24 @@ func TestResolvePlan_OutputDirDefaultsPerArtifact(t *testing.T) { require.Equal(t, "./from-flag", override.OutputDir) } -// Without a model there is nothing to bill the job against, and the refusal has -// to name the flag that supplies one. -func TestResolvePlan_RequiresAGenerationModel(t *testing.T) { +// A named target carries a deployment, and the spec makes it the default, so +// the plan settles without one and lets prepareGeneration read it. Refusing +// here would ask the caller for something the project already knows. +func TestResolvePlan_DefersToTheAgentForTheModel(t *testing.T) { f := evalsDir(t) f.target = "shop-agent" + plan, err := resolvePlan(f, "d", project.DefaultDatasetsDir) + require.NoError(t, err) + require.Empty(t, plan.Model, "the deployment is read from the agent, not guessed here") + require.Equal(t, "shop-agent", plan.Agent) +} + +// With no target either there is nothing to read a deployment from, so the +// refusal happens before authentication and names the flag that supplies one. +func TestResolvePlan_RequiresAGenerationModelWithNoAgent(t *testing.T) { + f := evalsDir(t) + _, err := resolvePlan(f, "d", project.DefaultDatasetsDir) require.Error(t, err) require.Contains(t, err.Error(), "--generation-model") diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go b/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go index 2f5d1782c14..bdc0865c9ea 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go @@ -59,3 +59,15 @@ func ResponsesNeedIDs(eval string) error { "eval %q evaluates stored responses but lists none. Set source.response_ids", eval) } + +// --------------------------------------------------------------------------- +// Generation +// --------------------------------------------------------------------------- + +// GenerationModelRequired reports a generation with no deployment to run on. +// +// Reached only when the target agent could not supply one either, so the flag +// is the whole of the way out. +func GenerationModelRequired() error { + return errors.New("a model deployment is required to generate: pass --generation-model") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/models.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/models.go index 1f09adefaed..033fcff35ff 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/models.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/models.go @@ -78,6 +78,17 @@ func (a *Agent) Instructions() string { return strings.TrimSpace(a.Versions.Latest.Definition.Instructions) } +// Model returns the newest version's deployment, or "" when the agent has no +// published version. It is what generation falls back to when the caller names +// no deployment of its own: the model already judged good enough to answer as +// this agent is the sensible default for writing its test cases. +func (a *Agent) Model() string { + if a == nil || a.Versions.Latest == nil { + return "" + } + return strings.TrimSpace(a.Versions.Latest.Definition.Model) +} + // GenerationJob is the response for data and evaluator generation job operations. type GenerationJob struct { ID string `json:"id"` From 8946cdd42b340ed2b241841604d914533e1f7b13 Mon Sep 17 00:00:00 2001 From: mohessie Date: Sun, 9 Aug 2026 19:26:43 +0300 Subject: [PATCH 141/320] Give run list the dataset the scenario compares on Scenario 3 reads a trend off two rows, and a pass rate means nothing without knowing what it scored. The run's own data source cannot answer it -- the rows travel inline, so the name that selected them is not in what the service keeps -- so the run records it at creation and the listing reads it back. Recorded per run rather than read from the configuration at list time: the dataset under an eval can change between runs, which is the one thing the column exists to reveal. --- .../azure.ai.evaluations/internal/cmd/run.go | 9 +++++ .../internal/cmd/run_dataset_column_test.go | 37 +++++++++++++++++++ .../internal/cmd/run_ops.go | 28 +++++++++++++- 3 files changed, 73 insertions(+), 1 deletion(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_dataset_column_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go index c3b7668bbb0..e0eec217b6c 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go @@ -161,6 +161,15 @@ func buildRunCommand(use, short string) *cobra.Command { if lvl := resolveLevel(group); lvl != "" { metadata["evaluation_level"] = lvl } + // Recorded per run, not read from the configuration at list time: + // comparing two runs is the point of that listing, and the dataset + // under an eval can change between them. + if group != nil && group.Dataset != "" { + metadata[metaDataset] = group.Dataset + if v := ec.getEnvValue(ctx, versionKey("dataset", group.Dataset)); v != "" { + metadata[metaDatasetVersion] = v + } + } run, err := ec.evalClient.CreateOpenAIEvalRun(ctx, evalID, &eval_api.CreateOpenAIEvalRunRequest{ Name: runName, diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_dataset_column_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_dataset_column_test.go new file mode 100644 index 00000000000..f4f8864aaf0 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_dataset_column_test.go @@ -0,0 +1,37 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +// Scenario 3 reads a trend off two rows of `run list`, and the dataset column +// is what makes the comparison honest: two rates over different rows are not +// the same claim. +func TestRunDataset_NameAndVersion(t *testing.T) { + got := runDataset(map[string]string{ + metaDataset: "support-agent-regression", + metaDatasetVersion: "1", + }) + + assert.Equal(t, "support-agent-regression (v1)", got) +} + +// A dataset recorded before it was published has a name but no version yet. +func TestRunDataset_NameWithoutVersion(t *testing.T) { + got := runDataset(map[string]string{metaDataset: "support-golden"}) + + assert.Equal(t, "support-golden", got) +} + +// Runs started before the extension recorded this show nothing. Falling back to +// the configuration would print today's dataset against a run that scored a +// different one, which is exactly the drift the column exists to reveal. +func TestRunDataset_UnrecordedShowsNothing(t *testing.T) { + assert.Empty(t, runDataset(nil)) + assert.Empty(t, runDataset(map[string]string{"evaluation_level": "turn"})) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go index 71180968fac..df3038bdb19 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go @@ -76,6 +76,7 @@ func newRunListCommand() *cobra.Command { for _, run := range list.Data { rows = append(rows, []string{ run.ID, + runDataset(run.Metadata), timestampString(run.CreatedAt), run.Status, sampleCount(run.ResultCounts), @@ -83,7 +84,7 @@ func newRunListCommand() *cobra.Command { }) } return emitTable(cmd.OutOrStdout(), - []string{"RUN", "STARTED", "STATUS", "SAMPLES", "PASS RATE"}, rows) + []string{"RUN", "DATASET", "STARTED", "STATUS", "SAMPLES", "PASS RATE"}, rows) }, } addEvalFlag(cmd, &groupName) @@ -309,6 +310,31 @@ func summarizeCounts(counts *eval_api.EvalRunResultCounts) string { counts.Passed, counts.Failed, counts.Errored) } +// metaDataset and metaDatasetVersion record which rows a run scored. The run's +// own data source cannot answer it: the rows travel inline, so the name that +// selected them is not in the request the service keeps. +const ( + metaDataset = "azd_dataset" + metaDatasetVersion = "azd_dataset_version" +) + +// runDataset renders the dataset a run scored, versioned when a version was +// recorded with it. +// +// A run started before this was recorded shows nothing rather than the name in +// the configuration today, which is the one thing the column exists to detect +// having changed. +func runDataset(metadata map[string]string) string { + name := metadata[metaDataset] + if name == "" { + return "" + } + if version := metadata[metaDatasetVersion]; version != "" { + return fmt.Sprintf("%s (v%s)", name, version) + } + return name +} + // sampleCount is how many rows the run scored, which is what makes two rows of // `run list` comparable: a rate over 15 samples and one over 200 are not the // same claim. From 7c991ab2d6aa7cc6eefe06a6f375fe7dfb703c19 Mon Sep 17 00:00:00 2001 From: mohessie Date: Sun, 9 Aug 2026 20:12:05 +0300 Subject: [PATCH 142/320] Put every message the CLI shows in one file The extension's wording was spread across 31 files, so reviewing its voice meant reading the whole command tree, and a phrasing change meant finding every place that phrased it. internal/messages already held six messages; this moves the rest there -- errors, warnings, and the progress and success lines -- as named functions grouped by area, and leaves the call sites saying only which situation they are in. Wording is unchanged except where two messages described one situation. --- .../internal/cmd/build.go | 42 +- .../internal/cmd/catalog.go | 13 +- .../internal/cmd/context.go | 13 +- .../internal/cmd/dataset.go | 35 +- .../internal/cmd/eval_group.go | 19 +- .../internal/cmd/evalref.go | 10 +- .../internal/cmd/evaluator.go | 45 +- .../internal/cmd/gating.go | 20 +- .../internal/cmd/generate.go | 78 +- .../internal/cmd/generate_commands.go | 6 +- .../azure.ai.evaluations/internal/cmd/init.go | 60 +- .../azure.ai.evaluations/internal/cmd/job.go | 21 +- .../internal/cmd/output.go | 15 +- .../internal/cmd/reconciler.go | 57 +- .../azure.ai.evaluations/internal/cmd/run.go | 70 +- .../internal/cmd/run_ops.go | 35 +- .../internal/cmd/run_output.go | 45 +- .../internal/foundry/projectctx/store.go | 8 +- .../internal/foundry/projectctx/validator.go | 45 +- .../internal/messages/messages.go | 1689 ++++++++++++++++- .../internal/pkg/dataset_api/models.go | 9 +- .../internal/pkg/dataset_api/operations.go | 69 +- .../internal/pkg/eval_api/insights.go | 4 +- .../internal/pkg/eval_api/operations.go | 29 +- .../internal/pkg/eval_api/poller.go | 13 +- .../internal/pkg/eval_api/portal_urls.go | 13 +- .../internal/pkg/evalcore/evaluator.go | 28 +- .../internal/project/agent_instructions.go | 17 +- .../internal/project/artifacts.go | 11 +- .../internal/project/eval_config.go | 72 +- .../internal/project/eval_config_store.go | 13 +- .../internal/project/service_target_eval.go | 37 +- 32 files changed, 2071 insertions(+), 570 deletions(-) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build.go index cdb6b5c3c4b..fcbb52d689d 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build.go @@ -7,9 +7,9 @@ import ( "context" "fmt" "maps" - "sort" "strings" + "azureaieval/internal/messages" "azureaieval/internal/pkg/eval_api" "azureaieval/internal/pkg/evalcore" "azureaieval/internal/project" @@ -197,18 +197,12 @@ func planCriterion( } } if len(missing) > 0 { - return nil, fmt.Errorf( - "evaluator %q requires %s, which the dataset does not provide; "+ - "add %s to the dataset, or bind it with `data_mapping`", - ref.Evaluator, quoteList(missing), pluralColumns(missing), - ) + return nil, messages.EvaluatorNeedsFields(ref.Evaluator, missing) } if !schema.SupportsLevel(level) { - return nil, fmt.Errorf( - "evaluator %q does not support evaluation level %q; it supports %s", - ref.Evaluator, level, quoteList(schema.SupportedEvaluationLevels), - ) + return nil, messages.EvaluatorLevelUnsupported( + ref.Evaluator, level, schema.SupportedEvaluationLevels) } initSchema := schema.InitSchema() @@ -246,11 +240,7 @@ func planCriterion( } } if len(missingInit) > 0 { - return nil, fmt.Errorf( - "evaluator %q requires %s; set it under the evaluator's "+ - "`initialization_parameters` in the eval config", - ref.Evaluator, quoteList(missingInit), - ) + return nil, messages.EvaluatorNeedsInitParams(ref.Evaluator, missingInit) } } @@ -382,25 +372,3 @@ func itemSchema(fields map[string]bool) map[string]any { "properties": properties, } } - -func quoteList(values []string) string { - if len(values) == 0 { - return "nothing" - } - quoted := make([]string, 0, len(values)) - for _, value := range values { - quoted = append(quoted, fmt.Sprintf("%q", value)) - } - sort.Strings(quoted) - if len(quoted) == 1 { - return quoted[0] - } - return strings.Join(quoted[:len(quoted)-1], ", ") + " and " + quoted[len(quoted)-1] -} - -func pluralColumns(values []string) string { - if len(values) == 1 { - return "that column" - } - return "those columns" -} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/catalog.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/catalog.go index 815c84b23c2..5d0dd8d50d3 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/catalog.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/catalog.go @@ -7,6 +7,7 @@ import ( "fmt" "path/filepath" + "azureaieval/internal/messages" "azureaieval/internal/project" "github.com/spf13/cobra" @@ -96,11 +97,10 @@ func updateCatalog( if !isJSON(cmd) { path := filepath.ToSlash(project.EvalConfigPath(evalDir)) if created { - fmt.Fprintf(cmd.OutOrStdout(), - "%s Created %s with the catalog entry\n", doneMark, path) + fmt.Fprint(cmd.OutOrStdout(), messages.CreatedCatalogFile(path)) } - fmt.Fprintf(cmd.OutOrStdout(), - "%s Added %s %s to %s\n", doneMark, kind, describeArtifact(ref), path) + fmt.Fprint(cmd.OutOrStdout(), + messages.AddedToCatalog(kind, describeArtifact(ref), path)) } return nil } @@ -111,8 +111,5 @@ func updateCatalog( // Single-quoted to match the spec's transcripts; the surrounding Done: lines // carry bare values, but a dataset name can hold a space and these cannot. func describeArtifact(ref *project.ArtifactRef) string { - if ref.Version == "" || ref.Version == "latest" { - return fmt.Sprintf("'%s'", ref.Name) - } - return fmt.Sprintf("'%s' (version %s)", ref.Name, ref.Version) + return messages.ArtifactDescription(ref.Name, ref.Version) } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/context.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/context.go index 75a6647f2d8..985ef5be6b2 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/context.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/context.go @@ -5,12 +5,11 @@ package cmd import ( "context" - "errors" - "fmt" "log" "strings" "azureaieval/internal/foundry/projectctx" + "azureaieval/internal/messages" "azureaieval/internal/pkg/dataset_api" "azureaieval/internal/pkg/eval_api" "azureaieval/internal/project" @@ -47,7 +46,7 @@ type evalContext struct { func newEvalContext(ctx context.Context, endpointFlag string) (*evalContext, error) { azdClient, err := azdext.NewAzdClient() if err != nil { - return nil, fmt.Errorf("connecting to azd: %w", err) + return nil, messages.ConnectingToAzd(err) } ec := &evalContext{azdClient: azdClient} @@ -70,7 +69,7 @@ func newEvalContext(ctx context.Context, endpointFlag string) (*evalContext, err &azidentity.AzureDeveloperCLICredentialOptions{}, ) if err != nil { - return nil, fmt.Errorf("creating Azure credential: %w", err) + return nil, messages.CreatingCredential(err) } ec.cred = cred @@ -102,7 +101,7 @@ func lookupEndpointFromAzd(ctx context.Context, azdClient *azdext.AzdClient) (en // The atomic commands are meant to work standalone against the data plane, so // running outside a project is ordinary rather than a problem worth reporting. // A write that fails for any other reason still is. -var errNoAzdEnvironment = errors.New("no active azd environment") +var errNoAzdEnvironment = messages.ErrNoAzdEnvironment // setEnvValue persists a value into the active azd environment. azd itself // writes none of these keys — the extension owns them. @@ -110,7 +109,7 @@ func (ec *evalContext) setEnvValue(ctx context.Context, key, value string) error if ec.envName == "" { envResp, err := ec.azdClient.Environment().GetCurrent(ctx, &azdext.EmptyRequest{}) if err != nil || envResp == nil || envResp.Environment == nil { - return fmt.Errorf("%w to write %s into", errNoAzdEnvironment, key) + return messages.NoAzdEnvironmentToWrite(key) } ec.envName = envResp.Environment.Name } @@ -120,7 +119,7 @@ func (ec *evalContext) setEnvValue(ctx context.Context, key, value string) error Value: value, }) if err != nil { - return fmt.Errorf("writing %s to the azd environment: %w", key, err) + return messages.WritingEnvValue(key, err) } return nil } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset.go index d131794be2c..7fc586febef 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset.go @@ -10,6 +10,7 @@ import ( "path/filepath" "strings" + "azureaieval/internal/messages" "azureaieval/internal/pkg/dataset_api" "azureaieval/internal/pkg/eval_api" @@ -106,7 +107,7 @@ func newDatasetWriteCommand(verb, short string) *cobra.Command { ctx, name, version, localDir, ProjectEndpointAPIVersion, ) if err != nil { - return fmt.Errorf("registering dataset %q: %w", name, err) + return messages.RegisteringDataset(name, err) } if err := ec.setEnvValue(ctx, envKeyDatasetVersion, ds.Version); err != nil { @@ -115,14 +116,14 @@ func newDatasetWriteCommand(verb, short string) *cobra.Command { // stderr, and is skipped outside a project, where having nowhere // to persist is expected rather than notable. if !errors.Is(err, errNoAzdEnvironment) && !isJSON(cmd) { - fmt.Fprintf(cmd.OutOrStdout(), "warning: %v\n", err) + fmt.Fprint(cmd.OutOrStdout(), messages.Warning(err)) } } if isJSON(cmd) { return emitJSON(cmd.OutOrStdout(), ds) } - fmt.Fprintf(cmd.OutOrStdout(), "Registered dataset %s version %s\n", ds.Name, ds.Version) + fmt.Fprint(cmd.OutOrStdout(), messages.DatasetRegistered(ds.Name, ds.Version)) return nil }, } @@ -139,14 +140,13 @@ func newDatasetWriteCommand(verb, short string) *cobra.Command { func datasetUploadDir(path string) (string, error) { info, err := os.Stat(path) if err != nil { - return "", fmt.Errorf("reading --from-file %q: %w", path, err) + return "", messages.ReadingFromFile(path, err) } if info.IsDir() { return path, nil } if !strings.EqualFold(filepath.Ext(path), ".jsonl") { - return "", fmt.Errorf( - "--from-file must be a .jsonl file or a directory containing one, got %q", path) + return "", messages.FromFileMustBeJSONL(path) } return filepath.Dir(path), nil } @@ -168,7 +168,7 @@ func newDatasetListCommand() *cobra.Command { list, err := ec.datasetClient.ListDatasets(ctx, ProjectEndpointAPIVersion) if err != nil { - return fmt.Errorf("listing datasets: %w", err) + return messages.ListingDatasets(err) } return renderDatasets(cmd, list) }, @@ -208,7 +208,7 @@ func newDatasetVersionsListCommand() *cobra.Command { list, err := ec.datasetClient.ListDatasetVersions(ctx, name, ProjectEndpointAPIVersion) if err != nil { - return fmt.Errorf("listing versions of dataset %q: %w", name, err) + return messages.ListingDatasetVersions(name, err) } return renderDatasets(cmd, list) }, @@ -227,7 +227,7 @@ func renderDatasets(cmd *cobra.Command, list *dataset_api.DatasetList) error { rows = append(rows, []string{d.Name, d.Version, d.Format}) } if len(rows) == 0 { - fmt.Fprintln(cmd.OutOrStdout(), "No datasets found.") + fmt.Fprint(cmd.OutOrStdout(), messages.NoDatasets()) return nil } return emitTable(cmd.OutOrStdout(), []string{"NAME", "VERSION", "FORMAT"}, rows) @@ -256,10 +256,10 @@ func newDatasetShowCommand() *cobra.Command { if version == "" { list, err := ec.datasetClient.ListDatasetVersions(ctx, name, ProjectEndpointAPIVersion) if err != nil { - return fmt.Errorf("resolving the latest version of %q: %w", name, err) + return messages.ResolvingLatestDatasetVersion(name, err) } if len(list.Value) == 0 { - return fmt.Errorf("dataset %q has no versions", name) + return messages.DatasetHasNoVersions(name) } version = dataset_api.LatestVersion(list.Value) } @@ -267,11 +267,9 @@ func newDatasetShowCommand() *cobra.Command { ds, err := ec.datasetClient.GetDataset(ctx, name, version, ProjectEndpointAPIVersion) if err != nil { if eval_api.IsNotFound(err) { - return fmt.Errorf( - "no dataset %q at version %q in this project; "+ - "`azd ai eval dataset list` shows the ones there are", name, version) + return messages.DatasetVersionNotFoundWithHint(name, version) } - return fmt.Errorf("reading dataset %q version %q: %w", name, version, err) + return messages.ReadingDatasetVersion(name, version, err) } if isJSON(cmd) { @@ -324,10 +322,9 @@ func newDatasetDeleteCommand() *cobra.Command { ctx, name, version, ProjectEndpointAPIVersion, ); err != nil { if eval_api.IsNotFound(err) { - return fmt.Errorf( - "no dataset %q at version %q in this project", name, version) + return messages.DatasetVersionNotFound(name, version) } - return fmt.Errorf("deleting dataset %q version %q: %w", name, version, err) + return messages.DeletingDatasetVersion(name, version, err) } if isJSON(cmd) { @@ -335,7 +332,7 @@ func newDatasetDeleteCommand() *cobra.Command { "name": name, "version": version, "status": "deleted", }) } - fmt.Fprintf(cmd.OutOrStdout(), "Deleted dataset %s version %s\n", name, version) + fmt.Fprint(cmd.OutOrStdout(), messages.DatasetDeleted(name, version)) return nil }, } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/eval_group.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/eval_group.go index bda511e8fa6..1b5563a58a8 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/eval_group.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/eval_group.go @@ -7,6 +7,7 @@ import ( "fmt" "path/filepath" + "azureaieval/internal/messages" "azureaieval/internal/pkg/eval_api" "azureaieval/internal/project" @@ -79,7 +80,7 @@ func newEvalCreateCommand() *cobra.Command { "id": id, "name": eval.Name, }) } - fmt.Fprintf(cmd.OutOrStdout(), "%s Created eval: %s (%s)\n", doneMark, eval.Name, id) + fmt.Fprint(cmd.OutOrStdout(), messages.EvalCreated(eval.Name, id)) return nil }, } @@ -112,14 +113,14 @@ func newEvalListCommand() *cobra.Command { list, err := ec.evalClient.ListOpenAIEvals(ctx, limit) if err != nil { - return fmt.Errorf("listing evals: %w", err) + return messages.ListingEvals(err) } if isJSON(cmd) { return emitJSONList(cmd.OutOrStdout(), list.Data) } if len(list.Data) == 0 { - fmt.Fprintln(cmd.OutOrStdout(), "No evals found.") + fmt.Fprint(cmd.OutOrStdout(), messages.NoEvals()) return nil } rows := make([][]string, 0, len(list.Data)) @@ -155,11 +156,9 @@ func newEvalShowCommand() *cobra.Command { group, err := ec.evalClient.GetOpenAIEval(ctx, evalID) if err != nil { if eval_api.IsNotFound(err) { - return fmt.Errorf( - "no eval %q in this project; "+ - "`azd ai eval list` shows the ones there are", evalID) + return messages.EvalNotFound(evalID) } - return fmt.Errorf("reading eval %q: %w", evalID, err) + return messages.ReadingEval(evalID, err) } if isJSON(cmd) { return emitJSON(cmd.OutOrStdout(), group) @@ -198,9 +197,9 @@ func newEvalDeleteCommand() *cobra.Command { if err := ec.evalClient.DeleteOpenAIEval(ctx, evalID); err != nil { if eval_api.IsNotFound(err) { - return fmt.Errorf("no eval %q in this project", evalID) + return messages.EvalGone(evalID) } - return fmt.Errorf("deleting eval %q: %w", evalID, err) + return messages.DeletingEval(evalID, err) } if isJSON(cmd) { @@ -208,7 +207,7 @@ func newEvalDeleteCommand() *cobra.Command { "id": evalID, "status": "deleted", }) } - fmt.Fprintf(cmd.OutOrStdout(), "Deleted eval %s\n", evalID) + fmt.Fprint(cmd.OutOrStdout(), messages.EvalDeleted(evalID)) return nil }, } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evalref.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evalref.go index f6d2d5ddfc3..21d8024de24 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evalref.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evalref.go @@ -5,8 +5,8 @@ package cmd import ( "context" - "fmt" + "azureaieval/internal/messages" "azureaieval/internal/project" ) @@ -55,9 +55,7 @@ func (ec *evalContext) resolveEvalRef( case err == nil: id := ec.recordedEvalID(ctx, eval.Name) if id == "" { - return evalRef{}, fmt.Errorf( - "eval %q is declared but has not been deployed to this environment yet; "+ - "run `azd up` first", eval.Name) + return evalRef{}, messages.EvalNotDeployedYet(eval.Name) } return evalRef{ID: id, Eval: eval, Config: cfg, ConfigPath: configPath}, nil case nameOrID == "": @@ -68,9 +66,7 @@ func (ec *evalContext) resolveEvalRef( } if nameOrID == "" { - return evalRef{}, fmt.Errorf( - "no eval was named and none is declared in %s; pass --eval with a name or an id", - configPath) + return evalRef{}, messages.NoEvalNamedOrDeclared(configPath) } // Not a declared name, so it is an id. return evalRef{ID: nameOrID}, nil diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator.go index a196a773c24..a489020b124 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator.go @@ -11,6 +11,7 @@ import ( "os" "strings" + "azureaieval/internal/messages" "azureaieval/internal/pkg/eval_api" "github.com/spf13/cobra" @@ -71,12 +72,12 @@ func newEvaluatorWriteCommand(verb, short string) *cobra.Command { raw, err := os.ReadFile(fromFile) if err != nil { - return fmt.Errorf("reading evaluator %q: %w", fromFile, err) + return messages.ReadingEvaluator(fromFile, err) } body, err := normalizeRubricBody(name, raw) if err != nil { - return fmt.Errorf("evaluator %q: %w", fromFile, err) + return messages.EvaluatorProblem(fromFile, err) } ctx := cmd.Context() @@ -94,7 +95,7 @@ func newEvaluatorWriteCommand(verb, short string) *cobra.Command { ctx, name, "", ProjectEndpointAPIVersion, ) if readErr != nil && !eval_api.IsNotFound(readErr) { - return fmt.Errorf("checking whether evaluator %q exists: %w", name, readErr) + return messages.CheckingEvaluatorExists(name, readErr) } if err := checkAssetExistence(verb, "evaluator", name, readErr == nil); err != nil { return err @@ -110,14 +111,14 @@ func newEvaluatorWriteCommand(verb, short string) *cobra.Command { ctx, name, body, existing, ProjectEndpointAPIVersion, ) if err != nil { - return fmt.Errorf("registering evaluator %q: %w", name, err) + return messages.RegisteringEvaluator(name, err) } if isJSON(cmd) { return emitJSON(cmd.OutOrStdout(), created) } - fmt.Fprintf(cmd.OutOrStdout(), - "Registered evaluator %s version %s\n", created.Name, created.Version) + fmt.Fprint(cmd.OutOrStdout(), + messages.EvaluatorRegistered(created.Name, created.Version)) return nil }, } @@ -131,11 +132,9 @@ func newEvaluatorWriteCommand(verb, short string) *cobra.Command { func checkAssetExistence(verb, kind, name string, exists bool) error { switch { case verb == "create" && exists: - return fmt.Errorf( - "%s %q already exists: use `update` to publish a new version", kind, name) + return messages.AssetAlreadyExists(kind, name) case verb == "update" && !exists: - return fmt.Errorf( - "%s %q does not exist: use `create` to register it", kind, name) + return messages.AssetDoesNotExist(kind, name) } return nil } @@ -154,7 +153,7 @@ const rubricDefinitionType = "rubric" func ensureDefinitionType(definition json.RawMessage) (json.RawMessage, error) { var doc map[string]json.RawMessage if err := json.Unmarshal(definition, &doc); err != nil { - return nil, fmt.Errorf("the definition is not a JSON object: %w", err) + return nil, messages.DefinitionNotJSONObject(err) } if _, ok := doc["type"]; ok { return definition, nil @@ -168,7 +167,7 @@ func ensureDefinitionType(definition json.RawMessage) (json.RawMessage, error) { func normalizeRubricBody(name string, raw []byte) (json.RawMessage, error) { var probe map[string]json.RawMessage if err := json.Unmarshal(raw, &probe); err != nil { - return nil, fmt.Errorf("not valid JSON: %w", err) + return nil, messages.NotValidJSON(err) } if definition, hasDefinition := probe["definition"]; hasDefinition { @@ -187,8 +186,7 @@ func normalizeRubricBody(name string, raw []byte) (json.RawMessage, error) { } if _, hasDimensions := probe["dimensions"]; !hasDimensions { - return nil, fmt.Errorf( - "expected a rubric definition with 'dimensions', or a document with 'definition'") + return nil, messages.RubricMissingDimensions() } typed, err := ensureDefinitionType(raw) @@ -232,7 +230,7 @@ func newEvaluatorListCommand() *cobra.Command { } list, err := ec.evalClient.ListEvaluators(ctx, filter, ProjectEndpointAPIVersion) if err != nil { - return fmt.Errorf("listing evaluators: %w", err) + return messages.ListingEvaluators(err) } return renderEvaluators(cmd, list) }, @@ -275,7 +273,7 @@ func newEvaluatorVersionsListCommand() *cobra.Command { list, err := ec.evalClient.ListEvaluatorVersions(ctx, name, ProjectEndpointAPIVersion) if err != nil { - return fmt.Errorf("listing versions of evaluator %q: %w", name, err) + return messages.ListingEvaluatorVersions(name, err) } return renderEvaluators(cmd, list) }, @@ -290,7 +288,7 @@ func renderEvaluators(cmd *cobra.Command, list *eval_api.EvaluatorListResponse) return emitJSONList(cmd.OutOrStdout(), list.Value) } if len(list.Value) == 0 { - fmt.Fprintln(cmd.OutOrStdout(), "No evaluators found.") + fmt.Fprint(cmd.OutOrStdout(), messages.NoEvaluators()) return nil } rows := make([][]string, 0, len(list.Value)) @@ -323,11 +321,9 @@ func newEvaluatorShowCommand() *cobra.Command { raw, err := ec.evalClient.GetEvaluatorRaw(ctx, name, version, ProjectEndpointAPIVersion) if err != nil { if eval_api.IsNotFound(err) { - return fmt.Errorf( - "no evaluator %q in this project; "+ - "`azd ai eval evaluator list` shows the ones there are", name) + return messages.EvaluatorNotFound(name) } - return fmt.Errorf("reading evaluator %q: %w", name, err) + return messages.ReadingEvaluator(name, err) } // -o json answers with the service's document untouched, because a @@ -411,10 +407,9 @@ func newEvaluatorDeleteCommand() *cobra.Command { ctx, name, version, ProjectEndpointAPIVersion, ); err != nil { if eval_api.IsNotFound(err) { - return fmt.Errorf( - "no evaluator %q at version %q in this project", name, version) + return messages.EvaluatorVersionNotFound(name, version) } - return fmt.Errorf("deleting evaluator %q version %q: %w", name, version, err) + return messages.DeletingEvaluatorVersion(name, version, err) } if isJSON(cmd) { @@ -422,7 +417,7 @@ func newEvaluatorDeleteCommand() *cobra.Command { "name": name, "version": version, "status": "deleted", }) } - fmt.Fprintf(cmd.OutOrStdout(), "Deleted evaluator %s version %s\n", name, version) + fmt.Fprint(cmd.OutOrStdout(), messages.EvaluatorDeleted(name, version)) return nil }, } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/gating.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/gating.go index c98e9fd96a7..db3eea7fa03 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/gating.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/gating.go @@ -9,6 +9,7 @@ import ( "strconv" "strings" + "azureaieval/internal/messages" "azureaieval/internal/pkg/eval_api" "github.com/spf13/cobra" @@ -47,15 +48,14 @@ func parseGate(spec string) (gate, error) { rate, ok := strings.CutPrefix(spec, "pass-rate=") if !ok { - return gate{}, fmt.Errorf( - "--fail-on must be any-failure or pass-rate=<0..1>, got %q", spec) + return gate{}, messages.FailOnInvalid(spec) } value, err := strconv.ParseFloat(rate, 64) if err != nil { - return gate{}, fmt.Errorf("--fail-on pass-rate must be a number, got %q", rate) + return gate{}, messages.FailOnRateNotNumber(rate) } if value < 0 || value > 1 { - return gate{}, fmt.Errorf("--fail-on pass-rate must be between 0 and 1, got %v", value) + return gate{}, messages.FailOnRateOutOfRange(value) } return gate{set: true, passRate: value}, nil } @@ -75,22 +75,21 @@ func (g gate) breach(counts *eval_api.EvalRunResultCounts) string { return "" } if counts == nil { - return "the run reported no result counts, so the threshold cannot be checked" + return messages.GateNoResultCounts() } if g.anyFailure { unpassed := counts.Total - counts.Passed if unpassed > 0 { - return fmt.Sprintf("%d of %d samples did not pass", unpassed, counts.Total) + return messages.GateSamplesDidNotPass(unpassed, counts.Total) } return "" } if counts.Total == 0 { - return "the run scored no rows, so its pass rate is below any threshold" + return messages.GateNoRowsScored() } actual := float64(counts.Passed) / float64(counts.Total) if actual < g.passRate { - return fmt.Sprintf("pass rate %.1f%% is below the required %.1f%%", - actual*100, g.passRate*100) + return messages.GatePassRateBelow(actual, g.passRate) } return "" } @@ -99,8 +98,7 @@ func (g gate) breach(counts *eval_api.EvalRunResultCounts) string { // exit so the wording can be tested: it is the block the spec's CI scenario // shows, and a pipeline's logs are where it is read. func gateBreachMessage(reason string) string { - return fmt.Sprintf("%s Evaluation gate: %s\n\nERROR: evaluation quality gate not met.\n", - failedMark, reason) + return messages.GateBreached(reason) } // applyGate ends the process with exit code 2 when the run missed its diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go index 010abc3ca5f..3fa6d3953d0 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go @@ -14,6 +14,7 @@ import ( "strings" "time" + "azureaieval/internal/messages" "azureaieval/internal/pkg/eval_api" "azureaieval/internal/project" @@ -74,11 +75,11 @@ func resolveInstruction(inline, path string) (string, error) { } raw, err := os.ReadFile(path) if err != nil { - return "", fmt.Errorf("reading --agent-instruction-file %q: %w", path, err) + return "", messages.ReadingInstructionFile(path, err) } text := strings.TrimSpace(string(raw)) if text == "" { - return "", fmt.Errorf("--agent-instruction-file %q is empty", path) + return "", messages.InstructionFileEmpty(path) } return text, nil } @@ -103,7 +104,7 @@ func declaredInstructions(named, configPath string) (string, error) { return "", nil } if err != nil { - return "", fmt.Errorf("reading instructions %q: %w", named, err) + return "", messages.ReadingInstructions(named, err) } return strings.TrimSpace(string(raw)), nil } @@ -142,7 +143,7 @@ func (ec *evalContext) resolveGenerationInstruction( } if local != "" { if !quiet { - fmt.Fprintf(out, " Seeding generation from %s.\n", filepath.ToSlash(path)) + fmt.Fprint(out, messages.SeedingFromFile(filepath.ToSlash(path))) } return local, nil } @@ -152,14 +153,13 @@ func (ec *evalContext) resolveGenerationInstruction( // Generation can still proceed from the agent source alone, so a // failure to read the agent is reported without stopping. if !quiet { - fmt.Fprintf(out, " warning: could not read agent %q for generation context: %v\n", - agentName, err) + fmt.Fprint(out, messages.WarningAgentUnreadable(agentName, err)) } return "", nil } instructions := agent.Instructions() if instructions != "" && !quiet { - fmt.Fprintf(out, " Seeding generation from the instructions of agent %q.\n", agentName) + fmt.Fprint(out, messages.SeedingFromAgent(agentName)) } return instructions, nil } @@ -191,7 +191,7 @@ func (ec *evalContext) generateRubric( out io.Writer, noWait bool, ) (*project.ArtifactRef, error) { - fmt.Fprintf(out, "Generating rubric %s...\n", plan.Name) + fmt.Fprint(out, messages.GeneratingRubric(plan.Name)) sources, unbuildable := eval_api.BuildGenerationSources( plan.From, plan.Agent, "", plan.Instruction, plan.traceOptions(), @@ -203,7 +203,7 @@ func (ec *evalContext) generateRubric( job, err := ec.evalClient.CreateEvaluatorGenerationJob(ctx, req, ProjectEndpointAPIVersion) if err != nil { - return nil, fmt.Errorf("submitting the rubric generation job: %w", err) + return nil, messages.SubmittingRubricJob(err) } if noWait { reportSubmitted(out, "azd ai eval evaluator", job.ID) @@ -213,14 +213,14 @@ func (ec *evalContext) generateRubric( completed, err := ec.pollGeneration(ctx, job.ID, ProjectEndpointAPIVersion, ec.evalClient.GetEvaluatorGenerationJob) if err != nil { - return nil, fmt.Errorf("rubric generation: %w", err) + return nil, messages.RubricGeneration(err) } path := project.ArtifactPath(plan.BaseDir, plan.OutputDir, plan.Name, ".json") if err := writeRubric(path, completed.Result); err != nil { return nil, err } - fmt.Fprintf(out, " wrote %s\n", path) + fmt.Fprint(out, messages.WroteArtifact(path)) _, version := completed.ResolvedNameVersion() return &project.ArtifactRef{ @@ -240,21 +240,19 @@ func refuseUnbuildableSources(kinds []string) error { return nil } reasons := map[string]string{ - "prompt": "--from prompt needs --agent-instruction or --agent-instruction-file", - "agent": "--from agent needs a target agent; pass --target, " + - "or declare one under target: in eval.yaml", - "file": "--from file is not a generation source; " + - "register the file with `azd ai eval dataset create` instead", + "prompt": messages.FromPromptNeedsInstruction(), + "agent": messages.FromAgentNeedsTarget(), + "file": messages.FromFileNotASource(), } - messages := make([]string, 0, len(kinds)) + reasonsForKinds := make([]string, 0, len(kinds)) for _, k := range kinds { if reason, ok := reasons[k]; ok { - messages = append(messages, reason) + reasonsForKinds = append(reasonsForKinds, reason) continue } - messages = append(messages, fmt.Sprintf("--from %s cannot be built from this plan", k)) + reasonsForKinds = append(reasonsForKinds, messages.FromNotBuildable(k)) } - return errors.New(strings.Join(messages, "; ")) + return messages.UnbuildableSources(reasonsForKinds) } // reportSubmitted says what was started and how to get back to it. @@ -264,8 +262,8 @@ func refuseUnbuildableSources(kinds []string) error { // with has to be the one they can paste when they come back. The group is named // too, because the two job types share no collection. func reportSubmitted(out io.Writer, group, jobID string) { - fmt.Fprintf(out, " submitted job %s\n", jobID) - fmt.Fprintf(out, "\nReattach with: %s job show %s\n", group, jobID) + fmt.Fprint(out, messages.JobSubmitted(jobID)) + fmt.Fprint(out, messages.ReattachToJob(group, jobID)) } // generateDataset submits the data generation job and downloads the result. @@ -275,7 +273,7 @@ func (ec *evalContext) generateDataset( out io.Writer, noWait bool, ) (*project.ArtifactRef, error) { - fmt.Fprintf(out, "Generating dataset %s (%d samples)...\n", plan.Name, plan.SampleSize) + fmt.Fprint(out, messages.GeneratingDataset(plan.Name, plan.SampleSize)) sources, unbuildable := eval_api.BuildGenerationSources( plan.From, plan.Agent, "", plan.Instruction, plan.traceOptions(), @@ -287,7 +285,7 @@ func (ec *evalContext) generateDataset( job, err := ec.evalClient.CreateDataGenerationJob(ctx, req, DataGenerationAPIVersion) if err != nil { - return nil, fmt.Errorf("submitting the data generation job: %w", err) + return nil, messages.SubmittingDataJob(err) } if noWait { reportSubmitted(out, "azd ai eval dataset", job.ID) @@ -303,27 +301,25 @@ func (ec *evalContext) generateDataset( // do anything about, so retry without the agent and say so. promptOnly := eval_api.WithoutAgentSource(sources) if eval_api.HasPromptSource(promptOnly) { - fmt.Fprintf(out, - " warning: generating from agent %q failed in the service; "+ - "retrying from the instruction alone.\n", plan.Agent) + fmt.Fprint(out, messages.WarningAgentSeedFailedRetrying(plan.Agent)) req = eval_api.NewDataGenerationJobRequest( plan.Name, plan.Model, plan.SampleSize, promptOnly) job, err = ec.evalClient.CreateDataGenerationJob(ctx, req, DataGenerationAPIVersion) if err != nil { - return nil, fmt.Errorf("submitting the data generation job: %w", err) + return nil, messages.SubmittingDataJob(err) } completed, err = ec.pollGeneration(ctx, job.ID, DataGenerationAPIVersion, ec.evalClient.GetDataGenerationJob) } } if err != nil { - return nil, fmt.Errorf("data generation: %w", explainDataGenerationFailure(err, plan.Agent)) + return nil, messages.DataGeneration(explainDataGenerationFailure(err, plan.Agent)) } name, version := completed.ResolvedNameVersion() if name == "" { - return nil, fmt.Errorf("the data generation job returned no dataset reference") + return nil, messages.DataJobReturnedNoDataset() } // Confirm the version exists before reading it, so a missing dataset is @@ -331,21 +327,21 @@ func (ec *evalContext) generateDataset( if _, err := ec.datasetClient.GetDataset( ctx, name, version, ProjectEndpointAPIVersion, ); err != nil { - return nil, fmt.Errorf("reading the generated dataset %q: %w", name, err) + return nil, messages.ReadingGeneratedDataset(name, err) } content, err := ec.datasetClient.DownloadDatasetContent(ctx, name, version, ProjectEndpointAPIVersion) if err != nil { - return nil, fmt.Errorf("downloading the generated dataset %q: %w", name, err) + return nil, messages.DownloadingGeneratedDataset(name, err) } path := project.ArtifactPath(plan.BaseDir, plan.OutputDir, plan.Name, ".jsonl") if err := os.MkdirAll(filepath.Dir(path), 0o750); err != nil { - return nil, fmt.Errorf("creating %q: %w", filepath.Dir(path), err) + return nil, messages.Creating(filepath.Dir(path), err) } if err := os.WriteFile(path, content, 0o600); err != nil { - return nil, fmt.Errorf("writing %q: %w", path, err) + return nil, messages.Writing(path, err) } - fmt.Fprintf(out, " wrote %s\n", path) + fmt.Fprint(out, messages.WroteArtifact(path)) return &project.ArtifactRef{ Name: plan.Name, @@ -383,13 +379,7 @@ func explainDataGenerationFailure(err error, agentName string) error { !strings.Contains(text, "Something went wrong during data generation") { return err } - return fmt.Errorf( - "%w\n\n"+ - "This job seeded generation from agent %q. Agent-seeded data generation is "+ - "currently failing in the service for every agent, so retrying will not help.\n"+ - "Workarounds: supply your own dataset with --dataset, or run without --target "+ - "to generate from the instruction alone.", - err, agentName) + return messages.AgentSeededGenerationFailing(err, agentName) } // pollGeneration waits for a generation job using the raised budget. @@ -407,10 +397,10 @@ func (ec *evalContext) pollGeneration( // weights and descriptions and publish a new version. func writeRubric(path string, result json.RawMessage) error { if len(result) == 0 { - return fmt.Errorf("the rubric generation job returned no result") + return messages.RubricJobReturnedNoResult() } if err := os.MkdirAll(filepath.Dir(path), 0o750); err != nil { - return fmt.Errorf("creating %q: %w", filepath.Dir(path), err) + return messages.Creating(filepath.Dir(path), err) } var parsed eval_api.EvaluatorResult diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_commands.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_commands.go index 87696485faa..97cc9adcb2a 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_commands.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_commands.go @@ -167,9 +167,7 @@ func refuseExistingArtifact(path string, force bool) error { return nil } if _, err := os.Stat(path); err == nil { - return fmt.Errorf( - "%s already exists; pass --force to overwrite it, or --output-dir to write elsewhere", - filepath.ToSlash(path)) + return messages.ArtifactExists(filepath.ToSlash(path)) } return nil } @@ -310,7 +308,7 @@ func reportGenerated(cmd *cobra.Command, ref *project.ArtifactRef, noWait bool) out := cmd.OutOrStdout() if ref == nil { if !noWait { - fmt.Fprintln(out, "Nothing was generated.") + fmt.Fprint(out, messages.NothingGenerated()) } return nil } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go index ea7e19e5a5c..27a7a7172d0 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go @@ -5,13 +5,13 @@ package cmd import ( "context" - "errors" "fmt" "maps" "os" "path/filepath" "strings" + "azureaieval/internal/messages" "azureaieval/internal/pkg/evalcore" "azureaieval/internal/project" @@ -59,18 +59,17 @@ func newInitCommand() *cobra.Command { switch source { case "", initSourceDataset, initSourceTraces: default: - return fmt.Errorf( - "--source %q is not a data source; use %q or %q", + return messages.SourceNotADataSource( source, initSourceDataset, initSourceTraces) } if source == initSourceTraces && dataset != "" { - return errors.New("--source traces reads production traces, so it takes no --dataset") + return messages.TracesTakesNoDataset() } if cmd.Flags().Changed("max-traces") && source != initSourceTraces { - return errors.New("--max-traces caps a trace-backed eval; pass --source traces") + return messages.MaxTracesNeedsTraceSource() } if maxTraces < 0 { - return errors.New("--max-traces must be positive") + return messages.MaxTracesMustBePositive() } if source == "" { source = initSourceDataset @@ -103,19 +102,17 @@ func newInitCommand() *cobra.Command { } if cfg.HasEval(evalName) { if !force { - return fmt.Errorf( - "an eval named %q already exists in %s; choose another name with --name, "+ - "or pass --force to replace it. `init` only adds: editing an eval is a file edit", + return messages.EvalAlreadyDeclared( evalName, filepath.ToSlash(configPath)) } cfg.RemoveEval(evalName) } if err := os.MkdirAll(filepath.Join(path, project.DefaultDatasetsDir), 0o750); err != nil { - return fmt.Errorf("creating the datasets directory: %w", err) + return messages.CreatingDatasetsDir(err) } if err := os.MkdirAll(filepath.Join(path, project.DefaultEvaluatorsDir), 0o750); err != nil { - return fmt.Errorf("creating the evaluators directory: %w", err) + return messages.CreatingEvaluatorsDir(err) } plan := planScaffold(scaffoldInput{ @@ -161,30 +158,30 @@ func newInitCommand() *cobra.Command { }) } - fmt.Fprintf(out, "%s Detected agent target: %s\n", doneMark, target) + fmt.Fprint(out, messages.DetectedTarget(target)) if source == initSourceTraces { - fmt.Fprintf(out, "%s Using data source: traces (Application Insights)\n", doneMark) + fmt.Fprint(out, messages.UsingTraceSource()) } if judgeModel != "" { - fmt.Fprintf(out, "%s Judge model deployment: %s\n", doneMark, judgeModel) + fmt.Fprint(out, messages.JudgeModelDeployment(judgeModel)) } - fmt.Fprintln(out, "\nCreated") - fmt.Fprintf(out, " %-33s evaluation configuration\n", filepath.ToSlash(configPath)) + fmt.Fprint(out, messages.CreatedHeading()) + fmt.Fprint(out, messages.CreatedConfigLine(filepath.ToSlash(configPath))) switch rootWiring { case wiringAdded: - fmt.Fprintf(out, " %-33s added service '%s'\n", rootConfigName, serviceName) + fmt.Fprint(out, messages.AddedServiceLine(rootConfigName, serviceName)) case wiringPresent: - fmt.Fprintf(out, " %-33s already declares service '%s'\n", rootConfigName, serviceName) + fmt.Fprint(out, messages.AlreadyDeclaresServiceLine(rootConfigName, serviceName)) } // Only what was actually scheduled is offered. Suggesting // `dataset generate` for a dataset the caller supplied sends them // to submit a billed job for an artifact they already have. next := plan.nextSteps() - fmt.Fprintf(out, "\nNext: %s\n", next[0]) + fmt.Fprint(out, messages.FirstNextStep(next[0])) for _, step := range next[1:] { - fmt.Fprintf(out, " %s\n", step) + fmt.Fprint(out, messages.FurtherNextStep(step)) } return nil }, @@ -443,22 +440,17 @@ const ( wiringPresent = "present" // an eval service was already declared ) -// noAzdProject is what init reports when there is nothing to attach to. -const noAzdProject = "no azd project found in this directory. Run `azd init` first, " + - "or run this from the root of an existing one; the eval service is added to " + - "its azure.yaml" - // readAzdProject returns the project, without changing it. func readAzdProject(ctx context.Context) (*azdext.ProjectConfig, error) { azdClient, err := azdext.NewAzdClient() if err != nil { - return nil, errors.New(noAzdProject) + return nil, messages.NoAzdProject() } defer azdClient.Close() resp, err := azdClient.Project().Get(ctx, &azdext.EmptyRequest{}) if err != nil || resp.GetProject() == nil { - return nil, errors.New(noAzdProject) + return nil, messages.NoAzdProject() } return resp.GetProject(), nil } @@ -529,13 +521,13 @@ func ensureRootEvalService( ) (string, error) { azdClient, err := azdext.NewAzdClient() if err != nil { - return "", fmt.Errorf("connecting to azd: %w", err) + return "", messages.ConnectingToAzd(err) } defer azdClient.Close() resp, err := azdClient.Project().Get(ctx, &azdext.EmptyRequest{}) if err != nil || resp.GetProject() == nil { - return "", errors.New(noAzdProject) + return "", messages.NoAzdProject() } // A service already pointing at this configuration is left alone: @@ -548,7 +540,7 @@ func ensureRootEvalService( "$ref": "./" + filepath.ToSlash(configPath), }) if err != nil { - return "", fmt.Errorf("building the eval service entry: %w", err) + return "", messages.BuildingServiceEntry(err) } _, err = azdClient.Project().AddService(ctx, &azdext.AddServiceRequest{ @@ -560,7 +552,7 @@ func ensureRootEvalService( }, }) if err != nil { - return "", fmt.Errorf("adding the eval service to %s: %w", rootConfigName, err) + return "", messages.AddingServiceTo(rootConfigName, err) } return wiringAdded, nil } @@ -599,14 +591,14 @@ func looksLikeLocalDataset(v string) bool { func writeYAML(path string, v any) error { if err := os.MkdirAll(filepath.Dir(path), 0o750); err != nil { - return fmt.Errorf("creating %q: %w", filepath.Dir(path), err) + return messages.Creating(filepath.Dir(path), err) } data, err := yaml.Marshal(v) if err != nil { - return fmt.Errorf("serializing %q: %w", path, err) + return messages.Serializing(path, err) } if err := os.WriteFile(path, data, 0o600); err != nil { - return fmt.Errorf("writing %q: %w", path, err) + return messages.Writing(path, err) } return nil } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/job.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/job.go index c4a9d48aa23..269879ad7ae 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/job.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/job.go @@ -7,6 +7,7 @@ import ( "context" "fmt" + "azureaieval/internal/messages" "azureaieval/internal/pkg/eval_api" "github.com/spf13/cobra" @@ -107,14 +108,14 @@ func newJobListCommand(kind jobKind) *cobra.Command { jobs, err := kind.list(ctx, ec) if err != nil { - return fmt.Errorf("listing %s generation jobs: %w", kind.name, err) + return messages.ListingJobs(kind.name, err) } if isJSON(cmd) { return emitJSONList(cmd.OutOrStdout(), jobs) } if len(jobs) == 0 { - fmt.Fprintf(cmd.OutOrStdout(), "No %s generation jobs found.\n", kind.name) + fmt.Fprint(cmd.OutOrStdout(), messages.NoJobs(kind.name)) return nil } table := make([][]string, 0, len(jobs)) @@ -154,9 +155,9 @@ func newJobShowCommand(kind jobKind) *cobra.Command { if isJSON(cmd) { return emitJSON(cmd.OutOrStdout(), job) } - fmt.Fprintf(cmd.OutOrStdout(), "%s %s\n", job.ID, job.Status) + fmt.Fprint(cmd.OutOrStdout(), messages.JobLine(job.ID, job.Status)) if job.Error != nil && job.Error.Message != "" { - fmt.Fprintf(cmd.OutOrStdout(), "error: %s\n", job.Error.Message) + fmt.Fprint(cmd.OutOrStdout(), messages.JobErrorLine(job.Error.Message)) } return nil }, @@ -191,8 +192,8 @@ func newJobCancelCommand(kind jobKind) *cobra.Command { if isJSON(cmd) { return emitJSON(cmd.OutOrStdout(), canceled) } - fmt.Fprintf(cmd.OutOrStdout(), "Cancelled %s generation job %s (%s)\n", - kind.name, jobID, canceled.Status) + fmt.Fprint(cmd.OutOrStdout(), + messages.JobCancelled(kind.name, jobID, canceled.Status)) return nil }, } @@ -230,7 +231,7 @@ func newJobDeleteCommand(kind jobKind) *cobra.Command { "id": jobID, "kind": kind.name, "status": "deleted", }) } - fmt.Fprintf(cmd.OutOrStdout(), "Deleted %s generation job %s\n", kind.name, jobID) + fmt.Fprint(cmd.OutOrStdout(), messages.JobDeleted(kind.name, jobID)) return nil }, } @@ -247,9 +248,7 @@ func jobLookupError(kind jobKind, jobID string, err error) error { if kind.name == jobKindEvaluator { other = jobKindDataset } - return fmt.Errorf( - "no %s generation job %q in this project; if it generated a %s, "+ - "use the %s job group instead", kind.name, jobID, other, other) + return messages.JobNotFound(kind.name, jobID, other) } - return fmt.Errorf("reading %s generation job %s: %w", kind.name, jobID, err) + return messages.ReadingJob(kind.name, jobID, err) } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/output.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/output.go index e89b19ae3b8..4a4f6b97446 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/output.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/output.go @@ -10,6 +10,8 @@ import ( "strings" "text/tabwriter" + "azureaieval/internal/messages" + "github.com/fatih/color" "github.com/spf13/cobra" ) @@ -17,11 +19,12 @@ import ( const outputJSON = "json" // Progress markers from the azd style guide, so the extension's lines sit -// alongside core's without a second vocabulary. +// alongside core's without a second vocabulary. The wording lives in messages +// with everything else the CLI says. const ( - doneMark = "(✓) Done:" // finished successfully - skippedMark = "(-) Skipped:" // intentionally not done, not a failure - failedMark = "(x) Failed:" // the step did not complete + doneMark = messages.DoneMark + skippedMark = messages.SkippedMark + failedMark = messages.FailedMark ) // writePortalLink closes a detail view with the asset's portal URL. @@ -33,7 +36,7 @@ func writePortalLink(w io.Writer, url string) { if url == "" { return } - fmt.Fprintf(w, "Portal: %s\n", color.CyanString(url)) + fmt.Fprint(w, messages.PortalLink(color.CyanString(url))) } // outputFormat reads the inherited -o/--output flag. @@ -130,5 +133,5 @@ func emitDetail(w io.Writer, fields []field) error { // requireFlag returns an error naming the missing flag, used when --no-prompt // prevents asking for a required value. func requireFlag(name string) error { - return fmt.Errorf("--%s is required (running with --no-prompt)", name) + return messages.FlagRequired(name) } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go index 0e0affef613..b83ea62b265 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go @@ -7,7 +7,6 @@ import ( "bufio" "context" "encoding/json" - "fmt" "os" "path/filepath" "reflect" @@ -15,6 +14,7 @@ import ( "strings" "time" + "azureaieval/internal/messages" "azureaieval/internal/pkg/dataset_api" "azureaieval/internal/pkg/eval_api" "azureaieval/internal/project" @@ -54,13 +54,10 @@ func (r *evalReconciler) EnsureDataset( ctx, decl.Name, ProjectEndpointAPIVersion, ) if err != nil { - return "", false, fmt.Errorf( - "dataset %q has no local source and could not be found on the project: %w", - decl.Name, err) + return "", false, messages.DatasetNotLocalNorFound(decl.Name, err) } if len(list.Value) == 0 { - return "", false, fmt.Errorf( - "dataset %q has no local source and is not registered on the project", decl.Name) + return "", false, messages.DatasetNotLocalNorRegistered(decl.Name) } version = dataset_api.LatestVersion(list.Value) } @@ -68,14 +65,14 @@ func (r *evalReconciler) EnsureDataset( } if _, err := os.Stat(localPath); err != nil { - return "", false, fmt.Errorf("dataset source %q: %w", localPath, err) + return "", false, messages.DatasetSource(localPath, err) } // A malformed row is only noticed once the service tries to evaluate it, // by which point a version has been published and the eval points at // it. Reading the file here costs nothing and names the offending line. if err := validateJSONL(localPath); err != nil { - return "", false, fmt.Errorf("dataset %q: %w", decl.Name, err) + return "", false, messages.DatasetProblem(decl.Name, err) } digest, err := project.Fingerprint(localPath) @@ -115,11 +112,7 @@ func (r *evalReconciler) EnsureDataset( ) if err != nil { if dataset_api.IsVersionConflict(err) { - return "", false, fmt.Errorf( - "dataset %q version %s already exists and the local file differs from it. "+ - "Raise `version:` to publish the change, or drop it to let each "+ - "deploy take the next version", - decl.Name, decl.Version) + return "", false, messages.DatasetVersionConflict(decl.Name, decl.Version) } return "", false, err } @@ -161,7 +154,7 @@ func (r *evalReconciler) EnsureDataset( func validateJSONL(path string) error { f, err := os.Open(path) if err != nil { - return fmt.Errorf("reading %s: %w", path, err) + return messages.ReadingPath(path, err) } defer f.Close() @@ -177,21 +170,18 @@ func validateJSONL(path string) error { } var row map[string]any if err := json.Unmarshal([]byte(text), &row); err != nil { - return fmt.Errorf( - "%s line %d is not valid JSON: %w. Every line must be one JSON object", - path, line, err) + return messages.JSONLRowInvalid(path, line, err) } if len(row) == 0 { - return fmt.Errorf( - "%s line %d is an empty object, which evaluates to nothing", path, line) + return messages.JSONLRowEmpty(path, line) } rows++ } if err := scanner.Err(); err != nil { - return fmt.Errorf("reading %s: %w", path, err) + return messages.ReadingPath(path, err) } if rows == 0 { - return fmt.Errorf("%s has no rows to evaluate", path) + return messages.JSONLNoRows(path) } return nil } @@ -207,12 +197,7 @@ func (r *evalReconciler) checkDatasetDrift( if !dataset_api.VersionGreater(latest, recorded) { return nil } - return fmt.Errorf( - "dataset %q is at version %s on the project but %s was recorded at the last deploy; "+ - "someone published a version outside this repo. "+ - "Pin it with `version: %s` on the dataset, or pull the newer content locally, "+ - "then deploy again", - name, latest, recorded, latest) + return messages.DatasetDrifted(name, latest, recorded) } // latestDatasetVersion reports the newest registered version, or empty when the @@ -244,25 +229,23 @@ func (r *evalReconciler) EnsureEvaluator( ctx, decl.Name, decl.Version, ProjectEndpointAPIVersion, ) if err != nil { - return "", false, fmt.Errorf( - "evaluator %q has no local source and could not be found on the project: %w", - decl.Name, err) + return "", false, messages.EvaluatorNotLocalNorFound(decl.Name, err) } return versionFromRaw(raw, decl.Version), false, nil } if _, err := os.Stat(localPath); err != nil { - return "", false, fmt.Errorf("evaluator source %q: %w", localPath, err) + return "", false, messages.EvaluatorSource(localPath, err) } raw, err := os.ReadFile(localPath) if err != nil { - return "", false, fmt.Errorf("evaluator source %q: %w", localPath, err) + return "", false, messages.EvaluatorSource(localPath, err) } body, err := normalizeRubricBody(decl.Name, raw) if err != nil { - return "", false, fmt.Errorf("evaluator %q: %w", decl.Name, err) + return "", false, messages.EvaluatorProblem(decl.Name, err) } // Compare against the definition already on the service. @@ -331,13 +314,7 @@ func checkEvaluatorDrift(name, recorded, remote string) error { if err != nil || remoteNumber <= recordedNumber { return nil } - return fmt.Errorf( - "evaluator %q is at version %s on the project but %s was recorded at the last "+ - "deploy, and the local definition does not match it: someone published a "+ - "version outside this repo. Publishing over it would leave their change "+ - "behind, so bring version %s into the declared source and deploy again, or "+ - "delete that version if it was a mistake", - name, remote, recorded, remote) + return messages.EvaluatorDrifted(name, remote, recorded) } // evaluatorPropagation bounds the wait for a freshly published evaluator to diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go index e0eec217b6c..eaef6821f22 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go @@ -51,7 +51,7 @@ func runCompleted(run *eval_api.OpenAIEvalRun) error { case "completed", "": return nil } - return fmt.Errorf("run %s finished with status %s", run.ID, run.Status) + return messages.RunFinishedWithStatus(run.ID, run.Status) } // newRunCommand builds the run group. @@ -115,13 +115,10 @@ func buildRunCommand(use, short string) *cobra.Command { if datasetName != "" { if !ref.Declared() { - return errors.New( - "--dataset overrides the dataset an eval declares, so it needs a " + - "declared eval; pass --eval with a name from the configuration") + return messages.DatasetOverrideNeedsDeclaredEval() } if _, ok := ref.Config.DatasetDeclaration(datasetName); !ok { - return fmt.Errorf( - "dataset %q is not in the catalog in %s", + return messages.DatasetNotInCatalog( datasetName, filepath.ToSlash(configPath)) } // The eval keeps its own declaration; only this run reads elsewhere. @@ -177,7 +174,7 @@ func buildRunCommand(use, short string) *cobra.Command { Metadata: metadata, }) if err != nil { - return fmt.Errorf("starting the evaluation run: %w", err) + return messages.StartingRun(err) } // Remembered per group as well as globally: a single shared key @@ -189,7 +186,7 @@ func buildRunCommand(use, short string) *cobra.Command { // Reported on stdout because azd does not surface an // extension's stderr, and skipped outside a project. if !errors.Is(err, errNoAzdEnvironment) && !isJSON(cmd) { - fmt.Fprintf(out, "warning: %v\n", err) + fmt.Fprint(out, messages.Warning(err)) } } @@ -197,8 +194,8 @@ func buildRunCommand(use, short string) *cobra.Command { if isJSON(cmd) { return emitJSON(out, startedRun(run, evalID, group)) } - fmt.Fprintf(out, "Started run %s (status: %s)\n", run.ID, run.Status) - fmt.Fprintf(out, "Reattach with: azd ai eval run show %s --eval %s\n", run.ID, evalID) + fmt.Fprint(out, messages.RunStarted(run.ID, run.Status)) + fmt.Fprint(out, messages.ReattachToRun(run.ID, evalID)) return nil } @@ -279,7 +276,7 @@ func (ec *evalContext) resolveEvalIDFromConfig( } if !jsonMode { - fmt.Fprintf(out, "Creating eval %q...\n", group.Name) + fmt.Fprint(out, messages.CreatingEval(group.Name)) } // The level from the flag wins over the eval's own declaration, so it has @@ -299,10 +296,10 @@ func (ec *evalContext) resolveEvalIDFromConfig( } created, err := ec.evalClient.CreateOpenAIEval(ctx, req) if err != nil { - return "", fmt.Errorf("creating eval %q: %w", group.Name, err) + return "", messages.CreatingEvalFailed(group.Name, err) } if err := ec.setEnvValue(ctx, idKey("eval", group.Name), created.ID); err != nil { - fmt.Fprintf(out, "warning: %v\n", err) + fmt.Fprint(out, messages.Warning(err)) } _ = ec.setEnvValue(ctx, envKeyEvalID, created.ID) return created.ID, nil @@ -367,11 +364,7 @@ func (ec *evalContext) checkDatasetRegistered( return nil } - return fmt.Errorf( - "dataset %q has local edits that are not registered.\n"+ - " Run `azd up` to register them, or `--eval-id ` to run against "+ - "an existing eval", - decl.Name) + return messages.DatasetHasUnregisteredEdits(decl.Name) } // reuseDataSourceFromLastRun rebuilds a run's data source from the group's most @@ -388,15 +381,10 @@ func (ec *evalContext) reuseDataSourceFromLastRun( ) (*eval_api.EvalRunDataSource, error) { list, err := ec.evalClient.ListOpenAIEvalRuns(ctx, evalID, 1) if err != nil { - return nil, fmt.Errorf("reading previous runs of eval %s: %w", evalID, err) + return nil, messages.ReadingPreviousRuns(evalID, err) } if list == nil || len(list.Data) == 0 || list.Data[0].DataSource == nil { - return nil, fmt.Errorf( - "eval %s has no previous run to repeat, so there is no target or dataset "+ - "to reuse.\n"+ - " Run it from the config once with `azd ai eval run start`, or name an "+ - "eval that declares one with `--eval`", - evalID) + return nil, messages.EvalHasNoPreviousRun(evalID) } return list.Data[0].DataSource, nil } @@ -513,28 +501,28 @@ func (ec *evalContext) readRegisteredDataset( if version == "" { versions, err := ec.datasetClient.ListDatasetVersions(ctx, name, ProjectEndpointAPIVersion) if err != nil { - return nil, fmt.Errorf("reading dataset %q: %w", name, err) + return nil, messages.ReadingDataset(name, err) } if versions != nil { version = dataset_api.LatestVersion(versions.Value) } } if version == "" { - return nil, fmt.Errorf("dataset %q has no versions to read", name) + return nil, messages.DatasetHasNoVersionsToRead(name) } content, err := ec.datasetClient.DownloadDatasetContent( ctx, name, version, ProjectEndpointAPIVersion) if err != nil { - return nil, fmt.Errorf("reading dataset %q version %s: %w", name, version, err) + return nil, messages.ReadingDatasetVersion(name, version, err) } items, err := readJSONLBytes(content, maxSamples) if err != nil { - return nil, fmt.Errorf("reading dataset %q version %s: %w", name, version, err) + return nil, messages.ReadingDatasetVersion(name, version, err) } if len(items) == 0 { - return nil, fmt.Errorf("dataset %q version %s has no rows", name, version) + return nil, messages.DatasetVersionEmpty(name, version) } return items, nil } @@ -589,13 +577,13 @@ func localDatasetPath(configPath string, group *project.Eval) string { func readJSONL(path string, limit int) ([]map[string]any, error) { f, err := os.Open(path) if err != nil { - return nil, fmt.Errorf("reading dataset %q: %w", path, err) + return nil, messages.ReadingDataset(path, err) } defer f.Close() items, err := scanJSONL(f, limit) if err != nil { - return nil, fmt.Errorf("reading dataset %q: %w", path, err) + return nil, messages.ReadingDataset(path, err) } return items, nil } @@ -621,7 +609,7 @@ func scanJSONL(r io.Reader, limit int) ([]map[string]any, error) { } var row map[string]any if err := json.Unmarshal([]byte(text), &row); err != nil { - return nil, fmt.Errorf("line %d is not valid JSON: %w", line, err) + return nil, messages.JSONLLineInvalid(line, err) } items = append(items, row) if limit > 0 && len(items) >= limit { @@ -677,12 +665,12 @@ func (ec *evalContext) pollRun( for { run, err := ec.evalClient.GetOpenAIEvalRun(ctx, evalID, runID) if err != nil { - return nil, fmt.Errorf("polling run %s: %w", runID, err) + return nil, messages.PollingRun(runID, err) } if run.Status != lastStatus { lastStatus = run.Status if !jsonMode { - fmt.Fprintf(out, " status: %s\n", run.Status) + fmt.Fprint(out, messages.RunStatusLine(run.Status)) } } if terminalRunStates[strings.ToLower(run.Status)] { @@ -817,19 +805,17 @@ func renderRun( // evaluators is one sample to go and look at, and reporting it as two // overstates how much is wrong. if c := run.ResultCounts; c != nil && c.Total > 0 { - fmt.Fprintf(out, "\nOverall pass rate: %s (%d/%d samples passed every evaluator)\n", - formatRate(c.Passed, c.Total), c.Passed, c.Total) + fmt.Fprint(out, messages.OverallPassRate(formatRate(c.Passed, c.Total), c.Passed, c.Total)) if c.Errored > 0 { - fmt.Fprintf(out, "%d sample(s) errored and were not scored.\n", c.Errored) + fmt.Fprint(out, messages.SamplesErrored(c.Errored)) } if c.Failed > 0 { - fmt.Fprintln(out, - "\nView failing samples: azd ai eval run output list --failed-only") + fmt.Fprint(out, messages.ViewFailingSamples()) } } if run.ReportURL != "" { - fmt.Fprintf(out, "Report: %s\n", run.ReportURL) + fmt.Fprint(out, messages.ReportLink(run.ReportURL)) } writePortalLink(out, run.PortalURL) return nil @@ -968,7 +954,7 @@ func criteriaMeans(items []eval_api.OutputItem) map[string]float64 { // errorNote describes rows an evaluator could not score. func errorNote(errored int) string { - return fmt.Sprintf("(%d errored, not scored)", errored) + return messages.ErroredNotScored(errored) } // formatRate renders a share as a percentage, and a rate over nothing as a diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go index df3038bdb19..4e260b19785 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go @@ -7,6 +7,7 @@ import ( "fmt" "strconv" + "azureaieval/internal/messages" "azureaieval/internal/pkg/eval_api" "github.com/spf13/cobra" @@ -54,11 +55,9 @@ func newRunListCommand() *cobra.Command { list, err := ec.evalClient.ListOpenAIEvalRuns(ctx, evalID, limit) if err != nil { if eval_api.IsNotFound(err) { - return fmt.Errorf( - "no eval %q in this project; "+ - "`azd up` creates the ones your config declares", evalID) + return messages.EvalNotDeployed(evalID) } - return fmt.Errorf("listing runs for %q: %w", evalID, err) + return messages.ListingRuns(evalID, err) } if isJSON(cmd) { var runs []eval_api.OpenAIEvalRun @@ -68,7 +67,7 @@ func newRunListCommand() *cobra.Command { return emitJSONList(cmd.OutOrStdout(), runs) } if list == nil || len(list.Data) == 0 { - fmt.Fprintf(cmd.OutOrStdout(), "Eval %s has no runs yet.\n", evalID) + fmt.Fprint(cmd.OutOrStdout(), messages.EvalHasNoRunsLine(evalID)) return nil } @@ -161,14 +160,14 @@ func newRunShowCommand() *cobra.Command { } out := cmd.OutOrStdout() - fmt.Fprintf(out, "Run %s\n", run.ID) - fmt.Fprintf(out, " name : %s\n", run.Name) - fmt.Fprintf(out, " status : %s\n", run.Status) + fmt.Fprint(out, messages.RunHeading(run.ID)) + fmt.Fprint(out, messages.RunNameLine(run.Name)) + fmt.Fprint(out, messages.RunStatusDetail(run.Status)) if counts := summarizeCounts(run.ResultCounts); counts != "" { - fmt.Fprintf(out, " results : %s\n", counts) + fmt.Fprint(out, messages.RunResultsLine(counts)) } if run.ReportURL != "" { - fmt.Fprintf(out, " report : %s\n", run.ReportURL) + fmt.Fprint(out, messages.RunReportLine(run.ReportURL)) } writePortalLink(out, run.PortalURL) if gateOnStatus { @@ -227,13 +226,12 @@ func newRunCancelCommand() *cobra.Command { // Cancelling a run that already finished is a no-op worth naming, // since the service reports success either way. if terminalRunStates[target.Status] { - return fmt.Errorf("run %s already finished with status %q", - target.ID, target.Status) + return messages.RunAlreadyFinished(target.ID, target.Status) } canceled, err := ec.evalClient.CancelOpenAIEvalRun(ctx, evalID, target.ID) if err != nil { - return fmt.Errorf("cancelling run %s: %w", target.ID, err) + return messages.CancellingRun(target.ID, err) } if isJSON(cmd) { return emitJSON(cmd.OutOrStdout(), canceled) @@ -242,7 +240,7 @@ func newRunCancelCommand() *cobra.Command { if status == "" { status = "cancelling" } - fmt.Fprintf(cmd.OutOrStdout(), "Run %s is now %s\n", target.ID, status) + fmt.Fprint(cmd.OutOrStdout(), messages.RunIsNow(target.ID, status)) return nil }, } @@ -283,9 +281,9 @@ func newRunDeleteCommand() *cobra.Command { if err := ec.evalClient.DeleteOpenAIEvalRun(ctx, evalID, runID); err != nil { if eval_api.IsNotFound(err) { - return fmt.Errorf("no run %q on eval %q", runID, evalID) + return messages.RunNotFound(runID, evalID) } - return fmt.Errorf("deleting run %s: %w", runID, err) + return messages.DeletingRun(runID, err) } if isJSON(cmd) { @@ -293,7 +291,7 @@ func newRunDeleteCommand() *cobra.Command { "id": runID, "eval_id": evalID, "status": "deleted", }) } - fmt.Fprintf(cmd.OutOrStdout(), "Deleted run %s\n", runID) + fmt.Fprint(cmd.OutOrStdout(), messages.RunDeleted(runID)) return nil }, } @@ -306,8 +304,7 @@ func summarizeCounts(counts *eval_api.EvalRunResultCounts) string { if counts == nil { return "" } - return fmt.Sprintf("%d passed, %d failed, %d errored", - counts.Passed, counts.Failed, counts.Errored) + return messages.CountsSummary(counts.Passed, counts.Failed, counts.Errored) } // metaDataset and metaDatasetVersion record which rows a run scored. The run's diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output.go index f31b868a099..b162a13a3b3 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output.go @@ -12,6 +12,7 @@ import ( "strconv" "strings" + "azureaieval/internal/messages" "azureaieval/internal/pkg/eval_api" "github.com/spf13/cobra" @@ -71,7 +72,7 @@ func newRunOutputListCommand() *cobra.Command { // totals rather than failing. items, err := ec.evalClient.ListOutputItems(ctx, evalID, run.ID, 0) if err != nil { - return fmt.Errorf("reading the results of run %s: %w", run.ID, err) + return messages.ReadingRunResults(run.ID, err) } rows := items.Data if failedOnly { @@ -88,7 +89,7 @@ func newRunOutputListCommand() *cobra.Command { if outFile != "" { f, err := os.Create(outFile) if err != nil { - return fmt.Errorf("creating %q: %w", outFile, err) + return messages.Creating(outFile, err) } defer f.Close() return emitJSON(f, payload) @@ -145,12 +146,9 @@ func newRunOutputShowCommand() *cobra.Command { item, err := ec.evalClient.GetOutputItem(ctx, evalID, run.ID, itemID) if err != nil { if eval_api.IsNotFound(err) { - return fmt.Errorf( - "no output item %q on run %s; "+ - "`azd ai eval run output list` shows the ones there are", - itemID, run.ID) + return messages.OutputItemNotFound(itemID, run.ID) } - return fmt.Errorf("reading output item %q: %w", itemID, err) + return messages.ReadingOutputItem(itemID, err) } return emitJSON(cmd.OutOrStdout(), item) }, @@ -177,7 +175,7 @@ func newRunOutputExportCommand() *cobra.Command { RunE: func(cmd *cobra.Command, args []string) error { format = strings.ToLower(format) if format != "json" && format != "csv" { - return fmt.Errorf("--format must be json or csv, got %q", format) + return messages.ExportFormatInvalid(format) } ctx := cmd.Context() @@ -202,7 +200,7 @@ func newRunOutputExportCommand() *cobra.Command { if outFile != "" { f, err := os.Create(outFile) if err != nil { - return fmt.Errorf("creating %q: %w", outFile, err) + return messages.Creating(outFile, err) } defer f.Close() w = f @@ -216,8 +214,7 @@ func newRunOutputExportCommand() *cobra.Command { case formatJSONL: return writeResultsJSONL(w, run) default: - return fmt.Errorf( - "--format %q is not supported; use %s, %s or %s", + return messages.ExportFormatUnsupported( format, formatCSV, formatJSON, formatJSONL) } }, @@ -259,8 +256,7 @@ func resolveEvalID( if cached := ec.getEnvValue(cmd.Context(), envKeyEvalID); cached != "" { return cached, nil } - return "", fmt.Errorf( - "no eval given; pass its id as an argument, or name one with --eval") + return "", messages.NoEvalGiven() } // addEvalFlag registers the flag that says which eval a command acts on. It @@ -305,21 +301,19 @@ func (ec *evalContext) latestOrNamedRun( return run, nil } if explicit { - return nil, fmt.Errorf("reading run %s: %w", runID, err) + return nil, messages.ReadingRun(runID, err) } } list, err := ec.evalClient.ListOpenAIEvalRuns(ctx, evalID, 1) if err != nil { if eval_api.IsNotFound(err) { - return nil, fmt.Errorf( - "no eval %q in this project; "+ - "`azd up` creates the ones your config declares", evalID) + return nil, messages.EvalNotDeployed(evalID) } - return nil, fmt.Errorf("listing runs for eval %s: %w", evalID, err) + return nil, messages.ListingRuns(evalID, err) } if len(list.Data) == 0 { - return nil, fmt.Errorf("eval %s has no runs yet", evalID) + return nil, messages.EvalHasNoRuns(evalID) } return &list.Data[0], nil } @@ -330,11 +324,10 @@ func renderResults( items []eval_api.OutputItem, failedOnly bool, ) error { - fmt.Fprintf(w, "Run %s status: %s\n", run.ID, run.Status) + fmt.Fprint(w, messages.RunStatusHeading(run.ID, run.Status)) if c := run.ResultCounts; c != nil { - fmt.Fprintf(w, "Totals: %d passed, %d failed, %d errored\n\n", - c.Passed, c.Failed, c.Errored) + fmt.Fprint(w, messages.ResultTotals(c.Passed, c.Failed, c.Errored)) } if len(run.PerTestingCriteria) > 0 { @@ -360,9 +353,9 @@ func renderResults( // these say which and why. if len(items) == 0 { if failedOnly { - fmt.Fprintln(w, "\nNo failing rows.") + fmt.Fprint(w, messages.NoFailingRows()) } else { - fmt.Fprintln(w, "\nNo rows have been scored yet.") + fmt.Fprint(w, messages.NoRowsScored()) } } else { fmt.Fprintln(w) @@ -402,12 +395,12 @@ func renderResults( return err } if n := len(rows); failedOnly && n > 0 { - fmt.Fprintf(w, "\n%d sample(s) failed at least one evaluator.\n", n) + fmt.Fprint(w, messages.SamplesFailedAtLeastOne(n)) } } if run.ReportURL != "" { - fmt.Fprintf(w, "\nReport: %s\n", run.ReportURL) + fmt.Fprint(w, messages.ReportLinkAfterRows(run.ReportURL)) } return nil } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/foundry/projectctx/store.go b/cli/azd/extensions/azure.ai.evaluations/internal/foundry/projectctx/store.go index 433e9ab8d41..dd899fdee9f 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/foundry/projectctx/store.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/foundry/projectctx/store.go @@ -5,7 +5,8 @@ package projectctx import ( "context" - "fmt" + + "azureaieval/internal/messages" "github.com/azure/azure-dev/cli/azd/pkg/azdext" ) @@ -22,14 +23,13 @@ func getProjectContext( ) (State, bool, error) { ch, err := azdext.NewConfigHelper(azdClient) if err != nil { - return State{}, false, fmt.Errorf("getProjectContext: %w", err) + return State{}, false, messages.ProjectContextClient(err) } var state State found, err := ch.GetUserJSON(ctx, projectContextConfigPath, &state) if err != nil { - return State{}, false, - fmt.Errorf("getProjectContext: failed to read config: %w", err) + return State{}, false, messages.ProjectContextRead(err) } if !found || state.Endpoint == "" { diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/foundry/projectctx/validator.go b/cli/azd/extensions/azure.ai.evaluations/internal/foundry/projectctx/validator.go index bf014dc99c4..9d2de6a5d95 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/foundry/projectctx/validator.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/foundry/projectctx/validator.go @@ -8,7 +8,7 @@ import ( "net/url" "strings" - "azureaieval/internal/exterrors" + "azureaieval/internal/messages" ) // foundryHostSuffixes is the authoritative list of accepted Foundry host suffixes. @@ -41,49 +41,25 @@ func isFoundryHost(hostname string) bool { func Validate(raw string) (normalized string, pathWarning bool, err error) { raw = strings.TrimSpace(raw) if raw == "" { - return "", false, exterrors.Validation( - exterrors.CodeInvalidParameter, - "project endpoint must not be empty", - "provide a Foundry project endpoint URL "+ - "(e.g. https://.services.ai.azure.com/api/projects/)", - ) + return "", false, messages.EndpointEmpty() } u, parseErr := url.Parse(raw) if parseErr != nil { - return "", false, exterrors.Validation( - exterrors.CodeInvalidParameter, - fmt.Sprintf("invalid project endpoint URL: %v", parseErr), - "provide a valid https:// Foundry project endpoint URL", - ) + return "", false, messages.EndpointUnparseable(parseErr) } if !strings.EqualFold(u.Scheme, "https") { - return "", false, exterrors.Validation( - exterrors.CodeInvalidParameter, - "project endpoint must use https", - "provide an https:// URL", - ) + return "", false, messages.EndpointNotHTTPS() } host := u.Hostname() if host == "" || !isFoundryHost(host) { - return "", false, exterrors.Validation( - exterrors.CodeInvalidParameter, - fmt.Sprintf( - "project endpoint host %q is not a recognized Foundry host (*%s)", - host, foundryHostSuffixes[0], - ), - "the host must end with "+foundryHostSuffixes[0], - ) + return "", false, messages.EndpointNotFoundryHost(host, foundryHostSuffixes[0]) } if u.Port() != "" { - return "", false, exterrors.Validation( - exterrors.CodeInvalidParameter, - fmt.Sprintf("project endpoint host %q must not include a port", u.Host), - "remove the explicit port from the URL", - ) + return "", false, messages.EndpointHasPort(u.Host) } // Normalize: lowercase host, strip trailing slash. @@ -102,12 +78,5 @@ func Validate(raw string) (normalized string, pathWarning bool, err error) { // NoEndpointError returns the structured dependency error used when no project // endpoint could be resolved from any source. func NoEndpointError() error { - return exterrors.Dependency( - exterrors.CodeMissingProjectEndpoint, - "no Foundry project endpoint resolved", - "persist a workspace default with `azd ai project set `, "+ - "or set FOUNDRY_PROJECT_ENDPOINT (or AZURE_AI_PROJECT_ENDPOINT) "+ - "in the active azd environment, "+ - "or export FOUNDRY_PROJECT_ENDPOINT (or AZURE_AI_PROJECT_ENDPOINT) in your shell", - ) + return messages.NoEndpoint() } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go b/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go index bdc0865c9ea..033903dfcd5 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go @@ -1,8 +1,9 @@ // Package messages holds every string this extension shows a user. // // One file, so the whole voice of the CLI can be reviewed in one sitting and a -// wording change never has to be hunted through the command tree. Nothing here -// imports anything from the extension, so any package can use it. +// wording change never has to be hunted through the command tree. The only +// extension package it imports is exterrors, which holds no wording of its own, +// so every other package can use this one. // // Conventions, so the set stays consistent: // @@ -11,12 +12,17 @@ // - A name the user chose is quoted with %q; an identifier the service // assigned is not, because it is already unmistakable. // - Progress and success lines are sentences with a capital and no period. +// - A printed line carries its own newlines, so a call site is a bare Fprint. // - Nothing here decides *whether* to print. That stays at the call site. package messages import ( "errors" "fmt" + "sort" + "strings" + + "azureaieval/internal/exterrors" ) // --------------------------------------------------------------------------- @@ -60,6 +66,316 @@ func ResponsesNeedIDs(eval string) error { eval) } +// DatasetOverrideNeedsDeclaredEval reports --dataset passed against a bare id. +func DatasetOverrideNeedsDeclaredEval() error { + return errors.New( + "--dataset overrides the dataset an eval declares, so it needs a " + + "declared eval; pass --eval with a name from the configuration") +} + +// DatasetNotInCatalog reports a --dataset the configuration does not declare. +func DatasetNotInCatalog(dataset, configPath string) error { + return fmt.Errorf("dataset %q is not in the catalog in %s", dataset, configPath) +} + +// DatasetHasUnregisteredEdits reports local rows no deployed version holds. +func DatasetHasUnregisteredEdits(dataset string) error { + return fmt.Errorf( + "dataset %q has local edits that are not registered.\n"+ + " Run `azd up` to register them, or `--eval-id ` to run against "+ + "an existing eval", + dataset) +} + +// CreatingEval reports an eval being created because it has never been deployed. +func CreatingEval(eval string) string { + return fmt.Sprintf("Creating eval %q...\n", eval) +} + +// CreatingEvalFailed reports the service refusing to create the eval. +func CreatingEvalFailed(eval string, err error) error { + return fmt.Errorf("creating eval %q: %w", eval, err) +} + +// StartingRun reports the service refusing to start the run. +func StartingRun(err error) error { + return fmt.Errorf("starting the evaluation run: %w", err) +} + +// RunStarted reports a submitted run that was not waited on. +func RunStarted(runID, status string) string { + return fmt.Sprintf("Started run %s (status: %s)\n", runID, status) +} + +// ReattachToRun says how to come back to a run started with --no-wait. +func ReattachToRun(runID, evalID string) string { + return fmt.Sprintf("Reattach with: azd ai eval run show %s --eval %s\n", runID, evalID) +} + +// ReadingPreviousRuns reports a failure to look up what an eval last ran. +func ReadingPreviousRuns(evalID string, err error) error { + return fmt.Errorf("reading previous runs of eval %s: %w", evalID, err) +} + +// EvalHasNoPreviousRun reports an eval named by id that has nothing to repeat. +func EvalHasNoPreviousRun(evalID string) error { + return fmt.Errorf( + "eval %s has no previous run to repeat, so there is no target or dataset "+ + "to reuse.\n"+ + " Run it from the config once with `azd ai eval run start`, or name an "+ + "eval that declares one with `--eval`", + evalID) +} + +// PollingRun reports a failure while waiting for a run to finish. +func PollingRun(runID string, err error) error { + return fmt.Errorf("polling run %s: %w", runID, err) +} + +// RunStatusLine reports a status change seen while polling. +func RunStatusLine(status string) string { + return fmt.Sprintf(" status: %s\n", status) +} + +// RunFinishedWithStatus reports a run that ended in something other than completed. +func RunFinishedWithStatus(runID, status string) error { + return fmt.Errorf("run %s finished with status %s", runID, status) +} + +// OverallPassRate reports the share of samples that passed every evaluator. +func OverallPassRate(rate string, passed, total int) string { + return fmt.Sprintf("\nOverall pass rate: %s (%d/%d samples passed every evaluator)\n", + rate, passed, total) +} + +// SamplesErrored reports rows the run could not score at all. +func SamplesErrored(errored int) string { + return fmt.Sprintf("%d sample(s) errored and were not scored.\n", errored) +} + +// ViewFailingSamples points at the command that lists the rows that failed. +func ViewFailingSamples() string { + return "\nView failing samples: azd ai eval run output list --failed-only\n" +} + +// ErroredNotScored annotates an evaluator's row with what it could not score. +func ErroredNotScored(errored int) string { + return fmt.Sprintf("(%d errored, not scored)", errored) +} + +// ReportLink closes a run summary with the service's own report. +func ReportLink(url string) string { + return fmt.Sprintf("Report: %s\n", url) +} + +// EvalNotDeployed reports an eval id the project does not hold. +func EvalNotDeployed(evalID string) error { + return fmt.Errorf( + "no eval %q in this project; "+ + "`azd up` creates the ones your config declares", evalID) +} + +// EvalNotDeployedYet reports a declared eval that no deploy has created. +func EvalNotDeployedYet(eval string) error { + return fmt.Errorf( + "eval %q is declared but has not been deployed to this environment yet; "+ + "run `azd up` first", eval) +} + +// NoEvalNamedOrDeclared reports a command with no eval to act on. +func NoEvalNamedOrDeclared(configPath string) error { + return fmt.Errorf( + "no eval was named and none is declared in %s; pass --eval with a name or an id", + configPath) +} + +// NoEvalGiven reports a command run outside a project with no eval id. +func NoEvalGiven() error { + return errors.New("no eval given; pass its id as an argument, or name one with --eval") +} + +// ListingRuns reports a failure to list an eval's runs. +func ListingRuns(evalID string, err error) error { + return fmt.Errorf("listing runs for %q: %w", evalID, err) +} + +// EvalHasNoRunsLine reports an eval with no runs to list. +func EvalHasNoRunsLine(evalID string) string { + return fmt.Sprintf("Eval %s has no runs yet.\n", evalID) +} + +// EvalHasNoRuns reports an eval with no run to fall back on. +func EvalHasNoRuns(evalID string) error { + return fmt.Errorf("eval %s has no runs yet", evalID) +} + +// ReadingRun reports a failure to read the run the caller named. +func ReadingRun(runID string, err error) error { + return fmt.Errorf("reading run %s: %w", runID, err) +} + +// RunHeading opens the detail view of one run. +func RunHeading(runID string) string { + return fmt.Sprintf("Run %s\n", runID) +} + +// RunNameLine reports the run's name in the detail view. +func RunNameLine(name string) string { + return fmt.Sprintf(" name : %s\n", name) +} + +// RunStatusDetail reports the run's status in the detail view. +func RunStatusDetail(status string) string { + return fmt.Sprintf(" status : %s\n", status) +} + +// RunResultsLine reports the run's counts in the detail view. +func RunResultsLine(counts string) string { + return fmt.Sprintf(" results : %s\n", counts) +} + +// RunReportLine reports the run's report URL in the detail view. +func RunReportLine(url string) string { + return fmt.Sprintf(" report : %s\n", url) +} + +// CountsSummary renders a run's verdict counts on one line. +func CountsSummary(passed, failed, errored int) string { + return fmt.Sprintf("%d passed, %d failed, %d errored", passed, failed, errored) +} + +// RunAlreadyFinished reports a cancel asked of a run that already ended. +func RunAlreadyFinished(runID, status string) error { + return fmt.Errorf("run %s already finished with status %q", runID, status) +} + +// CancellingRun reports the service refusing to cancel the run. +func CancellingRun(runID string, err error) error { + return fmt.Errorf("cancelling run %s: %w", runID, err) +} + +// RunIsNow reports the state a cancelled run moved to. +func RunIsNow(runID, status string) string { + return fmt.Sprintf("Run %s is now %s\n", runID, status) +} + +// RunNotFound reports a run id the eval does not hold. +func RunNotFound(runID, evalID string) error { + return fmt.Errorf("no run %q on eval %q", runID, evalID) +} + +// DeletingRun reports the service refusing to delete the run. +func DeletingRun(runID string, err error) error { + return fmt.Errorf("deleting run %s: %w", runID, err) +} + +// RunDeleted confirms a deleted run. +func RunDeleted(runID string) string { + return fmt.Sprintf("Deleted run %s\n", runID) +} + +// ReadingRunResults reports a failure to read a run's per-sample rows. +func ReadingRunResults(runID string, err error) error { + return fmt.Errorf("reading the results of run %s: %w", runID, err) +} + +// OutputItemNotFound reports an output item the run does not hold. +func OutputItemNotFound(itemID, runID string) error { + return fmt.Errorf( + "no output item %q on run %s; "+ + "`azd ai eval run output list` shows the ones there are", + itemID, runID) +} + +// ReadingOutputItem reports a failure to read one evaluated row. +func ReadingOutputItem(itemID string, err error) error { + return fmt.Errorf("reading output item %q: %w", itemID, err) +} + +// RunStatusHeading opens the per-sample view of a run. +func RunStatusHeading(runID, status string) string { + return fmt.Sprintf("Run %s status: %s\n", runID, status) +} + +// ResultTotals reports a run's verdict counts above the rows. +func ResultTotals(passed, failed, errored int) string { + return fmt.Sprintf("Totals: %d passed, %d failed, %d errored\n\n", passed, failed, errored) +} + +// NoFailingRows reports a --failed-only listing with nothing in it. +func NoFailingRows() string { + return "\nNo failing rows.\n" +} + +// NoRowsScored reports a run that has produced no rows yet. +func NoRowsScored() string { + return "\nNo rows have been scored yet.\n" +} + +// SamplesFailedAtLeastOne closes a --failed-only listing with its count. +func SamplesFailedAtLeastOne(samples int) string { + return fmt.Sprintf("\n%d sample(s) failed at least one evaluator.\n", samples) +} + +// ReportLinkAfterRows closes a per-sample listing with the service's report. +func ReportLinkAfterRows(url string) string { + return fmt.Sprintf("\nReport: %s\n", url) +} + +// ExportFormatInvalid reports an --format the export command cannot write. +func ExportFormatInvalid(format string) error { + return fmt.Errorf("--format must be json or csv, got %q", format) +} + +// ExportFormatUnsupported reports an --format that got past the flag check. +func ExportFormatUnsupported(format, csv, json, jsonl string) error { + return fmt.Errorf( + "--format %q is not supported; use %s, %s or %s", + format, csv, json, jsonl) +} + +// FailOnInvalid reports a --fail-on value that is neither form of threshold. +func FailOnInvalid(spec string) error { + return fmt.Errorf("--fail-on must be any-failure or pass-rate=<0..1>, got %q", spec) +} + +// FailOnRateNotNumber reports a --fail-on pass rate that will not parse. +func FailOnRateNotNumber(rate string) error { + return fmt.Errorf("--fail-on pass-rate must be a number, got %q", rate) +} + +// FailOnRateOutOfRange reports a --fail-on pass rate outside 0..1. +func FailOnRateOutOfRange(value float64) error { + return fmt.Errorf("--fail-on pass-rate must be between 0 and 1, got %v", value) +} + +// GateNoResultCounts reports a gate that has nothing to measure against. +func GateNoResultCounts() string { + return "the run reported no result counts, so the threshold cannot be checked" +} + +// GateSamplesDidNotPass reports an any-failure gate that was breached. +func GateSamplesDidNotPass(unpassed, total int) string { + return fmt.Sprintf("%d of %d samples did not pass", unpassed, total) +} + +// GateNoRowsScored reports a pass-rate gate over a run that scored nothing. +func GateNoRowsScored() string { + return "the run scored no rows, so its pass rate is below any threshold" +} + +// GatePassRateBelow reports a pass-rate gate that was breached. +func GatePassRateBelow(actual, required float64) string { + return fmt.Sprintf("pass rate %.1f%% is below the required %.1f%%", + actual*100, required*100) +} + +// GateBreached is the block a breached gate leaves in a pipeline's log. +func GateBreached(reason string) string { + return fmt.Sprintf("%s Evaluation gate: %s\n\nERROR: evaluation quality gate not met.\n", + FailedMark, reason) +} + // --------------------------------------------------------------------------- // Generation // --------------------------------------------------------------------------- @@ -71,3 +387,1372 @@ func ResponsesNeedIDs(eval string) error { func GenerationModelRequired() error { return errors.New("a model deployment is required to generate: pass --generation-model") } + +// ReadingInstructionFile reports an --agent-instruction-file that would not read. +func ReadingInstructionFile(path string, err error) error { + return fmt.Errorf("reading --agent-instruction-file %q: %w", path, err) +} + +// InstructionFileEmpty reports an --agent-instruction-file with nothing in it. +func InstructionFileEmpty(path string) error { + return fmt.Errorf("--agent-instruction-file %q is empty", path) +} + +// ReadingInstructions reports a declared instructions file that would not read. +func ReadingInstructions(named string, err error) error { + return fmt.Errorf("reading instructions %q: %w", named, err) +} + +// SeedingFromFile names the local file generation was seeded from. +func SeedingFromFile(path string) string { + return fmt.Sprintf(" Seeding generation from %s.\n", path) +} + +// SeedingFromAgent names the agent whose published instructions seeded generation. +func SeedingFromAgent(agent string) string { + return fmt.Sprintf(" Seeding generation from the instructions of agent %q.\n", agent) +} + +// WarningAgentUnreadable reports an agent that could not supply context. +func WarningAgentUnreadable(agent string, err error) string { + return fmt.Sprintf(" warning: could not read agent %q for generation context: %v\n", + agent, err) +} + +// WarningAgentSeedFailedRetrying reports the retry that drops the agent source. +func WarningAgentSeedFailedRetrying(agent string) string { + return fmt.Sprintf( + " warning: generating from agent %q failed in the service; "+ + "retrying from the instruction alone.\n", agent) +} + +// GeneratingRubric reports a rubric generation job about to be submitted. +func GeneratingRubric(name string) string { + return fmt.Sprintf("Generating rubric %s...\n", name) +} + +// GeneratingDataset reports a dataset generation job about to be submitted. +func GeneratingDataset(name string, samples int) string { + return fmt.Sprintf("Generating dataset %s (%d samples)...\n", name, samples) +} + +// SubmittingRubricJob reports the service refusing the rubric job. +func SubmittingRubricJob(err error) error { + return fmt.Errorf("submitting the rubric generation job: %w", err) +} + +// SubmittingDataJob reports the service refusing the data generation job. +func SubmittingDataJob(err error) error { + return fmt.Errorf("submitting the data generation job: %w", err) +} + +// RubricGeneration reports a rubric job that did not finish successfully. +func RubricGeneration(err error) error { + return fmt.Errorf("rubric generation: %w", err) +} + +// DataGeneration reports a data job that did not finish successfully. +func DataGeneration(err error) error { + return fmt.Errorf("data generation: %w", err) +} + +// RubricJobReturnedNoResult reports a completed rubric job with nothing to write. +func RubricJobReturnedNoResult() error { + return errors.New("the rubric generation job returned no result") +} + +// DataJobReturnedNoDataset reports a completed data job with nothing to fetch. +func DataJobReturnedNoDataset() error { + return errors.New("the data generation job returned no dataset reference") +} + +// ReadingGeneratedDataset reports the generated dataset not being there to read. +func ReadingGeneratedDataset(name string, err error) error { + return fmt.Errorf("reading the generated dataset %q: %w", name, err) +} + +// DownloadingGeneratedDataset reports a failure to fetch the generated rows. +func DownloadingGeneratedDataset(name string, err error) error { + return fmt.Errorf("downloading the generated dataset %q: %w", name, err) +} + +// AgentSeededGenerationFailing explains the service-side failure that hits +// every agent, so the caller does not retry against a deterministic failure. +func AgentSeededGenerationFailing(err error, agent string) error { + return fmt.Errorf( + "%w\n\n"+ + "This job seeded generation from agent %q. Agent-seeded data generation is "+ + "currently failing in the service for every agent, so retrying will not help.\n"+ + "Workarounds: supply your own dataset with --dataset, or run without --target "+ + "to generate from the instruction alone.", + err, agent) +} + +// FromPromptNeedsInstruction reports --from prompt with nothing to prompt with. +func FromPromptNeedsInstruction() string { + return "--from prompt needs --agent-instruction or --agent-instruction-file" +} + +// FromAgentNeedsTarget reports --from agent with no agent to read. +func FromAgentNeedsTarget() string { + return "--from agent needs a target agent; pass --target, " + + "or declare one under target: in eval.yaml" +} + +// FromFileNotASource reports --from file, which generation has no path for. +func FromFileNotASource() string { + return "--from file is not a generation source; " + + "register the file with `azd ai eval dataset create` instead" +} + +// FromNotBuildable reports a --from this plan cannot satisfy. +func FromNotBuildable(kind string) string { + return fmt.Sprintf("--from %s cannot be built from this plan", kind) +} + +// UnbuildableSources reports every --from the plan could not honour at once. +func UnbuildableSources(reasons []string) error { + return errors.New(strings.Join(reasons, "; ")) +} + +// JobSubmitted reports the id of a job started with --no-wait. +func JobSubmitted(jobID string) string { + return fmt.Sprintf(" submitted job %s\n", jobID) +} + +// ReattachToJob says how to come back to a job started with --no-wait. +func ReattachToJob(group, jobID string) string { + return fmt.Sprintf("\nReattach with: %s job show %s\n", group, jobID) +} + +// WroteArtifact reports where a generated artifact landed. +func WroteArtifact(path string) string { + return fmt.Sprintf(" wrote %s\n", path) +} + +// ArtifactExists reports a generation that would overwrite a checked-in file. +func ArtifactExists(path string) error { + return fmt.Errorf( + "%s already exists; pass --force to overwrite it, or --output-dir to write elsewhere", + path) +} + +// NothingGenerated reports a generation that produced no artifact. +func NothingGenerated() string { + return "Nothing was generated.\n" +} + +// ListingJobs reports a failure to list one kind of generation job. +func ListingJobs(kind string, err error) error { + return fmt.Errorf("listing %s generation jobs: %w", kind, err) +} + +// NoJobs reports a project with no generation jobs of that kind. +func NoJobs(kind string) string { + return fmt.Sprintf("No %s generation jobs found.\n", kind) +} + +// JobLine renders one generation job in a listing or a detail view. +func JobLine(jobID, status string) string { + return fmt.Sprintf("%s %s\n", jobID, status) +} + +// JobErrorLine reports why a generation job failed. +func JobErrorLine(message string) string { + return fmt.Sprintf("error: %s\n", message) +} + +// JobCancelled confirms a cancelled generation job. +func JobCancelled(kind, jobID, status string) string { + return fmt.Sprintf("Cancelled %s generation job %s (%s)\n", kind, jobID, status) +} + +// JobDeleted confirms a deleted generation job record. +func JobDeleted(kind, jobID string) string { + return fmt.Sprintf("Deleted %s generation job %s\n", kind, jobID) +} + +// JobNotFound reports a job id that is not in this group, naming the other one. +func JobNotFound(kind, jobID, other string) error { + return fmt.Errorf( + "no %s generation job %q in this project; if it generated a %s, "+ + "use the %s job group instead", kind, jobID, other, other) +} + +// ReadingJob reports a failure to read a generation job. +func ReadingJob(kind, jobID string, err error) error { + return fmt.Errorf("reading %s generation job %s: %w", kind, jobID, err) +} + +// JobFailedWithReason reports a polled job that failed and said why. +func JobFailedWithReason(status, message string) string { + return fmt.Sprintf("job failed with status %q: %s", status, message) +} + +// JobFailed reports a polled job that failed without saying why. +func JobFailed(status string) string { + return fmt.Sprintf("job failed with status %q", status) +} + +// PollerTimedOut reports a job that was still running when polling gave up. +func PollerTimedOut(operationID string, attempts int) string { + return fmt.Sprintf("operation %s did not complete within %d attempts", + operationID, attempts) +} + +// OperationIDEmpty reports a poll with nothing to poll for. +func OperationIDEmpty() error { + return errors.New("operation ID is empty") +} + +// --------------------------------------------------------------------------- +// Datasets +// --------------------------------------------------------------------------- + +// ReadingDataset reports a dataset that could not be read, by name or by path. +func ReadingDataset(dataset string, err error) error { + return fmt.Errorf("reading dataset %q: %w", dataset, err) +} + +// DatasetHasNoVersionsToRead reports a registered dataset with nothing published. +func DatasetHasNoVersionsToRead(dataset string) error { + return fmt.Errorf("dataset %q has no versions to read", dataset) +} + +// ReadingDatasetVersion reports one version of a dataset failing to read. +func ReadingDatasetVersion(dataset, version string, err error) error { + return fmt.Errorf("reading dataset %q version %s: %w", dataset, version, err) +} + +// DatasetVersionEmpty reports a published version that holds no rows. +func DatasetVersionEmpty(dataset, version string) error { + return fmt.Errorf("dataset %q version %s has no rows", dataset, version) +} + +// JSONLLineInvalid reports a row that is not JSON, by line. +func JSONLLineInvalid(line int, err error) error { + return fmt.Errorf("line %d is not valid JSON: %w", line, err) +} + +// JSONLRowInvalid reports a row that is not JSON before the file is published. +func JSONLRowInvalid(path string, line int, err error) error { + return fmt.Errorf( + "%s line %d is not valid JSON: %w. Every line must be one JSON object", + path, line, err) +} + +// JSONLRowEmpty reports a row that parses to nothing to evaluate. +func JSONLRowEmpty(path string, line int) error { + return fmt.Errorf("%s line %d is an empty object, which evaluates to nothing", path, line) +} + +// JSONLNoRows reports a dataset file with nothing in it to evaluate. +func JSONLNoRows(path string) error { + return fmt.Errorf("%s has no rows to evaluate", path) +} + +// ReadingFromFile reports a --from-file that would not stat. +func ReadingFromFile(path string, err error) error { + return fmt.Errorf("reading --from-file %q: %w", path, err) +} + +// FromFileMustBeJSONL reports a --from-file that is not a dataset. +func FromFileMustBeJSONL(path string) error { + return fmt.Errorf( + "--from-file must be a .jsonl file or a directory containing one, got %q", path) +} + +// RegisteringDataset reports the service refusing to publish the dataset. +func RegisteringDataset(dataset string, err error) error { + return fmt.Errorf("registering dataset %q: %w", dataset, err) +} + +// DatasetRegistered confirms a published dataset version. +func DatasetRegistered(dataset, version string) string { + return fmt.Sprintf("Registered dataset %s version %s\n", dataset, version) +} + +// ListingDatasets reports a failure to list the project's datasets. +func ListingDatasets(err error) error { + return fmt.Errorf("listing datasets: %w", err) +} + +// ListingDatasetVersions reports a failure to list one dataset's versions. +func ListingDatasetVersions(dataset string, err error) error { + return fmt.Errorf("listing versions of dataset %q: %w", dataset, err) +} + +// NoDatasets reports a project with no datasets to list. +func NoDatasets() string { + return "No datasets found.\n" +} + +// ResolvingLatestDatasetVersion reports a failure to find what "latest" means. +func ResolvingLatestDatasetVersion(dataset string, err error) error { + return fmt.Errorf("resolving the latest version of %q: %w", dataset, err) +} + +// DatasetHasNoVersions reports a dataset nothing was ever published under. +func DatasetHasNoVersions(dataset string) error { + return fmt.Errorf("dataset %q has no versions", dataset) +} + +// DatasetVersionNotFoundWithHint reports a dataset version the project does not hold. +func DatasetVersionNotFoundWithHint(dataset, version string) error { + return fmt.Errorf( + "no dataset %q at version %q in this project; "+ + "`azd ai eval dataset list` shows the ones there are", dataset, version) +} + +// DatasetVersionNotFound reports a dataset version there is nothing to delete at. +func DatasetVersionNotFound(dataset, version string) error { + return fmt.Errorf("no dataset %q at version %q in this project", dataset, version) +} + +// DeletingDatasetVersion reports the service refusing the delete. +func DeletingDatasetVersion(dataset, version string, err error) error { + return fmt.Errorf("deleting dataset %q version %q: %w", dataset, version, err) +} + +// DatasetDeleted confirms a deleted dataset version. +func DatasetDeleted(dataset, version string) string { + return fmt.Sprintf("Deleted dataset %s version %s\n", dataset, version) +} + +// DatasetProblem attributes a failure to the dataset it happened under. +func DatasetProblem(dataset string, err error) error { + return fmt.Errorf("dataset %q: %w", dataset, err) +} + +// DatasetSource reports a declared source that is not on disk. +func DatasetSource(path string, err error) error { + return fmt.Errorf("dataset source %q: %w", path, err) +} + +// DatasetNotLocalNorFound reports a source-less dataset the project rejected. +func DatasetNotLocalNorFound(dataset string, err error) error { + return fmt.Errorf( + "dataset %q has no local source and could not be found on the project: %w", + dataset, err) +} + +// DatasetNotLocalNorRegistered reports a source-less dataset nobody published. +func DatasetNotLocalNorRegistered(dataset string) error { + return fmt.Errorf( + "dataset %q has no local source and is not registered on the project", dataset) +} + +// DatasetVersionConflict reports a pinned version the local file disagrees with. +func DatasetVersionConflict(dataset, version string) error { + return fmt.Errorf( + "dataset %q version %s already exists and the local file differs from it. "+ + "Raise `version:` to publish the change, or drop it to let each "+ + "deploy take the next version", + dataset, version) +} + +// DatasetDrifted reports a version published outside the repo since the last deploy. +func DatasetDrifted(dataset, latest, recorded string) error { + return fmt.Errorf( + "dataset %q is at version %s on the project but %s was recorded at the last deploy; "+ + "someone published a version outside this repo. "+ + "Pin it with `version: %s` on the dataset, or pull the newer content locally, "+ + "then deploy again", + dataset, latest, recorded, latest) +} + +// ReadingDatasetDirectory reports the upload scan failing to read the directory. +func ReadingDatasetDirectory(err error) error { + return fmt.Errorf("reading directory: %w", err) +} + +// NoJSONLInDirectory reports an upload directory holding no dataset. +func NoJSONLInDirectory(dir string) error { + return fmt.Errorf("no .jsonl file found in %s", dir) +} + +// ReadingDatasetFromDir reports the upload failing to gather the local rows. +func ReadingDatasetFromDir(dir string, err error) error { + return fmt.Errorf("reading dataset from %s: %w", dir, err) +} + +// StartingPendingUpload reports the service refusing to open an upload. +func StartingPendingUpload(err error) error { + return fmt.Errorf("starting pending upload: %w", err) +} + +// NoUploadURI reports an accepted upload the service gave nowhere to write to. +func NoUploadURI() error { + return errors.New("no upload SAS URI returned from startPendingUpload") +} + +// UploadingBlob reports the dataset content failing to upload. +func UploadingBlob(err error) error { + return fmt.Errorf("uploading blob: %w", err) +} + +// ReadingDownloadCredentials reports the service refusing to hand out a read URI. +func ReadingDownloadCredentials(dataset string, err error) error { + return fmt.Errorf("reading download credentials for %q: %w", dataset, err) +} + +// NoDownloadURI reports a dataset the service gave nowhere to read from. +func NoDownloadURI(dataset string) error { + return fmt.Errorf("no download URI returned for dataset %q", dataset) +} + +// ListingDatasetContent reports a failure to list what a dataset version holds. +func ListingDatasetContent(dataset string, err error) error { + return fmt.Errorf("listing the content of dataset %q: %w", dataset, err) +} + +// DatasetHasNoFile reports a dataset version with nothing to download. +func DatasetHasNoFile(dataset string) error { + return fmt.Errorf("dataset %q holds no downloadable file", dataset) +} + +// --------------------------------------------------------------------------- +// Evaluators +// --------------------------------------------------------------------------- + +// EvaluatorNeedsFields reports required inputs the dataset does not carry. +func EvaluatorNeedsFields(evaluator string, missing []string) error { + return fmt.Errorf( + "evaluator %q requires %s, which the dataset does not provide; "+ + "add %s to the dataset, or bind it with `data_mapping`", + evaluator, quoteList(missing), pluralColumns(missing)) +} + +// EvaluatorLevelUnsupported reports an evaluation level the evaluator refuses. +func EvaluatorLevelUnsupported(evaluator, level string, supported []string) error { + return fmt.Errorf( + "evaluator %q does not support evaluation level %q; it supports %s", + evaluator, level, quoteList(supported)) +} + +// EvaluatorNeedsInitParams reports required initialization parameters left unset. +func EvaluatorNeedsInitParams(evaluator string, missing []string) error { + return fmt.Errorf( + "evaluator %q requires %s; set it under the evaluator's "+ + "`initialization_parameters` in the eval config", + evaluator, quoteList(missing)) +} + +// ReadingEvaluator reports an evaluator that could not be read, by name or path. +func ReadingEvaluator(evaluator string, err error) error { + return fmt.Errorf("reading evaluator %q: %w", evaluator, err) +} + +// EvaluatorProblem attributes a failure to the evaluator it happened under. +func EvaluatorProblem(evaluator string, err error) error { + return fmt.Errorf("evaluator %q: %w", evaluator, err) +} + +// EvaluatorSource reports a declared source that is not on disk. +func EvaluatorSource(path string, err error) error { + return fmt.Errorf("evaluator source %q: %w", path, err) +} + +// CheckingEvaluatorExists reports a failure to tell create from update. +func CheckingEvaluatorExists(evaluator string, err error) error { + return fmt.Errorf("checking whether evaluator %q exists: %w", evaluator, err) +} + +// RegisteringEvaluator reports the service refusing to publish the evaluator. +func RegisteringEvaluator(evaluator string, err error) error { + return fmt.Errorf("registering evaluator %q: %w", evaluator, err) +} + +// EvaluatorRegistered confirms a published evaluator version. +func EvaluatorRegistered(evaluator, version string) string { + return fmt.Sprintf("Registered evaluator %s version %s\n", evaluator, version) +} + +// AssetAlreadyExists reports `create` asked of a name already in use. +func AssetAlreadyExists(kind, name string) error { + return fmt.Errorf("%s %q already exists: use `update` to publish a new version", kind, name) +} + +// AssetDoesNotExist reports `update` asked of a name nobody registered. +func AssetDoesNotExist(kind, name string) error { + return fmt.Errorf("%s %q does not exist: use `create` to register it", kind, name) +} + +// DefinitionNotJSONObject reports an evaluator definition that is not an object. +func DefinitionNotJSONObject(err error) error { + return fmt.Errorf("the definition is not a JSON object: %w", err) +} + +// NotValidJSON reports an evaluator file that will not parse at all. +func NotValidJSON(err error) error { + return fmt.Errorf("not valid JSON: %w", err) +} + +// RubricMissingDimensions reports a file that is neither rubric nor document. +func RubricMissingDimensions() error { + return errors.New( + "expected a rubric definition with 'dimensions', or a document with 'definition'") +} + +// ListingEvaluators reports a failure to list the project's evaluators. +func ListingEvaluators(err error) error { + return fmt.Errorf("listing evaluators: %w", err) +} + +// ListingEvaluatorVersions reports a failure to list one evaluator's versions. +func ListingEvaluatorVersions(evaluator string, err error) error { + return fmt.Errorf("listing versions of evaluator %q: %w", evaluator, err) +} + +// NoEvaluators reports a project with no evaluators to list. +func NoEvaluators() string { + return "No evaluators found.\n" +} + +// EvaluatorNotFound reports an evaluator the project does not hold. +func EvaluatorNotFound(evaluator string) error { + return fmt.Errorf( + "no evaluator %q in this project; "+ + "`azd ai eval evaluator list` shows the ones there are", evaluator) +} + +// EvaluatorVersionNotFound reports an evaluator version there is nothing to delete at. +func EvaluatorVersionNotFound(evaluator, version string) error { + return fmt.Errorf("no evaluator %q at version %q in this project", evaluator, version) +} + +// DeletingEvaluatorVersion reports the service refusing the delete. +func DeletingEvaluatorVersion(evaluator, version string, err error) error { + return fmt.Errorf("deleting evaluator %q version %q: %w", evaluator, version, err) +} + +// EvaluatorDeleted confirms a deleted evaluator version. +func EvaluatorDeleted(evaluator, version string) string { + return fmt.Sprintf("Deleted evaluator %s version %s\n", evaluator, version) +} + +// EvaluatorNotLocalNorFound reports a source-less evaluator the project rejected. +func EvaluatorNotLocalNorFound(evaluator string, err error) error { + return fmt.Errorf( + "evaluator %q has no local source and could not be found on the project: %w", + evaluator, err) +} + +// EvaluatorDrifted reports a version published outside the repo since the last deploy. +func EvaluatorDrifted(evaluator, remote, recorded string) error { + return fmt.Errorf( + "evaluator %q is at version %s on the project but %s was recorded at the last "+ + "deploy, and the local definition does not match it: someone published a "+ + "version outside this repo. Publishing over it would leave their change "+ + "behind, so bring version %s into the declared source and deploy again, or "+ + "delete that version if it was a mistake", + evaluator, remote, recorded, remote) +} + +// EvaluatorVersionNotAdvancing reports a publish the service kept answering with +// a version that already existed. +func EvaluatorVersionNotAdvancing(evaluator, version string, waited fmt.Stringer) error { + return fmt.Errorf( + "publishing evaluator %q kept returning version %s, which already "+ + "existed. The service was still assigning that version after %s, so "+ + "version %s now holds what was just published and any eval bound to "+ + "it is scoring against it", + evaluator, version, waited, version) +} + +// EvaluatorHasNoVersions reports an evaluator nothing was ever published under. +func EvaluatorHasNoVersions(evaluator string) error { + return fmt.Errorf("evaluator %q has no versions", evaluator) +} + +// EvaluatorHasNoUsableVersion reports versions none of which can be resolved. +func EvaluatorHasNoUsableVersion(evaluator string) error { + return fmt.Errorf("evaluator %q has no usable version", evaluator) +} + +// BareEvaluatorEntry reports an evaluators: entry written as a plain string. +func BareEvaluatorEntry(name string) error { + return fmt.Errorf( + "an evaluator entry is a mapping, not a bare string: "+ + "write `- evaluator: %s`", name) +} + +// EvaluatorsMustBeSequence reports an evaluators: block that is not a list. +func EvaluatorsMustBeSequence(kind any) error { + return fmt.Errorf("evaluators must be a sequence, got %v", kind) +} + +// EvaluatorsMustBeList reports an evaluators: block that is not a JSON array. +func EvaluatorsMustBeList(err error) error { + return fmt.Errorf("evaluators must be a list: %w", err) +} + +// DecodingEvaluatorName reports an evaluator entry whose name will not decode. +func DecodingEvaluatorName(err error) error { + return fmt.Errorf("decoding evaluator name: %w", err) +} + +// DecodingEvaluator reports an evaluator entry that will not decode. +func DecodingEvaluator(err error) error { + return fmt.Errorf("decoding evaluator: %w", err) +} + +// EvaluatorEntryMissingEvaluator reports an entry that names no evaluator. +func EvaluatorEntryMissingEvaluator() error { + return errors.New("evaluator entry is missing 'evaluator'") +} + +// EvaluatorEntryMustBeMapping reports an entry that is neither map nor string. +func EvaluatorEntryMustBeMapping(kind any) error { + return fmt.Errorf("evaluator entry must be a mapping, got %v", kind) +} + +// --------------------------------------------------------------------------- +// Deploy and reconcile +// --------------------------------------------------------------------------- + +// EvalConfigInvalid reports a configuration a deploy will not act on. +func EvalConfigInvalid(err error) error { + return fmt.Errorf("eval config is invalid: %w", err) +} + +// ServiceCarriesNoConfig reports an azure.yaml entry with nothing to deploy. +func ServiceCarriesNoConfig(service string) error { + return fmt.Errorf( + "service %q carries no eval configuration; expected evaluators, datasets, or evals", + service) +} + +// ResolvingServiceRefs reports a $ref that could not be followed. +func ResolvingServiceRefs(err error) error { + return fmt.Errorf("resolving $ref in the eval service configuration: %w", err) +} + +// ReadingServiceConfig reports the service entry failing to serialize. +func ReadingServiceConfig(err error) error { + return fmt.Errorf("reading the eval service configuration: %w", err) +} + +// ParsingServiceConfig reports the service entry failing to parse. +func ParsingServiceConfig(err error) error { + return fmt.Errorf("parsing the eval service configuration: %w", err) +} + +// ReconcilingDataset reports the dataset a deploy has reached. +func ReconcilingDataset(dataset string) string { + return fmt.Sprintf("Reconciling dataset %s", dataset) +} + +// ReconcilingEvaluator reports the evaluator a deploy has reached. +func ReconcilingEvaluator(evaluator string) string { + return fmt.Sprintf("Reconciling evaluator %s", evaluator) +} + +// ReconcilingEval reports the eval a deploy has reached. +func ReconcilingEval(eval string) string { + return fmt.Sprintf("Reconciling eval %s", eval) +} + +// PublishedVersion reports an artifact a deploy published. +func PublishedVersion(kind, name, version string) string { + return fmt.Sprintf("Published %s %s version %s", kind, name, version) +} + +// UnchangedAtVersion reports an artifact a deploy left alone. +func UnchangedAtVersion(kind, name, version string) string { + return fmt.Sprintf("%s %s is unchanged at version %s", + strings.ToUpper(kind[:1])+kind[1:], name, version) +} + +// EvalProblem attributes a failure to the eval it happened under. +func EvalProblem(eval string, err error) error { + return fmt.Errorf("eval %q: %w", eval, err) +} + +// EvalIs reports the id a deployed eval resolved to. +func EvalIs(eval, id string) string { + return fmt.Sprintf("Eval %s is %s", eval, id) +} + +// EvalCreated confirms a single eval created outside a full deploy. +func EvalCreated(eval, id string) string { + return fmt.Sprintf("%s Created eval: %s (%s)\n", DoneMark, eval, id) +} + +// ListingEvals reports a failure to list the project's evals. +func ListingEvals(err error) error { + return fmt.Errorf("listing evals: %w", err) +} + +// NoEvals reports a project with no evals to list. +func NoEvals() string { + return "No evals found.\n" +} + +// EvalNotFound reports an eval id the project does not hold. +func EvalNotFound(evalID string) error { + return fmt.Errorf( + "no eval %q in this project; "+ + "`azd ai eval list` shows the ones there are", evalID) +} + +// EvalGone reports an eval id there is nothing to delete at. +func EvalGone(evalID string) error { + return fmt.Errorf("no eval %q in this project", evalID) +} + +// ReadingEval reports a failure to read one eval. +func ReadingEval(evalID string, err error) error { + return fmt.Errorf("reading eval %q: %w", evalID, err) +} + +// DeletingEval reports the service refusing the delete. +func DeletingEval(evalID string, err error) error { + return fmt.Errorf("deleting eval %q: %w", evalID, err) +} + +// EvalDeleted confirms a deleted eval. +func EvalDeleted(evalID string) string { + return fmt.Sprintf("Deleted eval %s\n", evalID) +} + +// Hashing reports a local artifact that could not be fingerprinted. +func Hashing(path string, err error) error { + return fmt.Errorf("hashing %q: %w", path, err) +} + +// HashingEval reports an eval declaration that could not be fingerprinted. +func HashingEval(eval string, err error) error { + return fmt.Errorf("hashing eval %q: %w", eval, err) +} + +// --------------------------------------------------------------------------- +// Config +// --------------------------------------------------------------------------- + +// NoAzdProject reports a command that found no project to attach to. +func NoAzdProject() error { + return errors.New( + "no azd project found in this directory. Run `azd init` first, " + + "or run this from the root of an existing one; the eval service is added to " + + "its azure.yaml") +} + +// ConnectingToAzd reports the azd daemon being unreachable. +func ConnectingToAzd(err error) error { + return fmt.Errorf("connecting to azd: %w", err) +} + +// CreatingCredential reports the Azure credential failing to build. +func CreatingCredential(err error) error { + return fmt.Errorf("creating Azure credential: %w", err) +} + +// ErrNoAzdEnvironment reports that there is no azd environment to persist into. +var ErrNoAzdEnvironment = errors.New("no active azd environment") + +// NoAzdEnvironmentToWrite reports a value with nowhere to be remembered. +func NoAzdEnvironmentToWrite(key string) error { + return fmt.Errorf("%w to write %s into", ErrNoAzdEnvironment, key) +} + +// WritingEnvValue reports the azd environment refusing a write. +func WritingEnvValue(key string, err error) error { + return fmt.Errorf("writing %s to the azd environment: %w", key, err) +} + +// BuildingServiceEntry reports the eval service entry failing to build. +func BuildingServiceEntry(err error) error { + return fmt.Errorf("building the eval service entry: %w", err) +} + +// AddingServiceTo reports azd refusing to add the eval service. +func AddingServiceTo(rootConfig string, err error) error { + return fmt.Errorf("adding the eval service to %s: %w", rootConfig, err) +} + +// SourceNotADataSource reports an --source that names nothing rows come from. +func SourceNotADataSource(source, dataset, traces string) error { + return fmt.Errorf("--source %q is not a data source; use %q or %q", source, dataset, traces) +} + +// TracesTakesNoDataset reports --dataset paired with a trace-backed eval. +func TracesTakesNoDataset() error { + return errors.New("--source traces reads production traces, so it takes no --dataset") +} + +// MaxTracesNeedsTraceSource reports --max-traces without a trace-backed eval. +func MaxTracesNeedsTraceSource() error { + return errors.New("--max-traces caps a trace-backed eval; pass --source traces") +} + +// MaxTracesMustBePositive reports a negative --max-traces. +func MaxTracesMustBePositive() error { + return errors.New("--max-traces must be positive") +} + +// EvalAlreadyDeclared reports an init that would overwrite a hand-tuned eval. +func EvalAlreadyDeclared(eval, configPath string) error { + return fmt.Errorf( + "an eval named %q already exists in %s; choose another name with --name, "+ + "or pass --force to replace it. `init` only adds: editing an eval is a file edit", + eval, configPath) +} + +// CreatingDatasetsDir reports the datasets directory failing to be created. +func CreatingDatasetsDir(err error) error { + return fmt.Errorf("creating the datasets directory: %w", err) +} + +// CreatingEvaluatorsDir reports the evaluators directory failing to be created. +func CreatingEvaluatorsDir(err error) error { + return fmt.Errorf("creating the evaluators directory: %w", err) +} + +// DetectedTarget reports the agent the scaffolded eval will evaluate. +func DetectedTarget(target string) string { + return fmt.Sprintf("%s Detected agent target: %s\n", DoneMark, target) +} + +// UsingTraceSource reports a scaffold that reads production traces. +func UsingTraceSource() string { + return fmt.Sprintf("%s Using data source: traces (Application Insights)\n", DoneMark) +} + +// JudgeModelDeployment reports the deployment the graders will judge with. +func JudgeModelDeployment(model string) string { + return fmt.Sprintf("%s Judge model deployment: %s\n", DoneMark, model) +} + +// CreatedHeading opens the list of what a scaffold wrote. +func CreatedHeading() string { + return "\nCreated\n" +} + +// CreatedConfigLine names the configuration a scaffold wrote. +func CreatedConfigLine(configPath string) string { + return fmt.Sprintf(" %-33s evaluation configuration\n", configPath) +} + +// AddedServiceLine reports the eval service being added to the root config. +func AddedServiceLine(rootConfig, service string) string { + return fmt.Sprintf(" %-33s added service '%s'\n", rootConfig, service) +} + +// AlreadyDeclaresServiceLine reports a root config that already referenced the eval. +func AlreadyDeclaresServiceLine(rootConfig, service string) string { + return fmt.Sprintf(" %-33s already declares service '%s'\n", rootConfig, service) +} + +// FirstNextStep opens the list of commands to run after a scaffold. +func FirstNextStep(step string) string { + return fmt.Sprintf("\nNext: %s\n", step) +} + +// FurtherNextStep continues the list of commands to run after a scaffold. +func FurtherNextStep(step string) string { + return fmt.Sprintf(" %s\n", step) +} + +// CreatedCatalogFile reports a configuration created to hold a catalog entry. +func CreatedCatalogFile(configPath string) string { + return fmt.Sprintf("%s Created %s with the catalog entry\n", DoneMark, configPath) +} + +// AddedToCatalog reports a generated artifact recorded in the configuration. +func AddedToCatalog(kind, artifact, configPath string) string { + return fmt.Sprintf("%s Added %s %s to %s\n", DoneMark, kind, artifact, configPath) +} + +// ArtifactDescription names a catalogued artifact, with its version when there is one. +func ArtifactDescription(name, version string) string { + if version == "" || version == "latest" { + return fmt.Sprintf("'%s'", name) + } + return fmt.Sprintf("'%s' (version %s)", name, version) +} + +// NoEvalsDeclared reports a configuration with nothing to act on. +func NoEvalsDeclared() error { + return errors.New("no evals are declared") +} + +// SeveralEvalsDeclared reports an unnamed eval where guessing would be wrong. +func SeveralEvalsDeclared(count int, names []string) error { + return fmt.Errorf( + "this configuration declares %d evals (%s); choose one with --eval", + count, strings.Join(names, ", ")) +} + +// EvalNotDeclared reports a name the configuration does not carry. +func EvalNotDeclared(eval string, names []string) error { + return fmt.Errorf( + "eval %q is not declared; this configuration has %s", + eval, strings.Join(names, ", ")) +} + +// AtLeastOneEvalRequired reports a configuration that declares no eval. +func AtLeastOneEvalRequired() error { + return errors.New("at least one eval is required") +} + +// EvalNameRequired reports an eval entry with no name. +func EvalNameRequired(index int) error { + return fmt.Errorf("evals[%d]: 'name' is required", index) +} + +// DuplicateEvalName reports two evals answering to the same name. +func DuplicateEvalName(index int, eval string) error { + return fmt.Errorf("evals[%d]: duplicate eval name %q", index, eval) +} + +// EvalsIdenticalApartFromName reports two evals nothing can tell apart once deployed. +func EvalsIdenticalApartFromName(index int, eval, first string) error { + return fmt.Errorf( + "evals[%d] (%s): identical to %q apart from its name and description; "+ + "give them different evaluators, datasets or settings, or declare one", + index, eval, first) +} + +// DatasetNameRequired reports a catalog entry with no name. +func DatasetNameRequired(index int) error { + return fmt.Errorf("datasets[%d]: 'name' is required", index) +} + +// DuplicateDatasetName reports two catalog entries answering to the same name. +func DuplicateDatasetName(index int, dataset string) error { + return fmt.Errorf("datasets[%d]: duplicate dataset name %q", index, dataset) +} + +// EvaluatorNameRequired reports a catalog entry with no name. +func EvaluatorNameRequired(index int) error { + return fmt.Errorf("evaluators[%d]: 'name' is required", index) +} + +// DuplicateEvaluatorName reports two catalog entries answering to the same name. +func DuplicateEvaluatorName(index int, evaluator string) error { + return fmt.Errorf("evaluators[%d]: duplicate evaluator name %q", index, evaluator) +} + +// BuiltinNeedsNoCatalogEntry reports a built-in declared as though it were custom. +func BuiltinNeedsNoCatalogEntry(index int, evaluator string) error { + return fmt.Errorf( + "evaluators[%d] (%s): a built-in needs no catalog entry; reference it "+ + "straight from an eval", index, evaluator) +} + +// EvaluatorVersionWithSource reports a pin the service would assign anyway. +func EvaluatorVersionWithSource(index int, evaluator string) error { + return fmt.Errorf( + "evaluators[%d] (%s): `version` cannot be set with `source`, because the "+ + "service assigns the version when it publishes. Drop `version` to "+ + "publish this file, or drop `source` to reference a version already "+ + "on the project", index, evaluator) +} + +// DatasetAndSourceBothDeclared reports two answers to where rows come from. +func DatasetAndSourceBothDeclared(index int, eval string) error { + return fmt.Errorf( + "evals[%d] (%s): `dataset` and `source` both say where rows come from; "+ + "declare one", index, eval) +} + +// DatasetNotInDatasetsCatalog reports an eval naming a dataset nobody declared. +func DatasetNotInDatasetsCatalog(index int, eval, dataset string) error { + return fmt.Errorf( + "evals[%d] (%s): dataset %q is not in the datasets catalog", + index, eval, dataset) +} + +// SourceTypeRequired reports a source: block that does not say what it reads. +func SourceTypeRequired(index int, eval string) error { + return fmt.Errorf("evals[%d] (%s): source.type is required", index, eval) +} + +// SourceTypeUnsupported reports a source.type the extension has no path for. +func SourceTypeUnsupported(index int, eval, got, traces, responses string) error { + return fmt.Errorf( + "evals[%d] (%s): source.type %q is not supported; use %q or %q", + index, eval, got, traces, responses) +} + +// AtLeastOneEvaluatorRequired reports an eval that scores nothing. +func AtLeastOneEvaluatorRequired(index int, eval string) error { + return fmt.Errorf("evals[%d] (%s): at least one evaluator is required", index, eval) +} + +// EvaluatorFieldRequired reports an evaluators: entry with no evaluator named. +func EvaluatorFieldRequired(evalIndex, refIndex int) error { + return fmt.Errorf("evals[%d].evaluators[%d]: 'evaluator' is required", evalIndex, refIndex) +} + +// DuplicateCriterion reports two result rows nothing could tell apart. +func DuplicateCriterion(evalIndex, refIndex int, criterion string) error { + return fmt.Errorf( + "evals[%d].evaluators[%d]: duplicate criterion %q; give one a `name`", + evalIndex, refIndex, criterion) +} + +// EvaluatorNotInCatalog reports a reference to an evaluator nobody declared. +func EvaluatorNotInCatalog(evalIndex, refIndex int, evaluator string) error { + return fmt.Errorf( + "evals[%d].evaluators[%d]: evaluator %q is not in the evaluators catalog", + evalIndex, refIndex, evaluator) +} + +// TargetTypeUnsupported reports a target.type the extension cannot invoke. +func TargetTypeUnsupported(index int, eval, got, agent, model string) error { + return fmt.Errorf( + "evals[%d] (%s): target.type %q is not supported; use %q or %q", + index, eval, got, agent, model) +} + +// EvaluationLevelInvalid reports a scoring granularity the service does not accept. +func EvaluationLevelInvalid(index int, eval, got, turn, conversation string) error { + return fmt.Errorf( + "evals[%d] (%s): evaluation_level %q is invalid; expected %q or %q", + index, eval, got, turn, conversation) +} + +// ReadingEvalConfig reports a configuration file that would not read. +func ReadingEvalConfig(path string, err error) error { + return fmt.Errorf("reading eval config %q: %w", path, err) +} + +// ParsingEvalConfig reports a configuration file that would not parse. +func ParsingEvalConfig(path string, err error) error { + return fmt.Errorf("parsing eval config %q: %w", path, err) +} + +// SerializingEvalConfig reports a configuration that would not serialize. +func SerializingEvalConfig(err error) error { + return fmt.Errorf("serializing eval config: %w", err) +} + +// WritingEvalConfig reports a configuration file that would not be written. +func WritingEvalConfig(path string, err error) error { + return fmt.Errorf("writing eval config %q: %w", path, err) +} + +// ErrAmbiguousAgentService reports that a target name matched more than one service. +var ErrAmbiguousAgentService = errors.New("more than one agent service matches") + +// AmbiguousAgentService reports a --target that names no single set of instructions. +func AmbiguousAgentService(agent string, matched []string) error { + return fmt.Errorf( + "%w %q: %s. Name one of them with --target, or pass the text with "+ + "--agent-instruction", + ErrAmbiguousAgentService, agent, strings.Join(matched, ", ")) +} + +// InstructionFileUnreadable reports optimize metadata pointing at a missing file. +func InstructionFileUnreadable(metadataPath, named string, err error) error { + return fmt.Errorf( + "%s names instruction_file %q, which could not be read: %w", + metadataPath, named, err) +} + +// FromNotASource reports a --from value the generation service has no path for. +func FromNotASource(from string, sources []string) error { + return fmt.Errorf( + "--from %q is not a source; use one of %s", + from, strings.Join(sources, ", ")) +} + +// SampleSizeOutOfRange reports a row count the generation service would reject. +func SampleSizeOutOfRange(min, max, got int) error { + return fmt.Errorf("sample size must be between %d and %d, got %d", min, max, got) +} + +// EndpointEmpty reports a project endpoint given as blank. +func EndpointEmpty() error { + return exterrors.Validation( + exterrors.CodeInvalidParameter, + "project endpoint must not be empty", + "provide a Foundry project endpoint URL "+ + "(e.g. https://.services.ai.azure.com/api/projects/)", + ) +} + +// EndpointUnparseable reports a project endpoint that is not a URL. +func EndpointUnparseable(err error) error { + return exterrors.Validation( + exterrors.CodeInvalidParameter, + fmt.Sprintf("invalid project endpoint URL: %v", err), + "provide a valid https:// Foundry project endpoint URL", + ) +} + +// EndpointNotHTTPS reports a project endpoint on the wrong scheme. +func EndpointNotHTTPS() error { + return exterrors.Validation( + exterrors.CodeInvalidParameter, + "project endpoint must use https", + "provide an https:// URL", + ) +} + +// EndpointNotFoundryHost reports a project endpoint pointing somewhere else. +func EndpointNotFoundryHost(host, suffix string) error { + return exterrors.Validation( + exterrors.CodeInvalidParameter, + fmt.Sprintf( + "project endpoint host %q is not a recognized Foundry host (*%s)", + host, suffix, + ), + "the host must end with "+suffix, + ) +} + +// EndpointHasPort reports a project endpoint carrying an explicit port. +func EndpointHasPort(host string) error { + return exterrors.Validation( + exterrors.CodeInvalidParameter, + fmt.Sprintf("project endpoint host %q must not include a port", host), + "remove the explicit port from the URL", + ) +} + +// NoEndpoint reports a project endpoint that no source could supply. +func NoEndpoint() error { + return exterrors.Dependency( + exterrors.CodeMissingProjectEndpoint, + "no Foundry project endpoint resolved", + "persist a workspace default with `azd ai project set `, "+ + "or set FOUNDRY_PROJECT_ENDPOINT (or AZURE_AI_PROJECT_ENDPOINT) "+ + "in the active azd environment, "+ + "or export FOUNDRY_PROJECT_ENDPOINT (or AZURE_AI_PROJECT_ENDPOINT) in your shell", + ) +} + +// ProjectContextClient reports the config helper failing to build. +func ProjectContextClient(err error) error { + return fmt.Errorf("getProjectContext: %w", err) +} + +// ProjectContextRead reports the persisted project context failing to read. +func ProjectContextRead(err error) error { + return fmt.Errorf("getProjectContext: failed to read config: %w", err) +} + +// --------------------------------------------------------------------------- +// Output +// --------------------------------------------------------------------------- + +// Progress markers from the azd style guide, so the extension's lines sit +// alongside core's without a second vocabulary. +const ( + DoneMark = "(✓) Done:" // finished successfully + SkippedMark = "(-) Skipped:" // intentionally not done, not a failure + FailedMark = "(x) Failed:" // the step did not complete +) + +// Warning reports a problem that is not worth failing the command over. +func Warning(err error) string { + return fmt.Sprintf("warning: %v\n", err) +} + +// PortalLink closes a detail view with the asset's portal URL. +func PortalLink(url string) string { + return fmt.Sprintf("Portal: %s\n", url) +} + +// FlagRequired reports a value that cannot be prompted for. +func FlagRequired(name string) error { + return fmt.Errorf("--%s is required (running with --no-prompt)", name) +} + +// Creating reports a directory or file that could not be created. +func Creating(path string, err error) error { + return fmt.Errorf("creating %q: %w", path, err) +} + +// Serializing reports a value that could not be written out. +func Serializing(path string, err error) error { + return fmt.Errorf("serializing %q: %w", path, err) +} + +// Writing reports a file that could not be written. +func Writing(path string, err error) error { + return fmt.Errorf("writing %q: %w", path, err) +} + +// ReadingPath reports a file or directory that could not be read. +func ReadingPath(path string, err error) error { + return fmt.Errorf("reading %s: %w", path, err) +} + +// quoteList renders names as a readable "a", "b" and "c". +func quoteList(values []string) string { + if len(values) == 0 { + return "nothing" + } + quoted := make([]string, 0, len(values)) + for _, value := range values { + quoted = append(quoted, fmt.Sprintf("%q", value)) + } + sort.Strings(quoted) + if len(quoted) == 1 { + return quoted[0] + } + return strings.Join(quoted[:len(quoted)-1], ", ") + " and " + quoted[len(quoted)-1] +} + +// pluralColumns agrees with however many columns are missing. +func pluralColumns(values []string) string { + if len(values) == 1 { + return "that column" + } + return "those columns" +} + +// --------------------------------------------------------------------------- +// Talking to the service +// --------------------------------------------------------------------------- + +// InvalidEndpointURL reports a client built on an endpoint that will not parse. +func InvalidEndpointURL(err error) error { + return fmt.Errorf("invalid endpoint URL: %w", err) +} + +// InvalidRequestPath reports a request path that will not parse. +func InvalidRequestPath(path string, err error) error { + return fmt.Errorf("invalid request path %q: %w", path, err) +} + +// CreatingRequest reports a request that could not be built. +func CreatingRequest(err error) error { + return fmt.Errorf("failed to create request: %w", err) +} + +// MarshalingRequest reports a request body that would not serialize. +func MarshalingRequest(err error) error { + return fmt.Errorf("failed to marshal request: %w", err) +} + +// SettingRequestBody reports a request body that would not attach. +func SettingRequestBody(err error) error { + return fmt.Errorf("failed to set request body: %w", err) +} + +// RequestFailed reports a request that never reached an answer. +func RequestFailed(err error) error { + return fmt.Errorf("HTTP request failed: %w", err) +} + +// ReadingResponseBody reports a response that could not be read. +func ReadingResponseBody(err error) error { + return fmt.Errorf("failed to read response body: %w", err) +} + +// ParsingResponse reports a response that could not be parsed. +func ParsingResponse(err error) error { + return fmt.Errorf("failed to parse response: %w", err) +} + +// ParsingNumber reports a numeric field that did not arrive as a number. +func ParsingNumber(data string, err error) error { + return fmt.Errorf("parsing number %s: %w", data, err) +} + +// InvalidContainerURI reports a storage URI the service handed back unusable. +func InvalidContainerURI(err error) error { + return fmt.Errorf("invalid container SAS URI: %w", err) +} + +// CreatingUploadRequest reports the blob upload request failing to build. +func CreatingUploadRequest(err error) error { + return fmt.Errorf("failed to create upload request: %w", err) +} + +// UploadingBlobFailed reports the blob upload never reaching an answer. +func UploadingBlobFailed(err error) error { + return fmt.Errorf("failed to upload blob: %w", err) +} + +// BlobUploadStatus reports storage refusing the upload. +func BlobUploadStatus(status int, body string) error { + return fmt.Errorf("blob upload failed with status %d: %s", status, body) +} + +// CreatingDownloadRequest reports the dataset download request failing to build. +func CreatingDownloadRequest(err error) error { + return fmt.Errorf("failed to create download request: %w", err) +} + +// DownloadingDatasetBlob reports the dataset download never reaching an answer. +func DownloadingDatasetBlob(err error) error { + return fmt.Errorf("failed to download dataset from blob: %w", err) +} + +// BlobDownloadStatus reports storage refusing the download. +func BlobDownloadStatus(status int) error { + return fmt.Errorf("blob download failed with status %d", status) +} + +// ReadingDatasetContent reports a downloaded dataset that could not be read. +func ReadingDatasetContent(err error) error { + return fmt.Errorf("failed to read dataset content: %w", err) +} + +// CreatingListRequest reports the container listing request failing to build. +func CreatingListRequest(err error) error { + return fmt.Errorf("failed to create list request: %w", err) +} + +// ListingContainerBlobs reports the container listing never reaching an answer. +func ListingContainerBlobs(err error) error { + return fmt.Errorf("failed to list container blobs: %w", err) +} + +// ContainerListStatus reports storage refusing the listing. +func ContainerListStatus(status int) error { + return fmt.Errorf("container list failed with status %d", status) +} + +// ReadingListResponse reports a container listing that could not be read. +func ReadingListResponse(err error) error { + return fmt.Errorf("failed to read list response: %w", err) +} + +// CreatingBlobDownloadRequest reports the blob download request failing to build. +func CreatingBlobDownloadRequest(err error) error { + return fmt.Errorf("failed to create blob download request: %w", err) +} + +// DownloadingBlob reports one blob's download never reaching an answer. +func DownloadingBlob(err error) error { + return fmt.Errorf("failed to download blob: %w", err) +} + +// BlobDownloadStatusFor reports storage refusing one named blob. +func BlobDownloadStatusFor(status int, blobName string) error { + return fmt.Errorf("blob download failed with status %d for %s", status, blobName) +} + +// ReadingBlobContent reports a downloaded blob that could not be read. +func ReadingBlobContent(err error) error { + return fmt.Errorf("failed to read blob content: %w", err) +} + +// ParsingProjectResourceID reports a project ARM id that will not parse. +func ParsingProjectResourceID(err error) error { + return fmt.Errorf("failed to parse project resource ID: %w", err) +} + +// EncodingSubscriptionID reports a subscription that would not encode for a URL. +func EncodingSubscriptionID(err error) error { + return fmt.Errorf("failed to encode subscription ID: %w", err) +} + +// NotAFoundryProjectResourceID reports an ARM id that names something else. +func NotAFoundryProjectResourceID(resourceID string) error { + return fmt.Errorf( + "resource ID does not represent a Foundry project (missing parent account): %s", + resourceID) +} + +// InvalidSubscriptionID reports a subscription id that is not a GUID. +func InvalidSubscriptionID(err error) error { + return fmt.Errorf("invalid subscription ID format: %w", err) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/models.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/models.go index 13fe6da2d9d..aa306c24e20 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/models.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/models.go @@ -4,12 +4,13 @@ package dataset_api import ( - "fmt" "math" "os" "path/filepath" "strconv" "strings" + + "azureaieval/internal/messages" ) // CreateDatasetRequest is the request body for creating (uploading) a dataset. @@ -192,7 +193,7 @@ func NextVersion(current string) string { func ReadFirstJSONLFile(dir string) (string, error) { entries, err := os.ReadDir(dir) if err != nil { - return "", fmt.Errorf("reading directory: %w", err) + return "", messages.ReadingDatasetDirectory(err) } for _, e := range entries { if e.IsDir() { @@ -201,10 +202,10 @@ func ReadFirstJSONLFile(dir string) (string, error) { if filepath.Ext(e.Name()) == ".jsonl" { data, err := os.ReadFile(filepath.Join(dir, e.Name())) //nolint:gosec // local artifact path if err != nil { - return "", fmt.Errorf("reading %s: %w", e.Name(), err) + return "", messages.ReadingPath(e.Name(), err) } return string(data), nil } } - return "", fmt.Errorf("no .jsonl file found in %s", dir) + return "", messages.NoJSONLInDirectory(dir) } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/operations.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/operations.go index d6d5323dd1c..78542fb267b 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/operations.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/operations.go @@ -17,6 +17,7 @@ import ( "path" "strings" + "azureaieval/internal/messages" "azureaieval/internal/version" "github.com/Azure/azure-sdk-for-go/sdk/azcore" @@ -197,7 +198,7 @@ func (c *DatasetClient) UploadVersion( ) (*Dataset, error) { content, err := ReadFirstJSONLFile(localDir) if err != nil { - return nil, fmt.Errorf("reading dataset from %s: %w", localDir, err) + return nil, messages.ReadingDatasetFromDir(localDir, err) } newVersion := version @@ -205,18 +206,18 @@ func (c *DatasetClient) UploadVersion( // Step 1: Start pending upload to get a SAS URI. pending, err := c.StartPendingUpload(ctx, name, newVersion, apiVersion) if err != nil { - return nil, fmt.Errorf("starting pending upload: %w", err) + return nil, messages.StartingPendingUpload(err) } uploadURI := pending.ResolvedUploadURI() if uploadURI == "" { - return nil, fmt.Errorf("no upload SAS URI returned from startPendingUpload") + return nil, messages.NoUploadURI() } // Step 2: Upload the JSONL file to blob storage. blobName := name + ".jsonl" if err := c.UploadBlob(ctx, uploadURI, blobName, []byte(content)); err != nil { - return nil, fmt.Errorf("uploading blob: %w", err) + return nil, messages.UploadingBlob(err) } // Step 3: Finalize the dataset version with the full blob URI. @@ -243,7 +244,7 @@ func (c *DatasetClient) StartPendingUpload( func (c *DatasetClient) UploadBlob(ctx context.Context, containerSASUri, blobName string, data []byte) error { u, err := url.Parse(containerSASUri) if err != nil { - return fmt.Errorf("invalid container SAS URI: %w", err) + return messages.InvalidContainerURI(err) } // Append blob name to the container path. @@ -251,7 +252,7 @@ func (c *DatasetClient) UploadBlob(ctx context.Context, containerSASUri, blobNam req, err := http.NewRequestWithContext(ctx, http.MethodPut, u.String(), bytes.NewReader(data)) if err != nil { - return fmt.Errorf("failed to create upload request: %w", err) + return messages.CreatingUploadRequest(err) } req.Header.Set("x-ms-blob-type", "BlockBlob") req.Header.Set("Content-Type", "application/octet-stream") @@ -259,13 +260,13 @@ func (c *DatasetClient) UploadBlob(ctx context.Context, containerSASUri, blobNam httpClient := &http.Client{} resp, err := httpClient.Do(req) if err != nil { - return fmt.Errorf("failed to upload blob: %w", err) + return messages.UploadingBlobFailed(err) } defer resp.Body.Close() if resp.StatusCode != http.StatusCreated && resp.StatusCode != http.StatusOK { body, _ := io.ReadAll(resp.Body) - return fmt.Errorf("blob upload failed with status %d: %s", resp.StatusCode, string(body)) + return messages.BlobUploadStatus(resp.StatusCode, string(body)) } return nil @@ -334,12 +335,12 @@ func (c *DatasetClient) DownloadDatasetContent( ) ([]byte, error) { cred, err := c.GetDatasetCredential(ctx, name, version, apiVersion) if err != nil { - return nil, fmt.Errorf("reading download credentials for %q: %w", name, err) + return nil, messages.ReadingDownloadCredentials(name, err) } sasURI := cred.ResolvedDownloadURI() if sasURI == "" { - return nil, fmt.Errorf("no download URI returned for dataset %q", name) + return nil, messages.NoDownloadURI(name) } // A URI whose last path segment carries a file extension is the blob @@ -354,11 +355,11 @@ func (c *DatasetClient) DownloadDatasetContent( names, err := c.ListContainerBlobs(ctx, sasURI) if err != nil { - return nil, fmt.Errorf("listing the content of dataset %q: %w", name, err) + return nil, messages.ListingDatasetContent(name, err) } blobName := pickDatasetBlob(names) if blobName == "" { - return nil, fmt.Errorf("dataset %q holds no downloadable file", name) + return nil, messages.DatasetHasNoFile(name) } return c.DownloadBlob(ctx, sasURI, blobName) } @@ -395,7 +396,7 @@ func pickDatasetBlob(names []string) string { func (c *DatasetClient) DownloadDataset(ctx context.Context, downloadURL string) ([]byte, error) { req, err := runtime.NewRequest(ctx, http.MethodGet, downloadURL) if err != nil { - return nil, fmt.Errorf("failed to create download request: %w", err) + return nil, messages.CreatingDownloadRequest(err) } // Use a plain HTTP client for blob downloads — the SAS token in the URL provides @@ -404,17 +405,17 @@ func (c *DatasetClient) DownloadDataset(ctx context.Context, downloadURL string) httpClient := &http.Client{} resp, err := httpClient.Do(req.Raw()) if err != nil { - return nil, fmt.Errorf("failed to download dataset from blob: %w", err) + return nil, messages.DownloadingDatasetBlob(err) } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("blob download failed with status %d", resp.StatusCode) + return nil, messages.BlobDownloadStatus(resp.StatusCode) } data, err := io.ReadAll(resp.Body) if err != nil { - return nil, fmt.Errorf("failed to read dataset content: %w", err) + return nil, messages.ReadingDatasetContent(err) } log.Printf("[dataset_api] downloaded %d bytes", len(data)) @@ -428,7 +429,7 @@ func (c *DatasetClient) ListContainerBlobs(ctx context.Context, containerSASUri // Parse the container URI and append list query parameters. u, err := url.Parse(containerSASUri) if err != nil { - return nil, fmt.Errorf("invalid container SAS URI: %w", err) + return nil, messages.InvalidContainerURI(err) } q := u.Query() @@ -440,23 +441,23 @@ func (c *DatasetClient) ListContainerBlobs(ctx context.Context, containerSASUri req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil) if err != nil { - return nil, fmt.Errorf("failed to create list request: %w", err) + return nil, messages.CreatingListRequest(err) } httpClient := &http.Client{} resp, err := httpClient.Do(req) if err != nil { - return nil, fmt.Errorf("failed to list container blobs: %w", err) + return nil, messages.ListingContainerBlobs(err) } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("container list failed with status %d", resp.StatusCode) + return nil, messages.ContainerListStatus(resp.StatusCode) } body, err := io.ReadAll(resp.Body) if err != nil { - return nil, fmt.Errorf("failed to read list response: %w", err) + return nil, messages.ReadingListResponse(err) } // Parse XML blob listing to extract blob names. @@ -470,7 +471,7 @@ func (c *DatasetClient) ListContainerBlobs(ctx context.Context, containerSASUri func (c *DatasetClient) DownloadBlob(ctx context.Context, containerSASUri, blobName string) ([]byte, error) { u, err := url.Parse(containerSASUri) if err != nil { - return nil, fmt.Errorf("invalid container SAS URI: %w", err) + return nil, messages.InvalidContainerURI(err) } // Append blob name to the container path. @@ -478,23 +479,23 @@ func (c *DatasetClient) DownloadBlob(ctx context.Context, containerSASUri, blobN req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil) if err != nil { - return nil, fmt.Errorf("failed to create blob download request: %w", err) + return nil, messages.CreatingBlobDownloadRequest(err) } httpClient := &http.Client{} resp, err := httpClient.Do(req) if err != nil { - return nil, fmt.Errorf("failed to download blob: %w", err) + return nil, messages.DownloadingBlob(err) } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("blob download failed with status %d for %s", resp.StatusCode, blobName) + return nil, messages.BlobDownloadStatusFor(resp.StatusCode, blobName) } data, err := io.ReadAll(resp.Body) if err != nil { - return nil, fmt.Errorf("failed to read blob content: %w", err) + return nil, messages.ReadingBlobContent(err) } log.Printf("[dataset_api] downloaded blob %s (%d bytes)", blobName, len(data)) @@ -539,7 +540,7 @@ func (c *DatasetClient) doRequest( ) ([]byte, error) { u, err := url.Parse(c.endpoint) if err != nil { - return nil, fmt.Errorf("invalid endpoint URL: %w", err) + return nil, messages.InvalidEndpointURL(err) } // Callers escape the name and version they interpolate, so the path is set @@ -548,7 +549,7 @@ func (c *DatasetClient) doRequest( escapedPath := u.EscapedPath() + path decodedPath, err := url.PathUnescape(escapedPath) if err != nil { - return nil, fmt.Errorf("invalid request path %q: %w", escapedPath, err) + return nil, messages.InvalidRequestPath(escapedPath, err) } u.Path, u.RawPath = decodedPath, escapedPath @@ -563,7 +564,7 @@ func (c *DatasetClient) doRequest( req, err := runtime.NewRequest(ctx, method, u.String()) if err != nil { - return nil, fmt.Errorf("failed to create request: %w", err) + return nil, messages.CreatingRequest(err) } log.Printf("[dataset_api] %s %s", method, u.Redacted()) @@ -571,22 +572,22 @@ func (c *DatasetClient) doRequest( if body != nil { payload, err := json.Marshal(body) if err != nil { - return nil, fmt.Errorf("failed to marshal request: %w", err) + return nil, messages.MarshalingRequest(err) } if err := req.SetBody(streaming.NopCloser(bytes.NewReader(payload)), "application/json"); err != nil { - return nil, fmt.Errorf("failed to set request body: %w", err) + return nil, messages.SettingRequestBody(err) } } resp, err := c.pipeline.Do(req) if err != nil { - return nil, fmt.Errorf("HTTP request failed: %w", err) + return nil, messages.RequestFailed(err) } defer resp.Body.Close() respBody, err := io.ReadAll(resp.Body) if err != nil { - return nil, fmt.Errorf("failed to read response body: %w", err) + return nil, messages.ReadingResponseBody(err) } log.Printf("[dataset_api] response status: %d", resp.StatusCode) @@ -624,7 +625,7 @@ func doRequestTyped[T any]( var result T if err := json.Unmarshal(respBody, &result); err != nil { - return nil, fmt.Errorf("failed to parse response: %w", err) + return nil, messages.ParsingResponse(err) } return &result, nil diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/insights.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/insights.go index 0335f951d62..3a71ccd4970 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/insights.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/insights.go @@ -12,6 +12,8 @@ import ( "net/url" "strconv" "strings" + + "azureaieval/internal/messages" ) // InsightTypeEvaluationComparison compares evaluation runs. The service also @@ -65,7 +67,7 @@ func (f *LenientFloat) UnmarshalJSON(data []byte) error { } v, err := strconv.ParseFloat(s, 64) if err != nil { - return fmt.Errorf("parsing number %s: %w", data, err) + return messages.ParsingNumber(string(data), err) } *f = LenientFloat(v) return nil diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/operations.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/operations.go index 100e9d87b6d..0b05a9a31df 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/operations.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/operations.go @@ -15,6 +15,7 @@ import ( "strconv" "time" + "azureaieval/internal/messages" "azureaieval/internal/version" "github.com/Azure/azure-sdk-for-go/sdk/azcore" @@ -322,12 +323,8 @@ func (c *EvalClient) publishEvaluatorVersion( return created, nil } if time.Now().After(deadline) { - return nil, fmt.Errorf( - "publishing evaluator %q kept returning version %s, which already "+ - "existed. The service was still assigning that version after %s, so "+ - "version %s now holds what was just published and any eval bound to "+ - "it is scoring against it", - name, created.Version, versionSettleTimeout, created.Version) + return nil, messages.EvaluatorVersionNotAdvancing( + name, created.Version, versionSettleTimeout) } select { case <-ctx.Done(): @@ -372,11 +369,11 @@ func (c *EvalClient) LatestEvaluatorVersion( return "", err } if list == nil || len(list.Value) == 0 { - return "", fmt.Errorf("evaluator %q has no versions", name) + return "", messages.EvaluatorHasNoVersions(name) } latest := pickLatestVersion(list.Value) if latest == "" { - return "", fmt.Errorf("evaluator %q has no usable version", name) + return "", messages.EvaluatorHasNoUsableVersion(name) } return latest, nil } @@ -514,7 +511,7 @@ func (c *EvalClient) doRequestWithHeaders( ) ([]byte, error) { u, err := url.Parse(c.endpoint) if err != nil { - return nil, fmt.Errorf("invalid endpoint URL: %w", err) + return nil, messages.InvalidEndpointURL(err) } // Callers escape the ids they interpolate, so the path is set as the raw @@ -523,7 +520,7 @@ func (c *EvalClient) doRequestWithHeaders( escapedPath := u.EscapedPath() + path decodedPath, err := url.PathUnescape(escapedPath) if err != nil { - return nil, fmt.Errorf("invalid request path %q: %w", escapedPath, err) + return nil, messages.InvalidRequestPath(escapedPath, err) } u.Path, u.RawPath = decodedPath, escapedPath @@ -538,7 +535,7 @@ func (c *EvalClient) doRequestWithHeaders( req, err := runtime.NewRequest(ctx, method, u.String()) if err != nil { - return nil, fmt.Errorf("failed to create request: %w", err) + return nil, messages.CreatingRequest(err) } for k, v := range headers { req.Raw().Header.Set(k, v) @@ -549,22 +546,22 @@ func (c *EvalClient) doRequestWithHeaders( if body != nil { payload, err := json.Marshal(body) if err != nil { - return nil, fmt.Errorf("failed to marshal request: %w", err) + return nil, messages.MarshalingRequest(err) } if err := req.SetBody(streaming.NopCloser(bytes.NewReader(payload)), "application/json"); err != nil { - return nil, fmt.Errorf("failed to set request body: %w", err) + return nil, messages.SettingRequestBody(err) } } resp, err := c.pipeline.Do(req) if err != nil { - return nil, fmt.Errorf("HTTP request failed: %w", err) + return nil, messages.RequestFailed(err) } defer resp.Body.Close() respBody, err := io.ReadAll(resp.Body) if err != nil { - return nil, fmt.Errorf("failed to read response body: %w", err) + return nil, messages.ReadingResponseBody(err) } log.Printf("[eval_api] response status: %d", resp.StatusCode) @@ -603,7 +600,7 @@ func doRequestTyped[T any]( var result T if err := json.Unmarshal(respBody, &result); err != nil { - return nil, fmt.Errorf("failed to parse response: %w", err) + return nil, messages.ParsingResponse(err) } return &result, nil diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/poller.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/poller.go index 4976248d87a..454a574c496 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/poller.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/poller.go @@ -5,11 +5,11 @@ package eval_api import ( "context" - "fmt" "log" "strings" "time" + "azureaieval/internal/messages" "azureaieval/internal/pkg/evalcore" ) @@ -73,9 +73,9 @@ type JobFailedError struct { func (e *JobFailedError) Error() string { if e.Job != nil && e.Job.Error != nil && e.Job.Error.Message != "" { - return fmt.Sprintf("job failed with status %q: %s", e.Status, e.Job.Error.Message) + return messages.JobFailedWithReason(string(e.Status), e.Job.Error.Message) } - return fmt.Sprintf("job failed with status %q", e.Status) + return messages.JobFailed(string(e.Status)) } // --------------------------------------------------------------------------- @@ -90,10 +90,7 @@ type PollerTimeoutError struct { } func (e *PollerTimeoutError) Error() string { - return fmt.Sprintf( - "operation %s did not complete within %d attempts", - e.OperationID, e.Attempts, - ) + return messages.PollerTimedOut(e.OperationID, e.Attempts) } // --------------------------------------------------------------------------- @@ -154,7 +151,7 @@ func NewPoller(operationID, apiVersion string, getJob GetJobFunc) *Poller { // On timeout it returns a plain error. func (p *Poller) Poll(ctx context.Context) (*GenerationJob, error) { if p.OperationID == "" { - return nil, fmt.Errorf("operation ID is empty") + return nil, messages.OperationIDEmpty() } for range p.Options.MaxAttempts { diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/portal_urls.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/portal_urls.go index 8b1ccd0fd5d..8ed19d149c0 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/portal_urls.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/portal_urls.go @@ -8,6 +8,8 @@ import ( "fmt" "strings" + "azureaieval/internal/messages" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/arm" "github.com/google/uuid" ) @@ -23,20 +25,17 @@ type PortalPrefix struct { func NewPortalPrefix(projectResourceID string) (*PortalPrefix, error) { resourceID, err := arm.ParseResourceID(projectResourceID) if err != nil { - return nil, fmt.Errorf("failed to parse project resource ID: %w", err) + return nil, messages.ParsingProjectResourceID(err) } encodedSub, err := encodeSubscriptionForURL(resourceID.SubscriptionID) if err != nil { - return nil, fmt.Errorf("failed to encode subscription ID: %w", err) + return nil, messages.EncodingSubscriptionID(err) } if resourceID.Parent == nil || !strings.Contains(string(resourceID.ResourceType.Type), "/") { - return nil, fmt.Errorf( - "resource ID does not represent a Foundry project (missing parent account): %s", - projectResourceID, - ) + return nil, messages.NotAFoundryProjectResourceID(projectResourceID) } prefix := fmt.Sprintf( @@ -72,7 +71,7 @@ func (p *PortalPrefix) OptimizationURL(agentName, operationID string) string { func encodeSubscriptionForURL(subscriptionID string) (string, error) { guid, err := uuid.Parse(subscriptionID) if err != nil { - return "", fmt.Errorf("invalid subscription ID format: %w", err) + return "", messages.InvalidSubscriptionID(err) } guidBytes, _ := guid.MarshalBinary() return strings.TrimRight(base64.URLEncoding.EncodeToString(guidBytes), "="), nil diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/evaluator.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/evaluator.go index 75bc7f8a251..6ca405b2973 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/evaluator.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/evaluator.go @@ -6,9 +6,10 @@ package evalcore import ( "bytes" "encoding/json" - "fmt" "strings" + "azureaieval/internal/messages" + "go.yaml.in/yaml/v3" ) @@ -74,12 +75,9 @@ func (e EvaluatorRef) CriterionName() string { // same entry also carries. type EvaluatorList []EvaluatorRef -const bareEvaluatorRemedy = "an evaluator entry is a mapping, not a bare string: " + - "write `- evaluator: %s`" - func (el *EvaluatorList) UnmarshalYAML(value *yaml.Node) error { if value.Kind != yaml.SequenceNode { - return fmt.Errorf("evaluators must be a sequence, got %v", value.Kind) + return messages.EvaluatorsMustBeSequence(value.Kind) } result := make([]EvaluatorRef, 0, len(value.Content)) @@ -88,20 +86,20 @@ func (el *EvaluatorList) UnmarshalYAML(value *yaml.Node) error { case yaml.ScalarNode: var name string if err := node.Decode(&name); err != nil { - return fmt.Errorf("decoding evaluator name: %w", err) + return messages.DecodingEvaluatorName(err) } - return fmt.Errorf(bareEvaluatorRemedy, name) + return messages.BareEvaluatorEntry(name) case yaml.MappingNode: var ref EvaluatorRef if err := node.Decode(&ref); err != nil { - return fmt.Errorf("decoding evaluator: %w", err) + return messages.DecodingEvaluator(err) } if ref.Evaluator == "" { - return fmt.Errorf("evaluator entry is missing 'evaluator'") + return messages.EvaluatorEntryMissingEvaluator() } result = append(result, ref) default: - return fmt.Errorf("evaluator entry must be a mapping, got %v", node.Kind) + return messages.EvaluatorEntryMustBeMapping(node.Kind) } } @@ -118,7 +116,7 @@ func (el *EvaluatorList) UnmarshalYAML(value *yaml.Node) error { func (el *EvaluatorList) UnmarshalJSON(data []byte) error { var entries []json.RawMessage if err := json.Unmarshal(data, &entries); err != nil { - return fmt.Errorf("evaluators must be a list: %w", err) + return messages.EvaluatorsMustBeList(err) } result := make([]EvaluatorRef, 0, len(entries)) @@ -127,17 +125,17 @@ func (el *EvaluatorList) UnmarshalJSON(data []byte) error { if len(trimmed) > 0 && trimmed[0] == '"' { var name string if err := json.Unmarshal(trimmed, &name); err != nil { - return fmt.Errorf("decoding evaluator name: %w", err) + return messages.DecodingEvaluatorName(err) } - return fmt.Errorf(bareEvaluatorRemedy, name) + return messages.BareEvaluatorEntry(name) } var ref EvaluatorRef if err := json.Unmarshal(trimmed, &ref); err != nil { - return fmt.Errorf("decoding evaluator: %w", err) + return messages.DecodingEvaluator(err) } if ref.Evaluator == "" { - return fmt.Errorf("evaluator entry is missing 'evaluator'") + return messages.EvaluatorEntryMissingEvaluator() } result = append(result, ref) } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/agent_instructions.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/agent_instructions.go index d191fd184c2..3d80db3333c 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/agent_instructions.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/agent_instructions.go @@ -4,13 +4,13 @@ package project import ( - "errors" - "fmt" "os" "path/filepath" "sort" "strings" + "azureaieval/internal/messages" + "github.com/azure/azure-dev/cli/azd/pkg/azdext" "go.yaml.in/yaml/v3" ) @@ -41,7 +41,7 @@ type agentConfigMetadata struct { // ErrAmbiguousAgentService reports that a target name matched more than one // service, so there is no single set of instructions to read. -var ErrAmbiguousAgentService = errors.New("more than one agent service matches") +var ErrAmbiguousAgentService = messages.ErrAmbiguousAgentService // AgentInstructionsFromProject reads the target agent's instructions out of the // project, returning empty when the project does not hold them. @@ -75,8 +75,7 @@ func AgentInstructionsFromProject( var meta agentConfigMetadata if err := yaml.Unmarshal(data, &meta); err != nil { - return "", "", fmt.Errorf( - "reading %s: %w", filepath.Join(configDir, agentMetadataFile), err) + return "", "", messages.ReadingPath(filepath.Join(configDir, agentMetadataFile), err) } if meta.InstructionFile == "" { return "", "", nil @@ -90,8 +89,7 @@ func AgentInstructionsFromProject( if err != nil { // The metadata named a file that is not there. That is worth saying: // something wrote the pointer and not the target. - return "", "", fmt.Errorf( - "%s names instruction_file %q, which could not be read: %w", + return "", "", messages.InstructionFileUnreadable( filepath.Join(configDir, agentMetadataFile), meta.InstructionFile, err) } @@ -131,10 +129,7 @@ func findAgentService( return services[matched[0]], nil default: sort.Strings(matched) - return nil, fmt.Errorf( - "%w %q: %s. Name one of them with --target, or pass the text with "+ - "--agent-instruction", - ErrAmbiguousAgentService, agentName, strings.Join(matched, ", ")) + return nil, messages.AmbiguousAgentService(agentName, matched) } } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/artifacts.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/artifacts.go index a5e1462e4a6..6159261fc1b 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/artifacts.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/artifacts.go @@ -4,9 +4,10 @@ package project import ( - "fmt" "path/filepath" "strings" + + "azureaieval/internal/messages" ) // Conventional artifact locations, relative to the eval directory. @@ -51,9 +52,7 @@ func ValidateGenerateSource(from string) error { case "", GenerateFromTraces, GenerateFromAgent, GenerateFromPrompt, GenerateFromFile: return nil default: - return fmt.Errorf( - "--from %q is not a source; use one of %s", - from, strings.Join(GenerateSources, ", ")) + return messages.FromNotASource(from, GenerateSources) } } @@ -61,9 +60,7 @@ func ValidateGenerateSource(from string) error { // generation job is submitted and billed. func ValidateSampleSize(n int) error { if n != 0 && (n < MinSampleSize || n > MaxSampleSize) { - return fmt.Errorf( - "sample size must be between %d and %d, got %d", - MinSampleSize, MaxSampleSize, n) + return messages.SampleSizeOutOfRange(MinSampleSize, MaxSampleSize, n) } return nil } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go index 8b8f1581194..07a3e6b305e 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go @@ -6,9 +6,9 @@ package project import ( - "fmt" "strings" + "azureaieval/internal/messages" "azureaieval/internal/pkg/evalcore" ) @@ -120,13 +120,11 @@ func (c *EvalConfig) Eval(name string) (*Eval, error) { if name == "" { switch len(c.Evals) { case 0: - return nil, fmt.Errorf("no evals are declared") + return nil, messages.NoEvalsDeclared() case 1: return &c.Evals[0], nil default: - return nil, fmt.Errorf( - "this configuration declares %d evals (%s); choose one with --eval", - len(c.Evals), strings.Join(c.EvalNames(), ", ")) + return nil, messages.SeveralEvalsDeclared(len(c.Evals), c.EvalNames()) } } @@ -135,9 +133,7 @@ func (c *EvalConfig) Eval(name string) (*Eval, error) { return &c.Evals[i], nil } } - return nil, fmt.Errorf( - "eval %q is not declared; this configuration has %s", - name, strings.Join(c.EvalNames(), ", ")) + return nil, messages.EvalNotDeclared(name, c.EvalNames()) } // HasEval reports whether the named eval is declared. Unlike Eval it never @@ -215,17 +211,17 @@ func (c *EvalConfig) Validate() error { return err } if len(c.Evals) == 0 { - return fmt.Errorf("at least one eval is required") + return messages.AtLeastOneEvalRequired() } seen := map[string]bool{} substance := map[string]string{} for i, eval := range c.Evals { if eval.Name == "" { - return fmt.Errorf("evals[%d]: 'name' is required", i) + return messages.EvalNameRequired(i) } if seen[eval.Name] { - return fmt.Errorf("evals[%d]: duplicate eval name %q", i, eval.Name) + return messages.DuplicateEvalName(i, eval.Name) } seen[eval.Name] = true @@ -242,10 +238,7 @@ func (c *EvalConfig) Validate() error { return err } if first, clash := substance[digest]; clash { - return fmt.Errorf( - "evals[%d] (%s): identical to %q apart from its name and description; "+ - "give them different evaluators, datasets or settings, or declare one", - i, eval.Name, first) + return messages.EvalsIdenticalApartFromName(i, eval.Name, first) } substance[digest] = eval.Name } @@ -256,10 +249,10 @@ func (c *EvalConfig) validateCatalogs() error { datasets := map[string]bool{} for i, d := range c.Datasets { if d.Name == "" { - return fmt.Errorf("datasets[%d]: 'name' is required", i) + return messages.DatasetNameRequired(i) } if datasets[d.Name] { - return fmt.Errorf("datasets[%d]: duplicate dataset name %q", i, d.Name) + return messages.DuplicateDatasetName(i, d.Name) } datasets[d.Name] = true } @@ -267,28 +260,22 @@ func (c *EvalConfig) validateCatalogs() error { evaluators := map[string]bool{} for i, e := range c.Evaluators { if e.Name == "" { - return fmt.Errorf("evaluators[%d]: 'name' is required", i) + return messages.EvaluatorNameRequired(i) } if evaluators[e.Name] { - return fmt.Errorf("evaluators[%d]: duplicate evaluator name %q", i, e.Name) + return messages.DuplicateEvaluatorName(i, e.Name) } evaluators[e.Name] = true if strings.HasPrefix(e.Name, evalcore.BuiltinPrefix) { - return fmt.Errorf( - "evaluators[%d] (%s): a built-in needs no catalog entry; reference it "+ - "straight from an eval", i, e.Name) + return messages.BuiltinNeedsNoCatalogEntry(i, e.Name) } // The service assigns an evaluator's version on publish, so a declared // one cannot be honoured alongside a source: the upload lands on // whatever comes next and the eval binds that, leaving the pin // describing a version nothing uses. if e.Source != "" && e.Version != "" { - return fmt.Errorf( - "evaluators[%d] (%s): `version` cannot be set with `source`, because the "+ - "service assigns the version when it publishes. Drop `version` to "+ - "publish this file, or drop `source` to reference a version already "+ - "on the project", i, e.Name) + return messages.EvaluatorVersionWithSource(i, e.Name) } } return nil @@ -296,44 +283,37 @@ func (c *EvalConfig) validateCatalogs() error { func (c *EvalConfig) validateEval(i int, eval Eval) error { if eval.Dataset != "" && eval.Source != nil { - return fmt.Errorf( - "evals[%d] (%s): `dataset` and `source` both say where rows come from; "+ - "declare one", i, eval.Name) + return messages.DatasetAndSourceBothDeclared(i, eval.Name) } if eval.Dataset != "" { if _, ok := c.DatasetDeclaration(eval.Dataset); !ok { - return fmt.Errorf( - "evals[%d] (%s): dataset %q is not in the datasets catalog", - i, eval.Name, eval.Dataset) + return messages.DatasetNotInDatasetsCatalog(i, eval.Name, eval.Dataset) } } if eval.Source != nil { switch eval.Source.Type { case SourceTypeTraces, SourceTypeResponses: case "": - return fmt.Errorf("evals[%d] (%s): source.type is required", i, eval.Name) + return messages.SourceTypeRequired(i, eval.Name) default: - return fmt.Errorf( - "evals[%d] (%s): source.type %q is not supported; use %q or %q", + return messages.SourceTypeUnsupported( i, eval.Name, eval.Source.Type, SourceTypeTraces, SourceTypeResponses) } } if len(eval.Evaluators) == 0 { - return fmt.Errorf("evals[%d] (%s): at least one evaluator is required", i, eval.Name) + return messages.AtLeastOneEvaluatorRequired(i, eval.Name) } criteria := map[string]bool{} for j, ref := range eval.Evaluators { if ref.Evaluator == "" { - return fmt.Errorf("evals[%d].evaluators[%d]: 'evaluator' is required", i, j) + return messages.EvaluatorFieldRequired(i, j) } // The criterion name is what identifies a result row, so two rows that // cannot be told apart are refused here rather than in the results. criterion := ref.CriterionName() if criteria[criterion] { - return fmt.Errorf( - "evals[%d].evaluators[%d]: duplicate criterion %q; give one a `name`", - i, j, criterion) + return messages.DuplicateCriterion(i, j, criterion) } criteria[criterion] = true @@ -341,23 +321,19 @@ func (c *EvalConfig) validateEval(i int, eval Eval) error { continue } if _, ok := c.EvaluatorDeclaration(ref.Evaluator); !ok { - return fmt.Errorf( - "evals[%d].evaluators[%d]: evaluator %q is not in the evaluators catalog", - i, j, ref.Evaluator) + return messages.EvaluatorNotInCatalog(i, j, ref.Evaluator) } } if eval.Target != nil && eval.Target.Type != "" && eval.Target.Type != TargetTypeAgent && eval.Target.Type != TargetTypeModel { - return fmt.Errorf( - "evals[%d] (%s): target.type %q is not supported; use %q or %q", + return messages.TargetTypeUnsupported( i, eval.Name, eval.Target.Type, TargetTypeAgent, TargetTypeModel) } switch eval.EvaluationLevel { case "", EvaluationLevelTurn, EvaluationLevelConversation: default: - return fmt.Errorf( - "evals[%d] (%s): evaluation_level %q is invalid; expected %q or %q", + return messages.EvaluationLevelInvalid( i, eval.Name, eval.EvaluationLevel, EvaluationLevelTurn, EvaluationLevelConversation) } return nil diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_store.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_store.go index 2b2b587c4f2..2b358795ba0 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_store.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_store.go @@ -5,11 +5,12 @@ package project import ( "errors" - "fmt" "io/fs" "os" "path/filepath" + "azureaieval/internal/messages" + "go.yaml.in/yaml/v3" ) @@ -48,12 +49,12 @@ func OpenEvalConfig(evalDir string) (*EvalConfig, error) { func LoadEvalConfig(path string) (*EvalConfig, error) { data, err := os.ReadFile(path) if err != nil { - return nil, fmt.Errorf("reading eval config %q: %w", path, err) + return nil, messages.ReadingEvalConfig(path, err) } var cfg EvalConfig if err := yaml.Unmarshal(data, &cfg); err != nil { - return nil, fmt.Errorf("parsing eval config %q: %w", path, err) + return nil, messages.ParsingEvalConfig(path, err) } return &cfg, nil } @@ -62,7 +63,7 @@ func LoadEvalConfig(path string) (*EvalConfig, error) { // directory when it does not exist yet. func SaveEvalConfig(evalDir string, cfg *EvalConfig) error { if err := os.MkdirAll(evalDir, 0o750); err != nil { - return fmt.Errorf("creating %q: %w", evalDir, err) + return messages.Creating(evalDir, err) } return SaveEvalConfigTo(EvalConfigPath(evalDir), cfg) } @@ -72,10 +73,10 @@ func SaveEvalConfig(evalDir string, cfg *EvalConfig) error { func SaveEvalConfigTo(path string, cfg *EvalConfig) error { body, err := yaml.Marshal(cfg) if err != nil { - return fmt.Errorf("serializing eval config: %w", err) + return messages.SerializingEvalConfig(err) } if err := os.WriteFile(path, body, 0o600); err != nil { - return fmt.Errorf("writing eval config %q: %w", path, err) + return messages.WritingEvalConfig(path, err) } return nil } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go index 2097621ed6d..82ee8406ade 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go @@ -8,11 +8,12 @@ import ( "crypto/sha256" "encoding/hex" "encoding/json" - "fmt" "os" "path/filepath" "strings" + "azureaieval/internal/messages" + "github.com/azure/azure-dev/cli/azd/pkg/azdext" "github.com/azure/azure-dev/cli/azd/pkg/foundry" "google.golang.org/protobuf/types/known/structpb" @@ -135,7 +136,7 @@ func (p *EvalServiceTargetProvider) Deploy( return nil, err } if err := cfg.Validate(); err != nil { - return nil, fmt.Errorf("eval config is invalid: %w", err) + return nil, messages.EvalConfigInvalid(err) } reconciler, err := p.newReconciler(ctx) @@ -153,12 +154,12 @@ func (p *EvalServiceTargetProvider) Deploy( if decl.Source == "" { continue } - report(progress, fmt.Sprintf("Reconciling dataset %s", decl.Name)) + report(progress, messages.ReconcilingDataset(decl.Name)) localPath := resolveSource(baseDir, decl.Source) datasetPaths[decl.Name] = localPath version, changed, err := reconciler.EnsureDataset(ctx, decl, localPath) if err != nil { - return nil, fmt.Errorf("dataset %q: %w", decl.Name, err) + return nil, messages.DatasetProblem(decl.Name, err) } anyChanged = anyChanged || changed report(progress, describeResult("dataset", decl.Name, version, changed)) @@ -167,11 +168,11 @@ func (p *EvalServiceTargetProvider) Deploy( // 2. Evaluators this configuration owns. Built-ins and already-registered // ones need no publish. for _, decl := range cfg.CustomEvaluators() { - report(progress, fmt.Sprintf("Reconciling evaluator %s", decl.Name)) + report(progress, messages.ReconcilingEvaluator(decl.Name)) localPath := resolveSource(baseDir, decl.Source) version, changed, err := reconciler.EnsureEvaluator(ctx, decl, localPath) if err != nil { - return nil, fmt.Errorf("evaluator %q: %w", decl.Name, err) + return nil, messages.EvaluatorProblem(decl.Name, err) } anyChanged = anyChanged || changed report(progress, describeResult("evaluator", decl.Name, version, changed)) @@ -181,12 +182,12 @@ func (p *EvalServiceTargetProvider) Deploy( // must be created and the stored id replaced. for i := range cfg.Evals { eval := cfg.Evals[i] - report(progress, fmt.Sprintf("Reconciling eval %s", eval.Name)) + report(progress, messages.ReconcilingEval(eval.Name)) id, err := reconciler.EnsureEval(ctx, eval, datasetPaths[eval.Dataset], anyChanged) if err != nil { - return nil, fmt.Errorf("eval %q: %w", eval.Name, err) + return nil, messages.EvalProblem(eval.Name, err) } - report(progress, fmt.Sprintf("Eval %s is %s", eval.Name, id)) + report(progress, messages.EvalIs(eval.Name, id)) } return &azdext.ServiceDeployResult{}, nil @@ -209,9 +210,9 @@ func (p *EvalServiceTargetProvider) projectRoot(ctx context.Context) string { // no-op deploy is visibly a no-op. func describeResult(kind, name, version string, changed bool) string { if changed { - return fmt.Sprintf("Published %s %s version %s", kind, name, version) + return messages.PublishedVersion(kind, name, version) } - return fmt.Sprintf("%s %s is unchanged at version %s", strings.ToUpper(kind[:1])+kind[1:], name, version) + return messages.UnchangedAtVersion(kind, name, version) } func report(progress azdext.ProgressReporter, message string) { @@ -232,28 +233,26 @@ func report(progress azdext.ProgressReporter, message string) { func EvalConfigFromService(svc *azdext.ServiceConfig, projectRoot string) (*EvalConfig, error) { props := serviceProps(svc) if props == nil || len(props.GetFields()) == 0 { - return nil, fmt.Errorf( - "service %q carries no eval configuration; expected evaluators, datasets, or evals", - svc.GetName()) + return nil, messages.ServiceCarriesNoConfig(svc.GetName()) } values := props.AsMap() if projectRoot != "" { resolved, err := foundry.ResolveFileRefs(values, projectRoot) if err != nil { - return nil, fmt.Errorf("resolving $ref in the eval service configuration: %w", err) + return nil, messages.ResolvingServiceRefs(err) } values = resolved } raw, err := json.Marshal(values) if err != nil { - return nil, fmt.Errorf("reading the eval service configuration: %w", err) + return nil, messages.ReadingServiceConfig(err) } var cfg EvalConfig if err := json.Unmarshal(raw, &cfg); err != nil { - return nil, fmt.Errorf("parsing the eval service configuration: %w", err) + return nil, messages.ParsingServiceConfig(err) } return &cfg, nil } @@ -312,7 +311,7 @@ func resolveSource(baseDir, source string) string { func Fingerprint(path string) (string, error) { data, err := os.ReadFile(path) if err != nil { - return "", fmt.Errorf("hashing %q: %w", path, err) + return "", messages.Hashing(path, err) } sum := sha256.Sum256(data) return hex.EncodeToString(sum[:]), nil @@ -338,7 +337,7 @@ func FingerprintGroup(group Eval) (string, error) { data, err := json.Marshal(group) if err != nil { - return "", fmt.Errorf("hashing eval %q: %w", name, err) + return "", messages.HashingEval(name, err) } sum := sha256.Sum256(data) return hex.EncodeToString(sum[:]), nil From b384bf896912c44590f03d42287003704d56cc57 Mon Sep 17 00:00:00 2001 From: mohessie Date: Sun, 9 Aug 2026 21:15:05 +0300 Subject: [PATCH 143/320] Refuse at load what the run would refuse anyway Findings from the multi-model review. Validation accepted a trace source naming no agent and a responses source listing no ids, then the run rejected both -- the same validate-then-reject split that let target.type: model deploy and fail. Both are now caught where the config is read. A misspelled source.type fell through to the dataset path, scoring the wrong rows and then blaming the eval for declaring no source. It is refused, and the run no longer records a dataset for a source-backed run that scored none. run cancel compared the service's status without lowercasing it, unlike the polling path beside it. A generation that cannot read its target agent now says so rather than asking for a flag the project could have answered. --- .../internal/cmd/generate_commands.go | 18 +++++++++++---- .../azure.ai.evaluations/internal/cmd/run.go | 12 ++++++++-- .../internal/cmd/run_datasource_test.go | 20 +++++++++++++++++ .../internal/cmd/run_ops.go | 6 +++-- .../internal/messages/messages.go | 22 +++++++++++++++++++ .../internal/project/eval_config.go | 12 +++++++++- .../internal/project/eval_config_test.go | 15 +++++++++++++ 7 files changed, 96 insertions(+), 9 deletions(-) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_commands.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_commands.go index 97cc9adcb2a..b2b8a735b63 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_commands.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_commands.go @@ -6,6 +6,7 @@ package cmd import ( "context" "fmt" + "io" "os" "path/filepath" "strings" @@ -110,7 +111,7 @@ func prepareGeneration( } if plan.Model == "" { - plan.Model = ec.agentDeployment(ctx, plan.Agent) + plan.Model = ec.agentDeployment(ctx, plan.Agent, cmd.OutOrStdout(), isJSON(cmd)) } if plan.Model == "" { ec.Close() @@ -121,14 +122,23 @@ func prepareGeneration( // agentDeployment reads the deployment the target agent answers with. // -// Best effort: a caller who named no model gets one error naming the flag, not -// two errors about an agent they never mentioned. -func (ec *evalContext) agentDeployment(ctx context.Context, agentName string) string { +// Best effort, but not silent: a misspelled --target and an agent with no +// published version both end in "pass --generation-model", which names neither. +// The warning is what tells those two apart. +func (ec *evalContext) agentDeployment( + ctx context.Context, + agentName string, + out io.Writer, + quiet bool, +) string { if agentName == "" { return "" } agent, err := ec.evalClient.GetAgent(ctx, agentName, ProjectEndpointAPIVersion) if err != nil { + if !quiet { + fmt.Fprint(out, messages.CouldNotReadAgentForModel(agentName, err)) + } return "" } return agent.Model() diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go index eaef6821f22..39fa96440bb 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go @@ -160,8 +160,9 @@ func buildRunCommand(use, short string) *cobra.Command { } // Recorded per run, not read from the configuration at list time: // comparing two runs is the point of that listing, and the dataset - // under an eval can change between them. - if group != nil && group.Dataset != "" { + // under an eval can change between them. A source-backed run scored + // no dataset, so it records none. + if group != nil && group.Dataset != "" && group.Source == nil { metadata[metaDataset] = group.Dataset if v := ec.getEnvValue(ctx, versionKey("dataset", group.Dataset)); v != "" { metadata[metaDatasetVersion] = v @@ -411,6 +412,13 @@ func (ec *evalContext) buildRunDataSource( return tracesDataSource(group) case project.SourceTypeResponses: return responsesDataSource(group) + default: + // Config validation rejects this first, but a run reached by id has no + // config to have been validated. Falling through would score the wrong + // rows and say the eval declared no source. + return nil, messages.SourceTypeUnsupported( + 0, group.Name, group.Source.Type, + project.SourceTypeTraces, project.SourceTypeResponses) } } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_datasource_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_datasource_test.go index 9dac0df876d..32e48e6ab99 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_datasource_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_datasource_test.go @@ -195,6 +195,26 @@ func TestBuildRunDataSource_NoRowsFromAnywhere(t *testing.T) { assert.Contains(t, err.Error(), "source:") } +// A misspelled source.type used to fall through to the dataset path, which +// scored the wrong rows and then blamed the eval for declaring no source. +// Config validation catches it first, but a run reached by id has no config to +// have been validated. +func TestBuildRunDataSource_UnknownSourceTypeIsRefused(t *testing.T) { + ec := &evalContext{} + group := &project.Eval{ + Name: "typo", + Dataset: "d", + Source: &project.SourceDecl{Type: "trace"}, + } + + _, err := ec.buildRunDataSource(context.Background(), group, "", 0) + + require.Error(t, err) + assert.Contains(t, err.Error(), "trace") + assert.NotContains(t, err.Error(), "references no dataset", + "a declared source must not be reported as no source at all") +} + // --max-samples has to mean the same thing wherever the rows come from. func TestBuildRunDataSource_MaxSamplesCapsLocalRows(t *testing.T) { ec := &evalContext{} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go index 4e260b19785..6b780db08c9 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go @@ -6,6 +6,7 @@ package cmd import ( "fmt" "strconv" + "strings" "azureaieval/internal/messages" "azureaieval/internal/pkg/eval_api" @@ -224,8 +225,9 @@ func newRunCancelCommand() *cobra.Command { return err } // Cancelling a run that already finished is a no-op worth naming, - // since the service reports success either way. - if terminalRunStates[target.Status] { + // since the service reports success either way. Lowercased to match + // the polling path: the service's casing is not guaranteed. + if terminalRunStates[strings.ToLower(target.Status)] { return messages.RunAlreadyFinished(target.ID, target.Status) } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go b/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go index 033903dfcd5..d3ff8d9e730 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go @@ -1376,6 +1376,22 @@ func SourceTypeUnsupported(index int, eval, got, traces, responses string) error index, eval, got, traces, responses) } +// TracesSourceNeedsAgentName reports a trace source that does not say whose +// conversations to read. +func TracesSourceNeedsAgentName(index int, eval string) error { + return fmt.Errorf( + "evals[%d] (%s): source.agent_name is required for a trace source, "+ + "or declare target.name", index, eval) +} + +// ResponsesSourceNeedsIDs reports a stored-response source with nothing to +// retrieve. +func ResponsesSourceNeedsIDs(index int, eval string) error { + return fmt.Errorf( + "evals[%d] (%s): source.response_ids is required for a responses source", + index, eval) +} + // AtLeastOneEvaluatorRequired reports an eval that scores nothing. func AtLeastOneEvaluatorRequired(index int, eval string) error { return fmt.Errorf("evals[%d] (%s): at least one evaluator is required", index, eval) @@ -1756,3 +1772,9 @@ func NotAFoundryProjectResourceID(resourceID string) error { func InvalidSubscriptionID(err error) error { return fmt.Errorf("invalid subscription ID format: %w", err) } + +// CouldNotReadAgentForModel reports a target agent whose deployment could not +// be read, leaving generation without a default model. +func CouldNotReadAgentForModel(agent string, err error) string { + return fmt.Sprintf(" warning: could not read agent %q for its deployment: %v\n", agent, err) +} \ No newline at end of file diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go index 07a3e6b305e..36602bc9824 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go @@ -292,7 +292,17 @@ func (c *EvalConfig) validateEval(i int, eval Eval) error { } if eval.Source != nil { switch eval.Source.Type { - case SourceTypeTraces, SourceTypeResponses: + case SourceTypeTraces: + // The run needs one of these to say whose traces to read. Refusing + // here rather than at run time keeps a config that cannot run from + // deploying. + if eval.Source.AgentName == "" && (eval.Target == nil || eval.Target.Name == "") { + return messages.TracesSourceNeedsAgentName(i, eval.Name) + } + case SourceTypeResponses: + if len(eval.Source.ResponseIDs) == 0 { + return messages.ResponsesSourceNeedsIDs(i, eval.Name) + } case "": return messages.SourceTypeRequired(i, eval.Name) default: diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_test.go index cb51446d56c..b83440373ea 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_test.go @@ -214,6 +214,21 @@ func TestValidate_Rejects(t *testing.T) { body string wantErr string }{ + { + // The run path refuses a trace source that does not say whose + // conversations to read. Accepting it here would deploy a config + // that cannot run. + name: "trace source naming no agent", + body: "evals:\n - name: e\n source:\n type: traces\n" + + " evaluators:\n - evaluator: builtin.relevance\n", + wantErr: "source.agent_name is required", + }, + { + name: "responses source listing no ids", + body: "evals:\n - name: e\n source:\n type: responses\n" + + " evaluators:\n - evaluator: builtin.relevance\n", + wantErr: "source.response_ids is required", + }, { name: "dataset without a name", body: "datasets:\n - source: ./d.jsonl\n" + oneEval, From d20545003cc060d683794b56f6ea86738b4a863f Mon Sep 17 00:00:00 2001 From: mohessie Date: Sun, 9 Aug 2026 21:47:09 +0300 Subject: [PATCH 144/320] Stop the two panics and the leak a bug bash would find From the multi-model review. The shared endpoint resolver read envResp.Environment.Name and envVal.Value without checking either for nil, so a project with no azd environment could panic inside the cascade instead of falling through to the next source. cmd/context.go already guarded this; the resolver beside it did not. newEvalContext abandoned the azd client on two error paths. Callers only defer Close on a context they were handed, so every failed endpoint or credential setup left a gRPC client open. --- .../extensions/azure.ai.evaluations/internal/cmd/context.go | 4 ++++ .../internal/foundry/projectctx/resolver.go | 4 ++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/context.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/context.go index 985ef5be6b2..10e7ef85dab 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/context.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/context.go @@ -60,6 +60,9 @@ func newEvalContext(ctx context.Context, endpointFlag string) (*evalContext, err resolved, err := projectctx.Resolve(ctx, projectctx.ResolveOpts{FlagValue: endpointFlag}) if err != nil { + // The caller only defers Close on a context it was handed, so every + // path that abandons this one has to close it here. + ec.Close() return nil, err } ec.endpoint = strings.TrimSuffix(resolved.Endpoint, "/") @@ -69,6 +72,7 @@ func newEvalContext(ctx context.Context, endpointFlag string) (*evalContext, err &azidentity.AzureDeveloperCLICredentialOptions{}, ) if err != nil { + ec.Close() return nil, messages.CreatingCredential(err) } ec.cred = cred diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/foundry/projectctx/resolver.go b/cli/azd/extensions/azure.ai.evaluations/internal/foundry/projectctx/resolver.go index 3c6c3129a50..25fccf2b134 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/foundry/projectctx/resolver.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/foundry/projectctx/resolver.go @@ -36,13 +36,13 @@ func readAzdHostedSources(ctx context.Context) (AzdHostedSources, error) { if envResp, err := azdClient.Environment().GetCurrent( ctx, &azdext.EmptyRequest{}, - ); err == nil { + ); err == nil && envResp.GetEnvironment() != nil { for _, key := range []string{foundryEnvKey, azureAiEnvKey} { envVal, valErr := azdClient.Environment().GetValue(ctx, &azdext.GetEnvRequest{ EnvName: envResp.Environment.Name, Key: key, }) - if valErr == nil && envVal.Value != "" { + if valErr == nil && envVal.GetValue() != "" { out.EnvValue = envVal.Value out.EnvName = envResp.Environment.Name break From a6f962614baedfb083513c93f928b54eb4187906 Mon Sep 17 00:00:00 2001 From: mohessie Date: Sun, 9 Aug 2026 22:01:31 +0300 Subject: [PATCH 145/320] Bound the wait, and say when the answer was 'log in again' Two things a bug bash would find. pollRun looped until a terminal state or a cancelled context, with no budget. A run that never reaches one held the terminal open until someone killed it. It now stops after two hours -- long enough to clear a few hundred samples at the ~40s a sample runs are scored at -- and that boundary is handled where --no-wait is: the run is still going server-side, so the caller gets the same reattach line and the same exit code rather than an error about a failure that did not happen. An interrupted wait now names the run instead of surfacing a bare 'context canceled'. A stale login fails when the credential mints a token, not as a 401, and surfaced as 'AzureDeveloperCLICredential: exit status 1' with no mention of azd. Hit live while smoke-testing. Both that and a 401/403 answer now carry the suggestion to log in again. --- .../azure.ai.evaluations/internal/cmd/run.go | 32 ++++++- .../internal/messages/messages.go | 59 ++++++++++++- .../internal/messages/messages_test.go | 84 +++++++++++++++++++ .../internal/pkg/dataset_api/operations.go | 2 +- .../internal/pkg/eval_api/operations.go | 2 +- 5 files changed, 175 insertions(+), 4 deletions(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/messages/messages_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go index 39fa96440bb..bd727ea9306 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go @@ -201,6 +201,16 @@ func buildRunCommand(use, short string) *cobra.Command { } final, err := ec.pollRun(ctx, evalID, run.ID, out, isJSON(cmd)) + if errors.Is(err, errWaitBudgetSpent) { + // The run did not fail, the wait ran out. Same contract as + // --no-wait: exit 0 and say how to pick it back up. + if isJSON(cmd) { + return emitJSON(out, startedRun(run, evalID, group)) + } + fmt.Fprint(out, messages.WaitBudgetSpent(run.ID, waitBudget)) + fmt.Fprint(out, messages.ReattachToRun(run.ID, evalID)) + return nil + } if err != nil { return err } @@ -660,6 +670,20 @@ func resolveMaxSamples(flag int, group *project.Eval) int { return 0 } +// errWaitBudgetSpent says the run outlived the wait, not that anything failed. +// +// It is handled where --no-wait is: the run is still going server-side, so the +// caller is handed the same reattach line and the same exit code. +var errWaitBudgetSpent = errors.New("wait budget spent") + +// waitBudget bounds a foreground wait. +// +// Not a policy about how long an evaluation may take -- it is a guard against +// waiting on a run that will never reach a terminal state. Runs are scored +// sequentially at roughly 40s a sample, so this clears a few hundred samples +// before it ever fires. +const waitBudget = 2 * time.Hour + // pollRun waits for the run to reach a terminal state, reporting status changes. func (ec *evalContext) pollRun( ctx context.Context, @@ -669,6 +693,7 @@ func (ec *evalContext) pollRun( ) (*eval_api.OpenAIEvalRun, error) { const interval = 5 * time.Second lastStatus := "" + deadline := time.Now().Add(waitBudget) for { run, err := ec.evalClient.GetOpenAIEvalRun(ctx, evalID, runID) @@ -684,9 +709,14 @@ func (ec *evalContext) pollRun( if terminalRunStates[strings.ToLower(run.Status)] { return run, nil } + if time.Now().After(deadline) { + return nil, errWaitBudgetSpent + } select { case <-ctx.Done(): - return nil, ctx.Err() + // Name what is still running, or the run is lost to whoever + // interrupted the wait. + return nil, messages.WaitInterrupted(runID, ctx.Err()) case <-time.After(interval): } } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go b/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go index d3ff8d9e730..d5d083bd20f 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go @@ -19,8 +19,10 @@ package messages import ( "errors" "fmt" + "net/http" "sort" "strings" + "time" "azureaieval/internal/exterrors" ) @@ -132,6 +134,21 @@ func PollingRun(runID string, err error) error { return fmt.Errorf("polling run %s: %w", runID, err) } +// WaitBudgetSpent reports a run that outlived the foreground wait. +func WaitBudgetSpent(runID string, budget time.Duration) string { + return fmt.Sprintf( + "Run %s is still going after %s, so the wait stopped, not the run.\n", + runID, budget) +} + +// WaitInterrupted reports a wait cut short, naming the run still in flight. +func WaitInterrupted(runID string, err error) error { + return fmt.Errorf( + "stopped waiting on run %s, which is still running: %w. "+ + "Pick it back up with `azd ai eval run show %s`", + runID, err, runID) +} + // RunStatusLine reports a status change seen while polling. func RunStatusLine(status string) string { return fmt.Sprintf(" status: %s\n", status) @@ -1652,10 +1669,50 @@ func SettingRequestBody(err error) error { } // RequestFailed reports a request that never reached an answer. +// +// A credential that cannot mint a token fails here rather than as a 401, and +// the SDK's own text for it names neither azd nor the way out. Matched on the +// credential type's name because that is what the SDK puts in the message; +// anything else is passed through unchanged. func RequestFailed(err error) error { + if isCredentialFailure(err) { + return exterrors.Auth( + exterrors.CodeLoginExpired, + fmt.Sprintf("could not get a token for the Foundry project: %v", err), + "run `azd auth login`, then try again") + } return fmt.Errorf("HTTP request failed: %w", err) } +// ServiceRefused turns an unauthorised answer into one that says what to do. +// Every other status is left as the service reported it. +func ServiceRefused(status int, err error) error { + if status == http.StatusUnauthorized || status == http.StatusForbidden { + return exterrors.Auth( + exterrors.CodeAuthFailed, + fmt.Sprintf("the Foundry project refused the request (HTTP %d): %v", status, err), + "run `azd auth login`, and check you have access to this project") + } + return err +} + +func isCredentialFailure(err error) bool { + if err == nil { + return false + } + text := err.Error() + for _, marker := range []string{ + "AzureDeveloperCLICredential", + "DefaultAzureCredential", + "failed to acquire a token", + } { + if strings.Contains(text, marker) { + return true + } + } + return false +} + // ReadingResponseBody reports a response that could not be read. func ReadingResponseBody(err error) error { return fmt.Errorf("failed to read response body: %w", err) @@ -1777,4 +1834,4 @@ func InvalidSubscriptionID(err error) error { // be read, leaving generation without a default model. func CouldNotReadAgentForModel(agent string, err error) string { return fmt.Sprintf(" warning: could not read agent %q for its deployment: %v\n", agent, err) -} \ No newline at end of file +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages_test.go new file mode 100644 index 00000000000..985a9e95b01 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages_test.go @@ -0,0 +1,84 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package messages + +import ( + "errors" + "net/http" + "strings" + "testing" + + "github.com/azure/azure-dev/cli/azd/pkg/azdext" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// suggestionOf reads the way out azd renders beneath the message. It is a +// structured field, not part of Error(), so asserting on the text alone would +// pass whether or not the suggestion survived. +func suggestionOf(t *testing.T, err error) string { + t.Helper() + var local *azdext.LocalError + require.ErrorAs(t, err, &local, "expected a structured local error") + return local.Suggestion +} + +// A stale login fails when the credential tries to mint a token, not as a 401, +// and the SDK's text for it mentions neither azd nor logging in. This was seen +// live: "HTTP request failed: AzureDeveloperCLICredential: exit status 1". +func TestRequestFailed_ClassifiesAStaleLogin(t *testing.T) { + err := RequestFailed(errors.New("AzureDeveloperCLICredential: exit status 1")) + + require.Error(t, err) + assert.Contains(t, suggestionOf(t, err), "azd auth login", + "a token failure has to name the way out") + assert.Contains(t, err.Error(), "exit status 1", "the cause still has to be visible") +} + +// Anything that is not a credential problem keeps the cause it came with. +func TestRequestFailed_PassesOtherFailuresThrough(t *testing.T) { + cause := errors.New("dial tcp: connection refused") + + err := RequestFailed(cause) + + require.Error(t, err) + assert.ErrorIs(t, err, cause, "the cause has to survive wrapping") + assert.NotContains(t, err.Error(), "azd auth login") +} + +func TestServiceRefused_ClassifiesUnauthorized(t *testing.T) { + for _, status := range []int{http.StatusUnauthorized, http.StatusForbidden} { + err := ServiceRefused(status, errors.New("the service said no")) + + require.Error(t, err) + assert.Contains(t, suggestionOf(t, err), "azd auth login", + "HTTP %d has to name the way out", status) + } +} + +// A 404 is not an auth problem and must not be reported as one. +func TestServiceRefused_LeavesOtherStatusesAlone(t *testing.T) { + cause := errors.New("not found") + + err := ServiceRefused(http.StatusNotFound, cause) + + assert.Same(t, cause, err, "a non-auth status is returned untouched") +} + +// The wait ending is not the run failing, so the line says which stopped. +func TestWaitBudgetSpent_SaysTheRunIsStillGoing(t *testing.T) { + line := WaitBudgetSpent("evalrun_1", 0) + + assert.Contains(t, line, "evalrun_1") + assert.Contains(t, strings.ToLower(line), "still going") +} + +// An interrupted wait has to name the run, or it is lost to whoever stopped it. +func TestWaitInterrupted_NamesTheRunAndTheWayBack(t *testing.T) { + err := WaitInterrupted("evalrun_2", errors.New("context canceled")) + + require.Error(t, err) + assert.Contains(t, err.Error(), "evalrun_2") + assert.Contains(t, err.Error(), "azd ai eval run show") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/operations.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/operations.go index 78542fb267b..a855d118c9a 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/operations.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/operations.go @@ -598,7 +598,7 @@ func (c *DatasetClient) doRequest( if !runtime.HasStatusCode(resp, http.StatusOK, http.StatusCreated, http.StatusAccepted, http.StatusNoContent) { resp.Body = io.NopCloser(bytes.NewReader(respBody)) - return nil, runtime.NewResponseError(resp) + return nil, messages.ServiceRefused(resp.StatusCode, runtime.NewResponseError(resp)) } return respBody, nil diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/operations.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/operations.go index 0b05a9a31df..603c7d7c94e 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/operations.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/operations.go @@ -573,7 +573,7 @@ func (c *EvalClient) doRequestWithHeaders( http.StatusOK, http.StatusCreated, http.StatusAccepted, http.StatusNoContent) { // Restore the body so runtime.NewResponseError can read it. resp.Body = io.NopCloser(bytes.NewReader(respBody)) - return nil, runtime.NewResponseError(resp) + return nil, messages.ServiceRefused(resp.StatusCode, runtime.NewResponseError(resp)) } return respBody, nil From c95645bea58ee41c146054827ae2ea6928851a9c Mon Sep 17 00:00:00 2001 From: mohessie Date: Sun, 9 Aug 2026 22:07:48 +0300 Subject: [PATCH 146/320] Tell people to retry before telling them to log in The credential shells out to azd auth token, and that fails intermittently against a login that is perfectly valid -- observed three times while smoke testing, with azd auth token succeeding on its own each time. Sending a logged-in user to azd auth login would be wrong more often than right, so the retry comes first. The hint is also in the message, not only the suggestion: azd printed the message alone on the surface this was reproduced on. --- .../internal/messages/messages.go | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go b/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go index d5d083bd20f..4f4e1c2c668 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go @@ -1674,12 +1674,19 @@ func SettingRequestBody(err error) error { // the SDK's own text for it names neither azd nor the way out. Matched on the // credential type's name because that is what the SDK puts in the message; // anything else is passed through unchanged. +// +// The hint is in the message as well as the suggestion because the suggestion +// is not rendered on every surface, and it offers a retry first: this call +// shells out to `azd auth token`, which has been seen to fail transiently +// against a login that was perfectly valid. func RequestFailed(err error) error { if isCredentialFailure(err) { return exterrors.Auth( exterrors.CodeLoginExpired, - fmt.Sprintf("could not get a token for the Foundry project: %v", err), - "run `azd auth login`, then try again") + fmt.Sprintf( + "could not get a token for the Foundry project: %v. "+ + "Try again; if it keeps failing, run `azd auth login`", err), + "try the command again, then `azd auth login` if it keeps failing") } return fmt.Errorf("HTTP request failed: %w", err) } @@ -1690,7 +1697,10 @@ func ServiceRefused(status int, err error) error { if status == http.StatusUnauthorized || status == http.StatusForbidden { return exterrors.Auth( exterrors.CodeAuthFailed, - fmt.Sprintf("the Foundry project refused the request (HTTP %d): %v", status, err), + fmt.Sprintf( + "the Foundry project refused the request (HTTP %d): %v. "+ + "Run `azd auth login`, and check you have access to this project", + status, err), "run `azd auth login`, and check you have access to this project") } return err From 8eb72d6b0ffed6dbdc5fb3a321d77738cf9ad263 Mon Sep 17 00:00:00 2001 From: mohessie Date: Sun, 9 Aug 2026 22:26:25 +0300 Subject: [PATCH 147/320] Name the configuration for azd The spec calls it azure.eval.yaml. eval.yaml is generic enough to collide with an unrelated tool's file in the same folder, and the prefix says whose it is, the way azure.yaml does beside it. A project that already has eval.yaml keeps working: the file is resolved, not assumed, and a write goes back over the legacy one rather than leaving a second configuration beside the one azure.yaml \ -- which would be inert, because nothing points at it. Every path that names the file resolves it too. The first cut did not, so a generate into a legacy project wrote eval.yaml and announced azure.eval.yaml. --- .../internal/cmd/catalog.go | 5 +- .../internal/cmd/catalog_test.go | 6 +- .../internal/cmd/eval_group.go | 2 +- .../internal/cmd/evalref.go | 2 +- .../internal/cmd/generate_commands.go | 2 +- .../azure.ai.evaluations/internal/cmd/init.go | 2 +- .../internal/cmd/run_output.go | 2 +- .../internal/messages/messages.go | 2 +- .../internal/project/eval_config_name_test.go | 77 +++++++++++++++++++ .../internal/project/eval_config_store.go | 33 +++++++- .../tests/hero/init_test.go | 12 +-- 11 files changed, 126 insertions(+), 19 deletions(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_name_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/catalog.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/catalog.go index 5d0dd8d50d3..224dc3d114a 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/catalog.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/catalog.go @@ -95,7 +95,10 @@ func updateCatalog( return err } if !isJSON(cmd) { - path := filepath.ToSlash(project.EvalConfigPath(evalDir)) + // Resolved, not the current name: SaveEvalConfig writes back over a + // legacy file when that is what the project has, and the line has to + // name the file it actually wrote. + path := filepath.ToSlash(project.ResolveEvalConfigPath(evalDir)) if created { fmt.Fprint(cmd.OutOrStdout(), messages.CreatedCatalogFile(path)) } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/catalog_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/catalog_test.go index 126c36a35ba..9c9a91f4b8d 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/catalog_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/catalog_test.go @@ -74,8 +74,8 @@ func TestCatalogLineWithoutAVersion(t *testing.T) { } } -// The first generate in a repository has no eval.yaml to append to, so it says -// the file was created as well as what went into it. +// The first generate in a repository has no configuration to append to, so it +// says the file was created as well as what went into it. func TestCatalogLineWhenTheFileIsCreated(t *testing.T) { dir := t.TempDir() var buf bytes.Buffer @@ -90,7 +90,7 @@ func TestCatalogLineWhenTheFileIsCreated(t *testing.T) { "creating the file still has to say what was put in it") // The entry is really on disk, not just announced. - body, err := os.ReadFile(filepath.Join(dir, "eval.yaml")) + body, err := os.ReadFile(filepath.Join(dir, project.EvalConfigBase)) require.NoError(t, err) assert.Contains(t, string(body), "golden") } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/eval_group.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/eval_group.go index 1b5563a58a8..03ffccd65c6 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/eval_group.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/eval_group.go @@ -41,7 +41,7 @@ func newEvalCreateCommand() *cobra.Command { path := fromFile if path == "" { - path = project.EvalConfigPath(evalDir) + path = project.ResolveEvalConfigPath(evalDir) } cfg, err := project.LoadEvalConfig(path) if err != nil { diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evalref.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evalref.go index 21d8024de24..ad0ff0068b4 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evalref.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evalref.go @@ -40,7 +40,7 @@ func (ec *evalContext) resolveEvalRef( ctx context.Context, evalDir, nameOrID string, ) (evalRef, error) { - configPath := project.EvalConfigPath(evalDir) + configPath := project.ResolveEvalConfigPath(evalDir) cfg, err := project.OpenEvalConfig(evalDir) if err != nil { return evalRef{}, err diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_commands.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_commands.go index b2b8a735b63..79819f51be7 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_commands.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_commands.go @@ -22,7 +22,7 @@ import ( // partial failure undefined, cannot regenerate one artifact after the other has // been hand-edited, and gives --no-wait nothing to reattach to. // -// Neither command edits azure.yaml. Both add a catalog entry to eval.yaml for +// Neither command edits azure.yaml. Both add a catalog entry to azure.eval.yaml for // what they produced, so the artifact is referenceable without a hand edit. // generateFlags are the settings both generate commands share. diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go index 27a7a7172d0..02cd0d59cc8 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go @@ -92,7 +92,7 @@ func newInitCommand() *cobra.Command { judgeModel = detectModelDeployment(azdProject) } - configPath := project.EvalConfigPath(path) + configPath := project.ResolveEvalConfigPath(path) cfg, err := project.OpenEvalConfig(path) if err != nil { return err diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output.go index b162a13a3b3..097f26a2b21 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output.go @@ -274,7 +274,7 @@ func addEvalFlag(cmd *cobra.Command, target *string) { // recorded take effect in between. func addEvalPathFlag(cmd *cobra.Command, target *string) { cmd.Flags().StringVar(target, "path", "", - "Directory holding eval.yaml. Defaults to the path `init` used, then ./evals.") + "Directory holding azure.eval.yaml. Defaults to the path `init` used, then ./evals.") } // latestOrNamedRun returns the named run, or the most recent one for the eval. diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go b/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go index 4f4e1c2c668..d3ecc30f2f9 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go @@ -513,7 +513,7 @@ func FromPromptNeedsInstruction() string { // FromAgentNeedsTarget reports --from agent with no agent to read. func FromAgentNeedsTarget() string { return "--from agent needs a target agent; pass --target, " + - "or declare one under target: in eval.yaml" + "or declare one under target: in azure.eval.yaml" } // FromFileNotASource reports --from file, which generation has no path for. diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_name_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_name_test.go new file mode 100644 index 00000000000..b57f14fc630 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_name_test.go @@ -0,0 +1,77 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package project + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const minimalConfig = "datasets:\n - name: golden\n source: ./datasets/golden.jsonl\n" + +// The file is named for azd, the way azure.yaml is. +func TestEvalConfigPath_IsTheAzdPrefixedName(t *testing.T) { + assert.Equal(t, filepath.Join("evals", "azure.eval.yaml"), EvalConfigPath("evals")) +} + +// A project written before the rename has a checked-in eval.yaml and an +// azure.yaml $ref pointing at it. Reading has to find it, or every such project +// silently looks empty. +func TestOpenEvalConfig_ReadsALegacyFile(t *testing.T) { + dir := t.TempDir() + require.NoError(t, os.WriteFile( + filepath.Join(dir, LegacyEvalConfigBase), []byte(minimalConfig), 0o600)) + + cfg, err := OpenEvalConfig(dir) + + require.NoError(t, err) + require.NotNil(t, cfg, "a legacy configuration is still a configuration") + require.Len(t, cfg.Datasets, 1) + assert.Equal(t, "golden", cfg.Datasets[0].Name) +} + +// Writing back into such a project has to update the file it already +// references. Creating azure.eval.yaml beside it would leave the one azure.yaml +// $refs untouched, so the entry would be invisible to azd up. +func TestSaveEvalConfig_WritesBackOverALegacyFile(t *testing.T) { + dir := t.TempDir() + legacy := filepath.Join(dir, LegacyEvalConfigBase) + require.NoError(t, os.WriteFile(legacy, []byte(minimalConfig), 0o600)) + + require.NoError(t, SaveEvalConfig(dir, &EvalConfig{ + Datasets: []DatasetDecl{{Name: "added", Source: "./datasets/added.jsonl"}}, + })) + + body, err := os.ReadFile(legacy) + require.NoError(t, err) + assert.Contains(t, string(body), "added") + + _, err = os.Stat(EvalConfigPath(dir)) + assert.True(t, os.IsNotExist(err), + "a second configuration beside the one azure.yaml references would be inert") +} + +// With both present the current name wins, so a project that has migrated is +// not dragged back to the old file. +func TestResolveEvalConfigPath_PrefersTheCurrentName(t *testing.T) { + dir := t.TempDir() + require.NoError(t, os.WriteFile( + filepath.Join(dir, LegacyEvalConfigBase), []byte(minimalConfig), 0o600)) + require.NoError(t, os.WriteFile( + filepath.Join(dir, EvalConfigBase), []byte(minimalConfig), 0o600)) + + assert.Equal(t, EvalConfigPath(dir), ResolveEvalConfigPath(dir)) +} + +// An empty directory resolves to the current name, which is what a first +// generate creates. +func TestResolveEvalConfigPath_EmptyDirectoryUsesTheCurrentName(t *testing.T) { + dir := t.TempDir() + + assert.Equal(t, EvalConfigPath(dir), ResolveEvalConfigPath(dir)) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_store.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_store.go index 2b358795ba0..1b7dd925193 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_store.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_store.go @@ -23,7 +23,16 @@ import ( const DefaultEvalDir = "evals" // EvalConfigBase is the single configuration file inside that directory. -const EvalConfigBase = "eval.yaml" +// +// Prefixed for azd, the way azure.yaml is: eval.yaml is generic enough to +// collide with an unrelated tool's file in the same folder, and the prefix says +// whose it is. +const EvalConfigBase = "azure.eval.yaml" + +// LegacyEvalConfigBase is what the file was called before it was named for azd. +// Read, never written: a project that already has one keeps working, and does +// not silently grow a second configuration beside it. +const LegacyEvalConfigBase = "eval.yaml" // EvalConfigPath is the configuration file inside an eval directory. It is // exported for error messages and for the azure.yaml $ref; readers should @@ -32,12 +41,26 @@ func EvalConfigPath(evalDir string) string { return filepath.Join(evalDir, EvalConfigBase) } +// ResolveEvalConfigPath is the configuration this directory actually holds: +// the current name, or the legacy one when that is the only file there. +func ResolveEvalConfigPath(evalDir string) string { + current := EvalConfigPath(evalDir) + if _, err := os.Stat(current); err == nil { + return current + } + legacy := filepath.Join(evalDir, LegacyEvalConfigBase) + if _, err := os.Stat(legacy); err == nil { + return legacy + } + return current +} + // OpenEvalConfig reads the configuration under evalDir. // // A missing file returns (nil, nil): generate runs before init, so "no // configuration yet" is an ordinary state rather than a failure. func OpenEvalConfig(evalDir string) (*EvalConfig, error) { - cfg, err := LoadEvalConfig(EvalConfigPath(evalDir)) + cfg, err := LoadEvalConfig(ResolveEvalConfigPath(evalDir)) if errors.Is(err, fs.ErrNotExist) { return nil, nil } @@ -61,11 +84,15 @@ func LoadEvalConfig(path string) (*EvalConfig, error) { // SaveEvalConfig writes cfg as the configuration under evalDir, creating the // directory when it does not exist yet. +// +// Writes back over a legacy eval.yaml when that is the file the project has, so +// a generate into an existing project updates the configuration it already +// references rather than leaving an inert second one beside it. func SaveEvalConfig(evalDir string, cfg *EvalConfig) error { if err := os.MkdirAll(evalDir, 0o750); err != nil { return messages.Creating(evalDir, err) } - return SaveEvalConfigTo(EvalConfigPath(evalDir), cfg) + return SaveEvalConfigTo(ResolveEvalConfigPath(evalDir), cfg) } // SaveEvalConfigTo writes cfg over an explicit path, for callers that already diff --git a/cli/azd/extensions/azure.ai.evaluations/tests/hero/init_test.go b/cli/azd/extensions/azure.ai.evaluations/tests/hero/init_test.go index 6c02b3b8ecd..1ab00592ce1 100644 --- a/cli/azd/extensions/azure.ai.evaluations/tests/hero/init_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/tests/hero/init_test.go @@ -201,7 +201,7 @@ func TestHeroScenario1ColdStart(t *testing.T) { (✓) Done: Judge model deployment: gpt-5.6-luna Created - evals/eval.yaml evaluation configuration + evals/azure.eval.yaml evaluation configuration azure.yaml added service 'support-agent-evals' Next: azd up @@ -210,7 +210,7 @@ Next: azd up require.Equal(t, want, normalize(out)) } -// Scenario 1's second half: the eval.yaml the terminal block promised. The spec +// Scenario 1's second half: the azure.eval.yaml the terminal block promised. The spec // prints this file, so its shape is as much a promise as the output above — // and it is the file a reader reviews before running `azd up`. func TestHeroScenario1WritesTheDocumentedConfig(t *testing.T) { @@ -221,7 +221,7 @@ func TestHeroScenario1WritesTheDocumentedConfig(t *testing.T) { "--evaluator", "builtin.task_adherence", "--judge-model", "gpt-5.6-luna") require.Zero(t, code) - body, err := os.ReadFile(filepath.Join(dir, "evals", "eval.yaml")) + body, err := os.ReadFile(filepath.Join(dir, "evals", "azure.eval.yaml")) require.NoError(t, err) text := string(body) @@ -281,7 +281,7 @@ func TestHeroInitWiresTheServiceIntoTheProject(t *testing.T) { require.Contains(t, text, "support-agent-evals:", "the service is named for the agent it evaluates") require.Contains(t, text, "host: azure.ai.eval") - require.Contains(t, text, "$ref: ./evals/eval.yaml") + require.Contains(t, text, "$ref: ./evals/azure.eval.yaml") // azd owns the edit, so everything the project already declared survives it. require.Contains(t, text, "name: support-app") @@ -356,7 +356,7 @@ func TestHeroInitExplicitEvaluatorsOptOutOfGeneration(t *testing.T) { require.NotContains(t, text, "evaluator generate", "nothing was scheduled to be generated, so nothing should be suggested") - body, err := os.ReadFile(filepath.Join(dir, "evals", "eval.yaml")) + body, err := os.ReadFile(filepath.Join(dir, "evals", "azure.eval.yaml")) require.NoError(t, err) require.Contains(t, string(body), "evaluator: builtin.task_adherence") require.NotContains(t, string(body), "support-agent-quality", @@ -380,7 +380,7 @@ func TestHeroInitSuppliedDatasetIsNotGenerated(t *testing.T) { require.Contains(t, text, "Next: azd up", "with nothing left to generate, the next step is the deploy") - body, err := os.ReadFile(filepath.Join(dir, "evals", "eval.yaml")) + body, err := os.ReadFile(filepath.Join(dir, "evals", "azure.eval.yaml")) require.NoError(t, err) require.Contains(t, string(body), "dataset: prod-golden") require.NotContains(t, string(body), "source:", From f2cea91f980440da134a80696421256ab34721c9 Mon Sep 17 00:00:00 2001 From: mohessie Date: Sun, 9 Aug 2026 23:02:22 +0300 Subject: [PATCH 148/320] One generate, one job group, selected by a flag Spec #286. The two artifacts are separate long-running service resources, but a developer starting out wants both and should not have to know that: generate submits both, and --dataset / --evaluator narrows it to one, which is also how you regenerate one after the other has been hand-edited. The jobs go together because neither is an input to the other. Their output is buffered and replayed in a fixed order rather than written as it arrives: two generations reporting progress into one terminal interleave into nonsense. The catalog is written afterwards on the calling goroutine, because both entries land in the same file. dataset job and evaluator job collapse into one job group. There the selector is required: the two are separate service collections and share an id shape, so an id alone does not say which to call. Verified live: one generate produced both artifacts and wrote both catalog entries; the job group refuses no selector and refuses both. --- .../internal/cmd/dataset.go | 2 - .../internal/cmd/evaldir_test.go | 2 +- .../internal/cmd/evaluator.go | 2 - .../internal/cmd/generate.go | 10 + .../internal/cmd/generate_commands.go | 129 -------- .../internal/cmd/generate_composite.go | 284 ++++++++++++++++++ .../internal/cmd/generate_composite_test.go | 95 ++++++ .../azure.ai.evaluations/internal/cmd/job.go | 76 +++-- .../azure.ai.evaluations/internal/cmd/root.go | 2 + .../internal/cmd/surface_test.go | 60 ++-- .../internal/messages/messages.go | 25 ++ 11 files changed, 512 insertions(+), 175 deletions(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_composite.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_composite_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset.go index 7fc586febef..26c32c790c3 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset.go @@ -29,12 +29,10 @@ func newDatasetCommand() *cobra.Command { cmd.AddCommand( newDatasetCreateCommand(), newDatasetUpdateCommand(), - newDatasetGenerateCommand(), newDatasetListCommand(), newDatasetShowCommand(), newDatasetDeleteCommand(), newDatasetVersionsCommand(), - newJobCommand(datasetJobs), ) return cmd } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaldir_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaldir_test.go index aaba43e160a..89c74e79077 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaldir_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaldir_test.go @@ -31,7 +31,7 @@ func TestEvalDirCascade(t *testing.T) { // Every command that reads the configuration has to be able to say where it is, // or a project scaffolded with --path is unreachable from that command. func TestCommandsReadingTheConfigTakePath(t *testing.T) { - for _, path := range []string{"run start", "init", "dataset generate", "evaluator generate"} { + for _, path := range []string{"run start", "init", "generate"} { cmd := find(t, path) assert.NotNilf(t, cmd.Flags().Lookup("path"), "%s reads the configuration, so it must accept --path", path) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator.go index a489020b124..8fcc1d04320 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator.go @@ -25,12 +25,10 @@ func newEvaluatorCommand() *cobra.Command { cmd.AddCommand( newEvaluatorCreateCommand(), newEvaluatorUpdateCommand(), - newEvaluatorGenerateCommand(), newEvaluatorListCommand(), newEvaluatorShowCommand(), newEvaluatorDeleteCommand(), newEvaluatorVersionsCommand(), - newJobCommand(evaluatorJobs), ) return cmd } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go index 3fa6d3953d0..0157be36760 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go @@ -52,8 +52,18 @@ type generationPlan struct { From []string // TraceDays seeds generation from that many days of recent traces. TraceDays int + // Kind is which artifact this plan produces, so one runner can submit both. + Kind generateKind } +// generateKind names the two generation resources, which share no collection. +type generateKind string + +const ( + generateKindDataset generateKind = "dataset" + generateKindEvaluator generateKind = "evaluator" +) + // traceOptions converts the plan's trace window into the generation client's // day count. Traces seed generation only; they are never a run's data source. func (p generationPlan) traceOptions() *eval_api.TraceOptions { diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_commands.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_commands.go index 79819f51be7..2c004d10f81 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_commands.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_commands.go @@ -9,7 +9,6 @@ import ( "io" "os" "path/filepath" - "strings" "azureaieval/internal/messages" "azureaieval/internal/project" @@ -182,134 +181,6 @@ func refuseExistingArtifact(path string, force bool) error { return nil } -func newDatasetGenerateCommand() *cobra.Command { - var ( - flags generateFlags - maxSamples int - from []string - ) - - cmd := &cobra.Command{ - Use: "generate ", - Short: "Generate a dataset and download it.", - Long: "Generate a dataset and download it.\n\n" + - "--from selects one or more of the sources the service accepts, and " + - "is repeatable. Generating from the agent's own definition is a " + - "preference rather than a fallback: it covers cases no user has hit " + - "yet, and it can supply reference answers, which a transcript cannot.", - Args: cobra.ExactArgs(1), - RunE: func(cmd *cobra.Command, args []string) error { - name := args[0] - - for _, src := range from { - if err := project.ValidateGenerateSource(src); err != nil { - return err - } - } - if err := project.ValidateSampleSize(maxSamples); err != nil { - return err - } - - plan, err := resolvePlan(&flags, name, project.DefaultDatasetsDir) - if err != nil { - return err - } - plan.From = from - plan.SampleSize = maxSamples - if plan.SampleSize == 0 { - plan.SampleSize = project.DefaultSampleSize - } - if err := refuseExistingArtifact( - project.ArtifactPath(plan.BaseDir, plan.OutputDir, name, ".jsonl"), - flags.force, - ); err != nil { - return err - } - - ec, plan, err := prepareGeneration(cmd, &flags, plan) - if err != nil { - return err - } - defer ec.Close() - - if len(plan.From) == 0 { - plan.From = defaultGenerationSource( - ec.getEnvValue(cmd.Context(), appInsightsEnvKey), - ) - } - - ref, err := ec.generateDataset(cmd.Context(), plan, cmd.OutOrStdout(), flags.noWait) - if err != nil { - return err - } - if err := addDatasetToCatalog(cmd, flags.path, ref); err != nil { - return err - } - return reportGenerated(cmd, ref, flags.noWait) - }, - } - - cmd.Flags().IntVar(&maxSamples, "max-samples", 0, - fmt.Sprintf("Rows to synthesize (%d-%d). Defaults to %d.", - project.MinSampleSize, project.MaxSampleSize, project.DefaultSampleSize)) - cmd.Flags().StringSliceVar(&from, "from", nil, - fmt.Sprintf("Where rows come from: %s. Repeatable, and the service accepts "+ - "more than one. Defaults to %s when the project has Application Insights "+ - "connected, otherwise %s.", - strings.Join(project.GenerateSources, ", "), - project.GenerateFromTraces, project.GenerateFromAgent)) - addGenerateFlags(cmd, &flags) - return cmd -} - -func newEvaluatorGenerateCommand() *cobra.Command { - var ( - flags generateFlags - traceDays int - ) - - cmd := &cobra.Command{ - Use: "generate ", - Short: "Generate a rubric evaluator and download it.", - Args: cobra.ExactArgs(1), - RunE: func(cmd *cobra.Command, args []string) error { - name := args[0] - - plan, err := resolvePlan(&flags, name, project.DefaultEvaluatorsDir) - if err != nil { - return err - } - plan.TraceDays = traceDays - if err := refuseExistingArtifact( - project.ArtifactPath(plan.BaseDir, plan.OutputDir, name, ".json"), - flags.force, - ); err != nil { - return err - } - - ec, plan, err := prepareGeneration(cmd, &flags, plan) - if err != nil { - return err - } - defer ec.Close() - - ref, err := ec.generateRubric(cmd.Context(), plan, cmd.OutOrStdout(), flags.noWait) - if err != nil { - return err - } - if err := addEvaluatorToCatalog(cmd, flags.path, ref); err != nil { - return err - } - return reportGenerated(cmd, ref, flags.noWait) - }, - } - - cmd.Flags().IntVar(&traceDays, "trace-days", 0, - "Days of traces to seed generation. 0 disables.") - addGenerateFlags(cmd, &flags) - return cmd -} - // reportGenerated closes out either command. // // With --no-wait nothing was downloaded and there is no ref, which is success: diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_composite.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_composite.go new file mode 100644 index 00000000000..4c0a4babd7b --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_composite.go @@ -0,0 +1,284 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "bytes" + "fmt" + "strings" + "sync" + + "azureaieval/internal/messages" + "azureaieval/internal/project" + + "github.com/spf13/cobra" +) + +// One verb with a selector, per the spec. +// +// The two artifacts are separate long-running service resources, but a +// developer starting out wants both and should not have to know that. Omitting +// both flags generates both; passing one narrows generation to it, which is +// also how you regenerate one after the other has been hand-edited. +// +// The jobs are submitted together because neither is an input to the other. +// Their output is buffered and replayed in a fixed order rather than written as +// it arrives: two generations reporting progress into the same terminal +// interleave into nonsense. The catalog is written after both have finished, +// on this goroutine, because both entries land in the same file. + +func newGenerateCommand() *cobra.Command { + var ( + flags generateFlags + maxSamples int + from []string + traceDays int + wantDataset bool + wantEvaluator bool + datasetName string + evaluatorName string + ) + + cmd := &cobra.Command{ + Use: "generate", + Short: "Generate a dataset and a rubric evaluator, and download them.", + Long: "Generate a dataset and a rubric evaluator, and download them.\n\n" + + "Both are produced unless --dataset or --evaluator narrows it to one. " + + "Neither is an input to the other, so the jobs run together and each " + + "reports its own outcome; the command fails if either did.\n\n" + + "--from selects one or more of the sources the service generates the " + + "dataset from, and is repeatable.", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + dataset, evaluator := selectedArtifacts(wantDataset, wantEvaluator) + + if dataset { + for _, src := range from { + if err := project.ValidateGenerateSource(src); err != nil { + return err + } + } + if err := project.ValidateSampleSize(maxSamples); err != nil { + return err + } + } + + target := firstNonEmpty(flags.target, declaredTarget(flags.path)) + plans, err := buildGeneratePlans(generateRequest{ + flags: &flags, + target: target, + dataset: dataset, + evaluator: evaluator, + datasetName: datasetName, + evaluatorName: evaluatorName, + maxSamples: maxSamples, + from: from, + traceDays: traceDays, + }) + if err != nil { + return err + } + + ec, resolved, err := prepareGeneration(cmd, &flags, plans[0]) + if err != nil { + return err + } + defer ec.Close() + + // prepareGeneration settles the inputs only the service can supply. + // They are the same for both artifacts, so they are read once. + for i := range plans { + plans[i].Instruction = resolved.Instruction + plans[i].Model = resolved.Model + } + if dataset && len(plans[0].From) == 0 { + plans[0].From = defaultGenerationSource( + ec.getEnvValue(cmd.Context(), appInsightsEnvKey), + ) + } + + return ec.runGenerations(cmd, plans, flags) + }, + } + + cmd.Flags().BoolVar(&wantDataset, "dataset", false, + "Generate only the dataset. Omit both flags to generate both.") + cmd.Flags().BoolVar(&wantEvaluator, "evaluator", false, + "Generate only the evaluator. Omit both flags to generate both.") + cmd.Flags().StringVar(&datasetName, "dataset-name", "", + "Name for the generated dataset. Defaults to -dataset.") + cmd.Flags().StringVar(&evaluatorName, "evaluator-name", "", + "Name for the generated evaluator. Defaults to -evaluator.") + cmd.Flags().IntVar(&maxSamples, "max-samples", 0, + fmt.Sprintf("Rows to synthesize (%d-%d). Defaults to %d. Dataset only.", + project.MinSampleSize, project.MaxSampleSize, project.DefaultSampleSize)) + cmd.Flags().StringSliceVar(&from, "from", nil, + fmt.Sprintf("Where the dataset's rows come from: %s. Repeatable, and the "+ + "service accepts more than one. Defaults to %s when the project has "+ + "Application Insights connected, otherwise %s. Dataset only.", + strings.Join(project.GenerateSources, ", "), + project.GenerateFromTraces, project.GenerateFromAgent)) + cmd.Flags().IntVar(&traceDays, "trace-days", 0, + "Days of traces to seed the evaluator's rubric. 0 disables.") + addGenerateFlags(cmd, &flags) + return cmd +} + +// selectedArtifacts reads the pair of narrowing flags. Neither set means both, +// which is the zero-to-first-eval path the composite exists for. +func selectedArtifacts(dataset, evaluator bool) (bool, bool) { + if !dataset && !evaluator { + return true, true + } + return dataset, evaluator +} + +type generateRequest struct { + flags *generateFlags + target string + dataset bool + evaluator bool + datasetName string + evaluatorName string + maxSamples int + from []string + traceDays int +} + +// buildGeneratePlans settles everything that does not need the network, for +// each artifact asked for. Ordered dataset first, which is the order their +// progress is replayed in. +func buildGeneratePlans(req generateRequest) ([]generationPlan, error) { + plans := make([]generationPlan, 0, 2) + + if req.dataset { + name, err := generatedName(req.datasetName, req.target, "dataset") + if err != nil { + return nil, err + } + plan, err := resolvePlan(req.flags, name, project.DefaultDatasetsDir) + if err != nil { + return nil, err + } + plan.Kind = generateKindDataset + plan.From = req.from + plan.SampleSize = req.maxSamples + if plan.SampleSize == 0 { + plan.SampleSize = project.DefaultSampleSize + } + if err := refuseExistingArtifact( + project.ArtifactPath(plan.BaseDir, plan.OutputDir, name, ".jsonl"), + req.flags.force, + ); err != nil { + return nil, err + } + plans = append(plans, plan) + } + + if req.evaluator { + name, err := generatedName(req.evaluatorName, req.target, "evaluator") + if err != nil { + return nil, err + } + plan, err := resolvePlan(req.flags, name, project.DefaultEvaluatorsDir) + if err != nil { + return nil, err + } + plan.Kind = generateKindEvaluator + plan.TraceDays = req.traceDays + if err := refuseExistingArtifact( + project.ArtifactPath(plan.BaseDir, plan.OutputDir, name, ".json"), + req.flags.force, + ); err != nil { + return nil, err + } + plans = append(plans, plan) + } + + return plans, nil +} + +// generatedName is the explicit name, or one derived from the target. +func generatedName(explicit, target, suffix string) (string, error) { + if explicit != "" { + return explicit, nil + } + if target == "" { + return "", messages.GeneratedNameNeedsATarget(suffix) + } + return target + "-" + suffix, nil +} + +type generationOutcome struct { + plan generationPlan + ref *project.ArtifactRef + output bytes.Buffer + err error +} + +// runGenerations submits every plan at once and settles them together. +func (ec *evalContext) runGenerations( + cmd *cobra.Command, + plans []generationPlan, + flags generateFlags, +) error { + outcomes := make([]generationOutcome, len(plans)) + var wg sync.WaitGroup + + for i := range plans { + outcomes[i].plan = plans[i] + wg.Add(1) + go func(o *generationOutcome) { + defer wg.Done() + switch o.plan.Kind { + case generateKindDataset: + o.ref, o.err = ec.generateDataset( + cmd.Context(), o.plan, &o.output, flags.noWait) + default: + o.ref, o.err = ec.generateRubric( + cmd.Context(), o.plan, &o.output, flags.noWait) + } + }(&outcomes[i]) + } + wg.Wait() + + out := cmd.OutOrStdout() + for i := range outcomes { + if _, err := out.Write(outcomes[i].output.Bytes()); err != nil { + return err + } + } + + // Catalog entries land in one file, so they are written here rather than + // from the goroutines that produced them. + var failures []error + for i := range outcomes { + o := &outcomes[i] + if o.err != nil { + failures = append(failures, messages.GenerationFailed(string(o.plan.Kind), o.err)) + continue + } + var err error + switch o.plan.Kind { + case generateKindDataset: + err = addDatasetToCatalog(cmd, flags.path, o.ref) + default: + err = addEvaluatorToCatalog(cmd, flags.path, o.ref) + } + if err != nil { + failures = append(failures, err) + } + } + + if len(failures) > 0 { + return messages.SomeGenerationsFailed(failures) + } + + for i := range outcomes { + if err := reportGenerated(cmd, outcomes[i].ref, flags.noWait); err != nil { + return err + } + } + return nil +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_composite_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_composite_test.go new file mode 100644 index 00000000000..bae850aba24 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_composite_test.go @@ -0,0 +1,95 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// Omitting both flags is the zero-to-first-eval path the composite exists for, +// so it has to mean both rather than nothing. +func TestSelectedArtifacts(t *testing.T) { + cases := []struct { + name string + dataset, evaluator bool + wantDataset, wantEvaluator bool + }{ + {"neither means both", false, false, true, true}, + {"--dataset narrows", true, false, true, false}, + {"--evaluator narrows", false, true, false, true}, + {"both means both", true, true, true, true}, + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + gotDataset, gotEvaluator := selectedArtifacts(c.dataset, c.evaluator) + + assert.Equal(t, c.wantDataset, gotDataset, "dataset") + assert.Equal(t, c.wantEvaluator, gotEvaluator, "evaluator") + }) + } +} + +// The spec's defaults. Deriving from the target is what lets `generate` take no +// positional argument at all. +func TestGeneratedName_DerivesFromTheTarget(t *testing.T) { + name, err := generatedName("", "support-agent", "dataset") + require.NoError(t, err) + assert.Equal(t, "support-agent-dataset", name) + + name, err = generatedName("", "support-agent", "evaluator") + require.NoError(t, err) + assert.Equal(t, "support-agent-evaluator", name) +} + +func TestGeneratedName_ExplicitWins(t *testing.T) { + name, err := generatedName("golden", "support-agent", "dataset") + + require.NoError(t, err) + assert.Equal(t, "golden", name) +} + +// With neither there is nothing to name the artifact after, and the refusal has +// to name both flags that would answer it. +func TestGeneratedName_NeedsSomethingToNameItAfter(t *testing.T) { + _, err := generatedName("", "", "dataset") + + require.Error(t, err) + assert.Contains(t, err.Error(), "--dataset-name") + assert.Contains(t, err.Error(), "--target") +} + +// A composite that submits two jobs has to build a plan for each. +func TestBuildGeneratePlans_BuildsBothPlans(t *testing.T) { + plans, err := buildGeneratePlans(generateRequest{ + flags: &generateFlags{path: t.TempDir(), target: "support-agent"}, + target: "support-agent", + dataset: true, + evaluator: true, + }) + + require.NoError(t, err) + require.Len(t, plans, 2) + assert.Equal(t, generateKindDataset, plans[0].Kind, + "dataset first, which is the order its progress is replayed in") + assert.Equal(t, generateKindEvaluator, plans[1].Kind) + assert.Equal(t, "support-agent-dataset", plans[0].Name) + assert.Equal(t, "support-agent-evaluator", plans[1].Name) +} + +// Narrowing builds one plan, so nothing is submitted for the other. +func TestBuildGeneratePlans_NarrowedToOne(t *testing.T) { + plans, err := buildGeneratePlans(generateRequest{ + flags: &generateFlags{path: t.TempDir(), target: "support-agent"}, + target: "support-agent", + dataset: true, + }) + + require.NoError(t, err) + require.Len(t, plans, 1) + assert.Equal(t, generateKindDataset, plans[0].Kind) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/job.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/job.go index 269879ad7ae..b584922970f 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/job.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/job.go @@ -73,32 +73,60 @@ var evaluatorJobs = jobKind{ }, } -func newJobCommand(kind jobKind) *cobra.Command { +// jobSelector binds a command to one of the two generation collections. +// +// Required here, unlike on `generate`: an id alone does not say which +// collection to call, and the two share an id shape, so guessing would mean +// trying both and reporting whichever answered. +type jobSelector struct { + dataset bool + evaluator bool +} + +func (s *jobSelector) bind(cmd *cobra.Command) { + cmd.Flags().BoolVar(&s.dataset, "dataset", false, "Act on dataset generation jobs.") + cmd.Flags().BoolVar(&s.evaluator, "evaluator", false, "Act on evaluator generation jobs.") + cmd.MarkFlagsMutuallyExclusive("dataset", "evaluator") + cmd.MarkFlagsOneRequired("dataset", "evaluator") +} + +func (s *jobSelector) kind() jobKind { + if s.evaluator { + return evaluatorJobs + } + return datasetJobs +} + +func newJobCommand() *cobra.Command { cmd := &cobra.Command{ Use: "job", - Short: fmt.Sprintf("Inspect, cancel and delete %s generation jobs.", kind.name), - Long: fmt.Sprintf("Inspect, cancel and delete %s generation jobs.\n\n", kind.name) + - fmt.Sprintf("This is the resume path for `%s generate`: a job started with ", kind.name) + - "--no-wait, or one whose client was interrupted, is reattached to here " + - "rather than restarted.", + Short: "Inspect, cancel and delete generation jobs.", + Long: "Inspect, cancel and delete generation jobs.\n\n" + + "This is the resume path for `generate`: a job started with --no-wait, " + + "or one whose client was interrupted, is reattached to here rather than " + + "restarted.\n\n" + + "Pass --dataset or --evaluator to say which generation to act on. " + + "The two are separate service collections, so it is required.", } cmd.AddCommand( - newJobListCommand(kind), - newJobShowCommand(kind), - newJobCancelCommand(kind), - newJobDeleteCommand(kind), + newJobListCommand(), + newJobShowCommand(), + newJobCancelCommand(), + newJobDeleteCommand(), ) return cmd } -func newJobListCommand(kind jobKind) *cobra.Command { +func newJobListCommand() *cobra.Command { var endpointFlg string + sel := &jobSelector{} cmd := &cobra.Command{ Use: "list", - Short: fmt.Sprintf("List the project's %s generation jobs.", kind.name), + Short: "List the project's generation jobs.", Args: cobra.NoArgs, RunE: func(cmd *cobra.Command, args []string) error { + kind := sel.kind() ctx := cmd.Context() ec, err := newEvalContext(ctx, endpointFlg) if err != nil { @@ -126,19 +154,22 @@ func newJobListCommand(kind jobKind) *cobra.Command { }, } + sel.bind(cmd) cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") return cmd } -func newJobShowCommand(kind jobKind) *cobra.Command { +func newJobShowCommand() *cobra.Command { var endpointFlg string + sel := &jobSelector{} cmd := &cobra.Command{ Use: "show ", - Short: fmt.Sprintf("Show a %s generation job.", kind.name), + Short: "Show a generation job.", Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { jobID := args[0] + kind := sel.kind() ctx := cmd.Context() ec, err := newEvalContext(ctx, endpointFlg) @@ -163,19 +194,22 @@ func newJobShowCommand(kind jobKind) *cobra.Command { }, } + sel.bind(cmd) cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") return cmd } -func newJobCancelCommand(kind jobKind) *cobra.Command { +func newJobCancelCommand() *cobra.Command { var endpointFlg string + sel := &jobSelector{} cmd := &cobra.Command{ Use: "cancel ", - Short: fmt.Sprintf("Cancel an in-flight %s generation job.", kind.name), + Short: "Cancel an in-flight generation job.", Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { jobID := args[0] + kind := sel.kind() ctx := cmd.Context() ec, err := newEvalContext(ctx, endpointFlg) @@ -198,22 +232,25 @@ func newJobCancelCommand(kind jobKind) *cobra.Command { }, } + sel.bind(cmd) cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") return cmd } -func newJobDeleteCommand(kind jobKind) *cobra.Command { +func newJobDeleteCommand() *cobra.Command { var endpointFlg string + sel := &jobSelector{} cmd := &cobra.Command{ Use: "delete ", - Short: fmt.Sprintf("Delete a %s generation job record.", kind.name), - Long: fmt.Sprintf("Delete a %s generation job record.\n\n", kind.name) + + Short: "Delete a generation job record.", + Long: "Delete a generation job record.\n\n" + "The artifact the job produced is already registered as its own version " + "and is not affected.", Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { jobID := args[0] + kind := sel.kind() ctx := cmd.Context() ec, err := newEvalContext(ctx, endpointFlg) @@ -236,6 +273,7 @@ func newJobDeleteCommand(kind jobKind) *cobra.Command { }, } + sel.bind(cmd) cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") return cmd } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/root.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/root.go index 4f6fcd8860e..8756ef72683 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/root.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/root.go @@ -49,6 +49,8 @@ func NewRootCommand() *cobra.Command { newDatasetCommand(), newRunCommand(), newEvaluatorCommand(), + newGenerateCommand(), + newJobCommand(), newEvalCreateCommand(), newEvalListCommand(), newEvalShowCommand(), diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/surface_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/surface_test.go index 2a570ca1614..d2c20f10d1a 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/surface_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/surface_test.go @@ -51,33 +51,29 @@ func TestCommandTreeMatchesTheSpec(t *testing.T) { "dataset", "dataset create", "dataset delete", - "dataset generate", "dataset list", "dataset show", "dataset update", "dataset versions", "dataset versions list", - "dataset job", - "dataset job cancel", - "dataset job delete", - "dataset job list", - "dataset job show", "create", "delete", "evaluator", "evaluator create", "evaluator delete", - "evaluator generate", "evaluator list", "evaluator show", "evaluator update", "evaluator versions", "evaluator versions list", - "evaluator job", - "evaluator job cancel", - "evaluator job delete", - "evaluator job list", - "evaluator job show", + // One generate for both artifacts, and one job group for both + // collections, selected by --dataset / --evaluator. + "generate", + "job", + "job cancel", + "job delete", + "job list", + "job show", "init", "list", "run", @@ -216,9 +212,9 @@ func TestServiceCommandsTakeProjectEndpoint(t *testing.T) { // be repeatable. Declared as a plain string it would still accept every // documented single-source invocation and silently keep only the last of a // repeated one, which is the kind of difference no example in the spec shows. -func TestDatasetGenerateFromTakesMoreThanOneSource(t *testing.T) { - flag := find(t, "dataset generate").Flags().Lookup("from") - require.NotNil(t, flag, "dataset generate must offer --from") +func TestGenerateFromTakesMoreThanOneSource(t *testing.T) { + flag := find(t, "generate").Flags().Lookup("from") + require.NotNil(t, flag, "generate must offer --from") assert.Equal(t, "stringSlice", flag.Value.Type(), "--from selects one or more sources, so it cannot be a single string") @@ -226,8 +222,8 @@ func TestDatasetGenerateFromTakesMoreThanOneSource(t *testing.T) { // `--from` names sources; the set it accepts is the set the service has a path // for, and the help has to list exactly that set. -func TestDatasetGenerateFromListsEverySource(t *testing.T) { - usage := find(t, "dataset generate").Flags().Lookup("from").Usage +func TestGenerateFromListsEverySource(t *testing.T) { + usage := find(t, "generate").Flags().Lookup("from").Usage for _, source := range project.GenerateSources { assert.Containsf(t, usage, source, @@ -235,11 +231,31 @@ func TestDatasetGenerateFromListsEverySource(t *testing.T) { } } -// `--from` is the only place a source is named, so `evaluator generate`, which -// has no such flag, must not be left half-wired to one. -func TestEvaluatorGenerateHasNoFromFlag(t *testing.T) { - assert.Nil(t, find(t, "evaluator generate").Flags().Lookup("from"), - "the spec gives --from to dataset generate only") +// One command now generates both artifacts, but --from and --max-samples shape +// the dataset only. The help has to say so, or they read as applying to the +// rubric as well. +func TestGenerateSaysWhichFlagsAreDatasetOnly(t *testing.T) { + flags := find(t, "generate").Flags() + + for _, name := range []string{"from", "max-samples"} { + flag := flags.Lookup(name) + require.NotNilf(t, flag, "generate must offer --%s", name) + assert.Containsf(t, flag.Usage, "Dataset only", + "--%s shapes the dataset only, so its help has to say so", name) + } +} + +// The selector narrows generation; omitting both is the zero-to-first-eval +// path, so neither flag may be required. +func TestGenerateSelectorIsOptional(t *testing.T) { + cmd := find(t, "generate") + + for _, name := range []string{"dataset", "evaluator"} { + flag := cmd.Flags().Lookup(name) + require.NotNilf(t, flag, "generate must offer --%s", name) + assert.Equal(t, "false", flag.DefValue, + "--%s is off by default, which is what generates both", name) + } } // The spec's run table says which commands carry which flag. Where it says diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go b/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go index d3ecc30f2f9..cdb4336f702 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go @@ -397,6 +397,31 @@ func GateBreached(reason string) string { // Generation // --------------------------------------------------------------------------- +// GeneratedNameNeedsATarget reports a generation that can name neither the +// artifact nor the agent to derive its name from. +func GeneratedNameNeedsATarget(kind string) error { +return fmt.Errorf( +"no name for the generated %s and no target to derive one from: "+ +"pass --%s-name, or --target", kind, kind) +} + +// GenerationFailed labels one half of a composite generate that did not finish. +func GenerationFailed(kind string, err error) error { +return fmt.Errorf("generating the %s: %w", kind, err) +} + +// SomeGenerationsFailed reports a composite generate where at least one job +// did not finish. The others may well have. +func SomeGenerationsFailed(failures []error) error { +if len(failures) == 1 { +return failures[0] +} +parts := make([]string, 0, len(failures)) +for _, f := range failures { +parts = append(parts, f.Error()) +} +return errors.New(strings.Join(parts, "; ")) +} // GenerationModelRequired reports a generation with no deployment to run on. // // Reached only when the target agent could not supply one either, so the flag From 47f3dcae8f9eca4df3c4332614a08f39ce1c20f7 Mon Sep 17 00:00:00 2001 From: mohessie Date: Sun, 9 Aug 2026 23:09:23 +0300 Subject: [PATCH 149/320] Refuse an empty dataset early, bound the blob calls, guard the nil lists The last three findings from the review. An empty .jsonl uploaded and registered fine, and the failure then surfaced at the run that tried to score it -- a long way from the command that caused it. It is refused where the file is read. The direct blob calls bypass the SDK pipeline, so nothing stopped a hung storage endpoint from holding the command open until someone killed it. Ten minutes, which still clears a large dataset over a slow link. A list response can be nil even when the call succeeded, and reading .Data off it panics rather than saying the project is empty. --- .../internal/cmd/dataset.go | 4 ++++ .../internal/cmd/run_output.go | 2 +- .../internal/messages/messages.go | 5 +++++ .../internal/pkg/dataset_api/models.go | 7 +++++++ .../internal/pkg/dataset_api/operations.go | 17 +++++++++++++---- 5 files changed, 30 insertions(+), 5 deletions(-) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset.go index 26c32c790c3..6c9ae110b4d 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset.go @@ -217,6 +217,10 @@ func newDatasetVersionsListCommand() *cobra.Command { } func renderDatasets(cmd *cobra.Command, list *dataset_api.DatasetList) error { + if list == nil { + fmt.Fprint(cmd.OutOrStdout(), messages.NoDatasets()) + return nil + } if isJSON(cmd) { return emitJSONList(cmd.OutOrStdout(), list.Value) } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output.go index 097f26a2b21..7d5d936d901 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output.go @@ -312,7 +312,7 @@ func (ec *evalContext) latestOrNamedRun( } return nil, messages.ListingRuns(evalID, err) } - if len(list.Data) == 0 { + if list == nil || len(list.Data) == 0 { return nil, messages.EvalHasNoRuns(evalID) } return &list.Data[0], nil diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go b/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go index cdb4336f702..e6388f10f95 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go @@ -808,6 +808,11 @@ func ReadingDatasetDirectory(err error) error { return fmt.Errorf("reading directory: %w", err) } +// DatasetFileHasNoRows reports an empty dataset file, refused before upload. +func DatasetFileHasNoRows(name string) error { +return fmt.Errorf( +"dataset file %q has no rows, so there would be nothing to evaluate", name) +} // NoJSONLInDirectory reports an upload directory holding no dataset. func NoJSONLInDirectory(dir string) error { return fmt.Errorf("no .jsonl file found in %s", dir) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/models.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/models.go index aa306c24e20..c579bca1490 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/models.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/models.go @@ -190,6 +190,10 @@ func NextVersion(current string) string { } // ReadFirstJSONLFile finds and reads the first .jsonl file in a directory. +// +// An empty file is refused here rather than uploaded: registering it succeeds, +// and the failure then surfaces at the run that tries to score it, which is a +// long way from the command that caused it. func ReadFirstJSONLFile(dir string) (string, error) { entries, err := os.ReadDir(dir) if err != nil { @@ -204,6 +208,9 @@ func ReadFirstJSONLFile(dir string) (string, error) { if err != nil { return "", messages.ReadingPath(e.Name(), err) } + if strings.TrimSpace(string(data)) == "" { + return "", messages.DatasetFileHasNoRows(e.Name()) + } return string(data), nil } } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/operations.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/operations.go index a855d118c9a..cdb2b5807f8 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/operations.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/operations.go @@ -16,6 +16,7 @@ import ( "net/url" "path" "strings" + "time" "azureaieval/internal/messages" "azureaieval/internal/version" @@ -240,6 +241,14 @@ func (c *DatasetClient) StartPendingUpload( return doRequestTyped[PendingUploadResponse](c, ctx, http.MethodPost, path, nil, json.RawMessage(`{}`), apiVersion) } +// blobHTTPClient is the client used for direct blob calls. +// +// Bounded: these bypass the SDK pipeline, so nothing else stops a hung storage +// endpoint from holding the command open until someone kills it. Generous, so +// a large dataset over a slow link still finishes. +func blobHTTPClient() *http.Client { +return &http.Client{Timeout: 10 * time.Minute} +} // UploadBlob uploads data to a container SAS URI as a block blob. func (c *DatasetClient) UploadBlob(ctx context.Context, containerSASUri, blobName string, data []byte) error { u, err := url.Parse(containerSASUri) @@ -257,7 +266,7 @@ func (c *DatasetClient) UploadBlob(ctx context.Context, containerSASUri, blobNam req.Header.Set("x-ms-blob-type", "BlockBlob") req.Header.Set("Content-Type", "application/octet-stream") - httpClient := &http.Client{} + httpClient := blobHTTPClient() resp, err := httpClient.Do(req) if err != nil { return messages.UploadingBlobFailed(err) @@ -402,7 +411,7 @@ func (c *DatasetClient) DownloadDataset(ctx context.Context, downloadURL string) // Use a plain HTTP client for blob downloads — the SAS token in the URL provides // authentication, and Azure SDK pipeline policies (bearer token, correlation ID) // should not be sent to Azure Blob Storage endpoints. - httpClient := &http.Client{} + httpClient := blobHTTPClient() resp, err := httpClient.Do(req.Raw()) if err != nil { return nil, messages.DownloadingDatasetBlob(err) @@ -444,7 +453,7 @@ func (c *DatasetClient) ListContainerBlobs(ctx context.Context, containerSASUri return nil, messages.CreatingListRequest(err) } - httpClient := &http.Client{} + httpClient := blobHTTPClient() resp, err := httpClient.Do(req) if err != nil { return nil, messages.ListingContainerBlobs(err) @@ -482,7 +491,7 @@ func (c *DatasetClient) DownloadBlob(ctx context.Context, containerSASUri, blobN return nil, messages.CreatingBlobDownloadRequest(err) } - httpClient := &http.Client{} + httpClient := blobHTTPClient() resp, err := httpClient.Do(req) if err != nil { return nil, messages.DownloadingBlob(err) From 5e8a6d75e0400ed0148cb81de7a02e41faa609f5 Mon Sep 17 00:00:00 2001 From: mohessie Date: Sun, 9 Aug 2026 23:57:15 +0300 Subject: [PATCH 150/320] Fix what the new command surface broke, and the tests that let it Findings from the Opus 5 review, all of them regressions from #286. init's last line and the --no-wait reattach line both named commands the restructure had deleted -- azd ai eval dataset generate, and azd ai eval dataset job show. Those two lines exist to be pasted, so being wrong is the whole failure. Composite generate under -o json wrote progress prose and then two bare objects, which is not a document anything can parse. One object now, keyed by artifact, and the prose is suppressed. Error aggregation flattened structured errors: azd renders a LocalError's own message and drops any wrapper, so labelling by wrapping threw away the word saying which job failed, and joining two failures lost the auth category and its suggestion entirely. The label goes inside the error now, and two failures of one category stay that category. The whole generation was also silent while it ran, because the announcements were buffered along with the progress. Only the progress needs to be. The tests that should have caught the first two could not: the suggestion check trimmed the failing word off and asserted on the prefix that was left, and init_test pinned both dead commands as expected output. Next steps are now resolved against the real command tree. The CLI suite drove the removed commands and one case passed on an unknown-flag error rather than the refusal it claimed to test. --- .../internal/cmd/generate.go | 4 +- .../internal/cmd/generate_composite.go | 29 +++++-- .../azure.ai.evaluations/internal/cmd/init.go | 15 ++-- .../internal/cmd/init_test.go | 44 +++++++++-- .../azure.ai.evaluations/internal/cmd/job.go | 35 +++++++-- .../internal/messages/messages.go | 78 +++++++++++++++---- .../internal/pkg/dataset_api/operations.go | 16 ++-- .../tests/cli/generate_test.go | 65 ++++++++-------- 8 files changed, 205 insertions(+), 81 deletions(-) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go index 0157be36760..f95c97b983f 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go @@ -216,7 +216,7 @@ func (ec *evalContext) generateRubric( return nil, messages.SubmittingRubricJob(err) } if noWait { - reportSubmitted(out, "azd ai eval evaluator", job.ID) + reportSubmitted(out, "evaluator", job.ID) return nil, nil } @@ -298,7 +298,7 @@ func (ec *evalContext) generateDataset( return nil, messages.SubmittingDataJob(err) } if noWait { - reportSubmitted(out, "azd ai eval dataset", job.ID) + reportSubmitted(out, "dataset", job.ID) return nil, nil } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_composite.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_composite.go index 4c0a4babd7b..0f0fff47037 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_composite.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_composite.go @@ -226,6 +226,16 @@ func (ec *evalContext) runGenerations( outcomes := make([]generationOutcome, len(plans)) var wg sync.WaitGroup + // The announcements go out before the goroutines start, so a long + // generation is not silent while it runs. Only the per-job progress is + // buffered, which is what would interleave. + out := cmd.OutOrStdout() + if !isJSON(cmd) { + for i := range plans { + fmt.Fprint(out, messages.GenerationStarting(string(plans[i].Kind), plans[i].Name)) + } + } + for i := range plans { outcomes[i].plan = plans[i] wg.Add(1) @@ -243,10 +253,11 @@ func (ec *evalContext) runGenerations( } wg.Wait() - out := cmd.OutOrStdout() - for i := range outcomes { - if _, err := out.Write(outcomes[i].output.Bytes()); err != nil { - return err + if !isJSON(cmd) { + for i := range outcomes { + if _, err := out.Write(outcomes[i].output.Bytes()); err != nil { + return err + } } } @@ -275,10 +286,14 @@ func (ec *evalContext) runGenerations( return messages.SomeGenerationsFailed(failures) } - for i := range outcomes { - if err := reportGenerated(cmd, outcomes[i].ref, flags.noWait); err != nil { - return err + // One document, keyed by artifact: two bare objects on stdout is not + // something a caller can parse. + if isJSON(cmd) { + produced := map[string]*project.ArtifactRef{} + for i := range outcomes { + produced[string(outcomes[i].plan.Kind)] = outcomes[i].ref } + return emitJSON(out, produced) } return nil } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go index 02cd0d59cc8..c82e2d30028 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go @@ -387,11 +387,16 @@ func (s scaffold) evaluatorNames() []string { // for an artifact they already have. func (s scaffold) nextSteps() []string { var steps []string - if s.generateDataset { - steps = append(steps, "azd ai eval dataset generate "+s.datasetName) - } - if s.generateRubric { - steps = append(steps, "azd ai eval evaluator generate "+s.rubricName) + switch { + case s.generateDataset && s.generateRubric: + // One command produces both, which is the whole point of the composite. + steps = append(steps, "azd ai eval generate") + case s.generateDataset: + steps = append(steps, + "azd ai eval generate --dataset --dataset-name "+s.datasetName) + case s.generateRubric: + steps = append(steps, + "azd ai eval generate --evaluator --evaluator-name "+s.rubricName) } if len(steps) == 0 { steps = append(steps, "azd up", "azd ai eval run start") diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_test.go index 3b505df7f4d..a9215a18b0c 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_test.go @@ -6,6 +6,7 @@ package cmd import ( "os" "path/filepath" + "strings" "testing" "azureaieval/internal/project" @@ -143,10 +144,8 @@ func TestScaffold_NextStepsOfferOnlyWhatIsScheduled(t *testing.T) { plan, _ := scaffoldFor(t, scaffoldInput{ evalName: "support-agent-smoke", target: "support-agent", judgeModel: "m", }) - require.Equal(t, []string{ - "azd ai eval dataset generate support-agent-smoke", - "azd ai eval evaluator generate support-agent-quality", - }, plan.nextSteps()) + // One command produces both, so there is one step, not two. + require.Equal(t, []string{"azd ai eval generate"}, plan.nextSteps()) }) t.Run("dataset supplied", func(t *testing.T) { @@ -154,7 +153,7 @@ func TestScaffold_NextStepsOfferOnlyWhatIsScheduled(t *testing.T) { evalName: "smoke", target: "support-agent", dataset: "prod-golden", judgeModel: "m", }) require.Equal(t, - []string{"azd ai eval evaluator generate support-agent-quality"}, + []string{"azd ai eval generate --evaluator --evaluator-name support-agent-quality"}, plan.nextSteps()) }) @@ -171,6 +170,41 @@ func TestScaffold_NextStepsOfferOnlyWhatIsScheduled(t *testing.T) { }) } +// The literals above are only as good as the surface they name. This resolves +// every step against the real command tree, so a step naming a command that has +// been renamed or removed fails here rather than in a user's terminal — which +// is how `azd ai eval dataset generate` survived being deleted. +func TestScaffold_NextStepsNameCommandsThatExist(t *testing.T) { + inputs := []scaffoldInput{ + {evalName: "smoke", target: "support-agent", judgeModel: "m"}, + {evalName: "smoke", target: "support-agent", dataset: "prod-golden", judgeModel: "m"}, + } + + for _, in := range inputs { + plan, _ := scaffoldFor(t, in) + for _, step := range plan.nextSteps() { + words := strings.Fields(strings.TrimPrefix(step, "azd ai eval ")) + if len(words) == 0 || strings.HasPrefix(step, "azd up") { + continue + } + // Stop at the first flag: what follows is arguments, not commands. + var path []string + for _, w := range words { + if strings.HasPrefix(w, "-") { + break + } + path = append(path, w) + } + + cmd, rest, err := NewRootCommand().Find(path) + require.NoErrorf(t, err, "%q names no command", step) + require.Emptyf(t, rest, "%q left %v unresolved, so it is not a command", step, rest) + require.Equalf(t, path[len(path)-1], strings.Fields(cmd.Use)[0], + "%q resolved to %q, not the command it names", step, cmd.Use) + } + } +} + // Built-ins are referenced but never declared, so the scaffold must not give // one a catalog entry to publish. func TestScaffold_BuiltinEvaluatorsGetNoCatalogEntry(t *testing.T) { diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/job.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/job.go index b584922970f..654bd4ee0bf 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/job.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/job.go @@ -90,11 +90,18 @@ func (s *jobSelector) bind(cmd *cobra.Command) { cmd.MarkFlagsOneRequired("dataset", "evaluator") } -func (s *jobSelector) kind() jobKind { - if s.evaluator { - return evaluatorJobs +// kind resolves the selector. Total rather than defaulting: cobra enforces +// that one flag is set, and if that enforcement is ever dropped a silent +// default would query the wrong collection and report "not found". +func (s *jobSelector) kind() (jobKind, error) { + switch { + case s.dataset && !s.evaluator: + return datasetJobs, nil + case s.evaluator && !s.dataset: + return evaluatorJobs, nil + default: + return jobKind{}, messages.JobKindRequired() } - return datasetJobs } func newJobCommand() *cobra.Command { @@ -126,7 +133,10 @@ func newJobListCommand() *cobra.Command { Short: "List the project's generation jobs.", Args: cobra.NoArgs, RunE: func(cmd *cobra.Command, args []string) error { - kind := sel.kind() + kind, err := sel.kind() + if err != nil { + return err + } ctx := cmd.Context() ec, err := newEvalContext(ctx, endpointFlg) if err != nil { @@ -169,7 +179,10 @@ func newJobShowCommand() *cobra.Command { Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { jobID := args[0] - kind := sel.kind() + kind, err := sel.kind() + if err != nil { + return err + } ctx := cmd.Context() ec, err := newEvalContext(ctx, endpointFlg) @@ -209,7 +222,10 @@ func newJobCancelCommand() *cobra.Command { Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { jobID := args[0] - kind := sel.kind() + kind, err := sel.kind() + if err != nil { + return err + } ctx := cmd.Context() ec, err := newEvalContext(ctx, endpointFlg) @@ -250,7 +266,10 @@ func newJobDeleteCommand() *cobra.Command { Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { jobID := args[0] - kind := sel.kind() + kind, err := sel.kind() + if err != nil { + return err + } ctx := cmd.Context() ec, err := newEvalContext(ctx, endpointFlg) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go b/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go index e6388f10f95..d1ba676406c 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go @@ -25,6 +25,8 @@ import ( "time" "azureaieval/internal/exterrors" + + "github.com/azure/azure-dev/cli/azd/pkg/azdext" ) // --------------------------------------------------------------------------- @@ -400,28 +402,64 @@ func GateBreached(reason string) string { // GeneratedNameNeedsATarget reports a generation that can name neither the // artifact nor the agent to derive its name from. func GeneratedNameNeedsATarget(kind string) error { -return fmt.Errorf( -"no name for the generated %s and no target to derive one from: "+ -"pass --%s-name, or --target", kind, kind) + return fmt.Errorf( + "no name for the generated %s and no target to derive one from: "+ + "pass --%s-name, or --target", kind, kind) } // GenerationFailed labels one half of a composite generate that did not finish. +// +// The label goes inside a structured error rather than around it: azd +// serializes a LocalError's own message and drops any wrapper, so wrapping +// would throw away the one word saying which job failed. func GenerationFailed(kind string, err error) error { -return fmt.Errorf("generating the %s: %w", kind, err) + var local *azdext.LocalError + if errors.As(err, &local) { + labelled := *local + labelled.Message = "generating the " + kind + ": " + local.Message + return &labelled + } + return fmt.Errorf("generating the %s: %w", kind, err) } // SomeGenerationsFailed reports a composite generate where at least one job // did not finish. The others may well have. +// +// Two structured failures of the same category stay structured, so an expired +// login still arrives as an auth error carrying its suggestion rather than as +// a flat string. func SomeGenerationsFailed(failures []error) error { -if len(failures) == 1 { -return failures[0] -} -parts := make([]string, 0, len(failures)) -for _, f := range failures { -parts = append(parts, f.Error()) + if len(failures) == 1 { + return failures[0] + } + + parts := make([]string, 0, len(failures)) + for _, f := range failures { + parts = append(parts, f.Error()) + } + joined := strings.Join(parts, "; ") + + var first *azdext.LocalError + if !errors.As(failures[0], &first) { + return errors.New(joined) + } + for _, f := range failures[1:] { + var other *azdext.LocalError + if !errors.As(f, &other) || other.Category != first.Category { + return errors.New(joined) + } + } + merged := *first + merged.Message = joined + return &merged } -return errors.New(strings.Join(parts, "; ")) + +// GenerationStarting announces a job before it is submitted, so a long +// generation is not silent while it runs. +func GenerationStarting(kind, name string) string { + return fmt.Sprintf(" Starting %s generation for %q...\n", kind, name) } + // GenerationModelRequired reports a generation with no deployment to run on. // // Reached only when the target agent could not supply one either, so the flag @@ -563,8 +601,12 @@ func JobSubmitted(jobID string) string { } // ReattachToJob says how to come back to a job started with --no-wait. -func ReattachToJob(group, jobID string) string { - return fmt.Sprintf("\nReattach with: %s job show %s\n", group, jobID) +// ReattachToJob says how to come back to a job started with --no-wait. +// +// The selector is part of the line because `job` requires it: the two +// collections share an id shape, so an id alone does not say which to call. +func ReattachToJob(selector, jobID string) string { + return fmt.Sprintf("\nReattach with: azd ai eval job show %s --%s\n", jobID, selector) } // WroteArtifact reports where a generated artifact landed. @@ -584,6 +626,11 @@ func NothingGenerated() string { return "Nothing was generated.\n" } +// JobKindRequired reports a job command that does not say which collection. +func JobKindRequired() error { + return errors.New("pass --dataset or --evaluator to say which generation jobs to act on") +} + // ListingJobs reports a failure to list one kind of generation job. func ListingJobs(kind string, err error) error { return fmt.Errorf("listing %s generation jobs: %w", kind, err) @@ -810,9 +857,10 @@ func ReadingDatasetDirectory(err error) error { // DatasetFileHasNoRows reports an empty dataset file, refused before upload. func DatasetFileHasNoRows(name string) error { -return fmt.Errorf( -"dataset file %q has no rows, so there would be nothing to evaluate", name) + return fmt.Errorf( + "dataset file %q has no rows, so there would be nothing to evaluate", name) } + // NoJSONLInDirectory reports an upload directory holding no dataset. func NoJSONLInDirectory(dir string) error { return fmt.Errorf("no .jsonl file found in %s", dir) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/operations.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/operations.go index cdb2b5807f8..ee7654ee445 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/operations.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/operations.go @@ -245,10 +245,10 @@ func (c *DatasetClient) StartPendingUpload( // // Bounded: these bypass the SDK pipeline, so nothing else stops a hung storage // endpoint from holding the command open until someone kills it. Generous, so -// a large dataset over a slow link still finishes. -func blobHTTPClient() *http.Client { -return &http.Client{Timeout: 10 * time.Minute} -} +// a large dataset over a slow link still finishes. One client, so connections +// are reused across upload, finalise, list and download. +var blobHTTPClient = &http.Client{Timeout: 10 * time.Minute} + // UploadBlob uploads data to a container SAS URI as a block blob. func (c *DatasetClient) UploadBlob(ctx context.Context, containerSASUri, blobName string, data []byte) error { u, err := url.Parse(containerSASUri) @@ -266,7 +266,7 @@ func (c *DatasetClient) UploadBlob(ctx context.Context, containerSASUri, blobNam req.Header.Set("x-ms-blob-type", "BlockBlob") req.Header.Set("Content-Type", "application/octet-stream") - httpClient := blobHTTPClient() + httpClient := blobHTTPClient resp, err := httpClient.Do(req) if err != nil { return messages.UploadingBlobFailed(err) @@ -411,7 +411,7 @@ func (c *DatasetClient) DownloadDataset(ctx context.Context, downloadURL string) // Use a plain HTTP client for blob downloads — the SAS token in the URL provides // authentication, and Azure SDK pipeline policies (bearer token, correlation ID) // should not be sent to Azure Blob Storage endpoints. - httpClient := blobHTTPClient() + httpClient := blobHTTPClient resp, err := httpClient.Do(req.Raw()) if err != nil { return nil, messages.DownloadingDatasetBlob(err) @@ -453,7 +453,7 @@ func (c *DatasetClient) ListContainerBlobs(ctx context.Context, containerSASUri return nil, messages.CreatingListRequest(err) } - httpClient := blobHTTPClient() + httpClient := blobHTTPClient resp, err := httpClient.Do(req) if err != nil { return nil, messages.ListingContainerBlobs(err) @@ -491,7 +491,7 @@ func (c *DatasetClient) DownloadBlob(ctx context.Context, containerSASUri, blobN return nil, messages.CreatingBlobDownloadRequest(err) } - httpClient := blobHTTPClient() + httpClient := blobHTTPClient resp, err := httpClient.Do(req) if err != nil { return nil, messages.DownloadingBlob(err) diff --git a/cli/azd/extensions/azure.ai.evaluations/tests/cli/generate_test.go b/cli/azd/extensions/azure.ai.evaluations/tests/cli/generate_test.go index 3b7c80b7cd4..299881a8f18 100644 --- a/cli/azd/extensions/azure.ai.evaluations/tests/cli/generate_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/tests/cli/generate_test.go @@ -34,29 +34,33 @@ func TestCLIGenerateRefusesBadFlagCombinations(t *testing.T) { want string }{{ name: "the two instruction sources are mutually exclusive", - args: []string{"dataset", "generate", "d", "--target", "a", + args: []string{"generate", "--dataset", "--dataset-name", "d", "--target", "a", "--agent-instruction", "inline", "--agent-instruction-file", instruction}, want: "agent-instruction-file", }, { name: "below the minimum sample size", - args: []string{"dataset", "generate", "d", "--target", "a", "--max-samples", "14"}, + args: []string{"generate", "--dataset", "--dataset-name", "d", "--target", "a", + "--max-samples", "14"}, want: "between 15 and 1000", }, { name: "above the maximum sample size", - args: []string{"dataset", "generate", "d", "--target", "a", "--max-samples", "1001"}, + args: []string{"generate", "--dataset", "--dataset-name", "d", "--target", "a", + "--max-samples", "1001"}, want: "between 15 and 1000", }, { name: "a missing instruction file names the flag", - args: []string{"dataset", "generate", "d", "--target", "a", + args: []string{"generate", "--dataset", "--dataset-name", "d", "--target", "a", "--agent-instruction-file", filepath.Join(dir, "absent.md")}, want: "--agent-instruction-file", }, { name: "generating a dataset needs a model deployment", - args: []string{"dataset", "generate", "d", "--target", "a", "--agent-instruction", "inline"}, + args: []string{"generate", "--dataset", "--dataset-name", "d", + "--agent-instruction", "inline"}, want: "--generation-model", }, { name: "generating an evaluator needs a model deployment", - args: []string{"evaluator", "generate", "e", "--target", "a", "--agent-instruction", "inline"}, + args: []string{"generate", "--evaluator", "--evaluator-name", "e", + "--agent-instruction", "inline"}, want: "--generation-model", }} for _, tc := range cases { @@ -67,16 +71,14 @@ func TestCLIGenerateRefusesBadFlagCombinations(t *testing.T) { } } -// TestCLIGenerateNamesTheArtifact pins the positional argument. Without it the -// name would come from the spec, and two runs would quietly overwrite the same -// artifact. +// TestCLIGenerateNamesTheArtifact pins where the name comes from. The composite +// takes no positional, so with neither a name nor a target there is nothing to +// call the artifact, and the refusal has to name both ways out. func TestCLIGenerateNamesTheArtifact(t *testing.T) { - for _, group := range []string{"dataset", "evaluator"} { - t.Run(group, func(t *testing.T) { - r := requireFailure(t, runIn(t, t.TempDir(), group, "generate")) - require.Contains(t, r.Combined(), "accepts 1 arg") - }) - } + r := requireFailure(t, runIn(t, t.TempDir(), "generate", "--dataset")) + + require.Contains(t, r.Combined(), "--dataset-name") + require.Contains(t, r.Combined(), "--target") } // TestCLIGenerateNoPromptNamesWhatIsMissing is the CI case: with nothing to @@ -86,24 +88,25 @@ func TestCLIGenerateNamesTheArtifact(t *testing.T) { // but the generation model has no other source, so it is the one input a bare // directory cannot supply. func TestCLIGenerateNoPromptNamesWhatIsMissing(t *testing.T) { - r := requireFailure(t, runIn(t, t.TempDir(), "dataset", "generate", "d", "--no-prompt")) + r := requireFailure(t, runIn(t, t.TempDir(), + "generate", "--dataset", "--dataset-name", "d", "--target", "a", "--no-prompt")) require.Contains(t, r.Combined(), "--generation-model") } -// TestCLIGenerateFlagsAreScopedToTheirArtifact asserts the two commands do not -// share settings that only one of them can honour. A sample count means nothing -// to a rubric, and a trace window means nothing to a synthetic dataset; either -// would be accepted and dropped. -func TestCLIGenerateFlagsAreScopedToTheirArtifact(t *testing.T) { - dir := t.TempDir() - - r := requireFailure(t, runIn(t, dir, "evaluator", "generate", "e", - "--target", "a", "--max-samples", "20")) - require.Contains(t, r.Combined(), "max-samples", - "--max-samples belongs to dataset generate") +// TestCLIGenerateDatasetFlagsDoNotApplyToTheEvaluator asserts the scoping the +// help promises is real. +// +// One command now carries both artifacts' settings, so cobra can no longer +// refuse --max-samples on a rubric. What must still hold is that it is not +// *validated* against a generation that does not use it: narrowing to the +// evaluator has to get past the sample-size check, and fail on the model it +// genuinely lacks instead. +func TestCLIGenerateDatasetFlagsDoNotApplyToTheEvaluator(t *testing.T) { + r := requireFailure(t, runIn(t, t.TempDir(), "generate", "--evaluator", + "--evaluator-name", "e", "--target", "a", "--max-samples", "20")) - r = requireFailure(t, runIn(t, dir, "dataset", "generate", "d", - "--target", "a", "--trace-days", "7")) - require.Contains(t, r.Combined(), "trace-days", - "--trace-days belongs to evaluator generate") + require.NotContains(t, r.Combined(), "between 15 and 1000", + "--max-samples shapes the dataset, so it must not be validated for a rubric") + require.Contains(t, r.Combined(), "--generation-model", + "it should get as far as the input it actually lacks") } From eec8fe806b225feccde45c4010939ca0d47354af Mon Sep 17 00:00:00 2001 From: mohessie Date: Mon, 10 Aug 2026 02:28:56 +0300 Subject: [PATCH 151/320] Fix what the Sol review found in the new surface Six findings, five of them in code written today. Dataset generation submits and polls on its own API version, but the four dataset job delegates asked the project endpoint version, so job list, show, cancel and delete addressed the right collection the wrong way. Evaluator generation really is on the project version, which is why only these four moved. --no-wait with -o json returned nulls for both artifacts: the job ids only ever reached the buffered human output, which JSON mode suppresses. That is the one mode where the id is the whole point, so it is carried out of the generator and into the document. A directory holding both azure.eval.yaml and eval.yaml is now refused instead of silently preferring one. azure.yaml references a single file by name, so picking a winner meant editing one configuration while azd up deployed the other, with nothing to say so. A target with no name was scored as if nothing were invoked, which is a different evaluation from the one written down. Refused at load. A failed write to stdout abandoned the catalog entries for artifacts the service had already produced and billed for. The error is carried instead. Aggregated generation failures kept only the joined text; the causes are now reachable through errors.Is and errors.As. --- .../internal/cmd/generate.go | 12 +++ .../internal/cmd/generate_composite.go | 28 ++++-- .../azure.ai.evaluations/internal/cmd/job.go | 11 ++- .../internal/messages/messages.go | 33 ++++++- .../internal/project/eval_config.go | 15 ++- .../project/eval_config_ambiguity_test.go | 93 +++++++++++++++++++ .../internal/project/eval_config_store.go | 23 +++++ 7 files changed, 198 insertions(+), 17 deletions(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_ambiguity_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go index f95c97b983f..ab91da7a10f 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go @@ -200,6 +200,8 @@ func (ec *evalContext) generateRubric( plan generationPlan, out io.Writer, noWait bool, + // jobID receives the submitted job's id, for the same reason as above. + jobID *string, ) (*project.ArtifactRef, error) { fmt.Fprint(out, messages.GeneratingRubric(plan.Name)) @@ -215,6 +217,9 @@ func (ec *evalContext) generateRubric( if err != nil { return nil, messages.SubmittingRubricJob(err) } + if jobID != nil { + *jobID = job.ID + } if noWait { reportSubmitted(out, "evaluator", job.ID) return nil, nil @@ -282,6 +287,10 @@ func (ec *evalContext) generateDataset( plan generationPlan, out io.Writer, noWait bool, + // jobID receives the submitted job's id. Under --no-wait nothing is + // downloaded and there is no artifact to return, so this is the only thing + // the caller can report or reattach to. + jobID *string, ) (*project.ArtifactRef, error) { fmt.Fprint(out, messages.GeneratingDataset(plan.Name, plan.SampleSize)) @@ -297,6 +306,9 @@ func (ec *evalContext) generateDataset( if err != nil { return nil, messages.SubmittingDataJob(err) } + if jobID != nil { + *jobID = job.ID + } if noWait { reportSubmitted(out, "dataset", job.ID) return nil, nil diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_composite.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_composite.go index 0f0fff47037..8b42d488d89 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_composite.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_composite.go @@ -213,6 +213,7 @@ func generatedName(explicit, target, suffix string) (string, error) { type generationOutcome struct { plan generationPlan ref *project.ArtifactRef + jobID string output bytes.Buffer err error } @@ -244,26 +245,29 @@ func (ec *evalContext) runGenerations( switch o.plan.Kind { case generateKindDataset: o.ref, o.err = ec.generateDataset( - cmd.Context(), o.plan, &o.output, flags.noWait) + cmd.Context(), o.plan, &o.output, flags.noWait, &o.jobID) default: o.ref, o.err = ec.generateRubric( - cmd.Context(), o.plan, &o.output, flags.noWait) + cmd.Context(), o.plan, &o.output, flags.noWait, &o.jobID) } }(&outcomes[i]) } wg.Wait() + // A failed write must not cost the caller the catalog entries for work the + // service already billed them for, so it is carried rather than returned. + var failures []error if !isJSON(cmd) { for i := range outcomes { if _, err := out.Write(outcomes[i].output.Bytes()); err != nil { - return err + failures = append(failures, err) + break } } } // Catalog entries land in one file, so they are written here rather than // from the goroutines that produced them. - var failures []error for i := range outcomes { o := &outcomes[i] if o.err != nil { @@ -287,11 +291,21 @@ func (ec *evalContext) runGenerations( } // One document, keyed by artifact: two bare objects on stdout is not - // something a caller can parse. + // something a caller can parse. Under --no-wait there is no artifact yet, + // so the job id is what the caller gets and what they reattach with. if isJSON(cmd) { - produced := map[string]*project.ArtifactRef{} + produced := map[string]any{} for i := range outcomes { - produced[string(outcomes[i].plan.Kind)] = outcomes[i].ref + o := &outcomes[i] + if o.ref != nil { + produced[string(o.plan.Kind)] = o.ref + continue + } + if o.jobID != "" { + produced[string(o.plan.Kind)] = map[string]string{"job_id": o.jobID} + continue + } + produced[string(o.plan.Kind)] = nil } return emitJSON(out, produced) } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/job.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/job.go index 654bd4ee0bf..0a71da7dafd 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/job.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/job.go @@ -33,23 +33,26 @@ type jobKind struct { remove func(context.Context, *evalContext, string) error } +// Data generation is the one collection on its own API version, so the job +// commands have to ask for it the same way generate does. Evaluator generation +// is on the project endpoint version, which is why only these four differ. var datasetJobs = jobKind{ name: jobKindDataset, list: func(ctx context.Context, ec *evalContext) ([]eval_api.GenerationJob, error) { - out, err := ec.evalClient.ListDataGenerationJobs(ctx, ProjectEndpointAPIVersion) + out, err := ec.evalClient.ListDataGenerationJobs(ctx, DataGenerationAPIVersion) if err != nil { return nil, err } return out.Data, nil }, get: func(ctx context.Context, ec *evalContext, id string) (*eval_api.GenerationJob, error) { - return ec.evalClient.GetDataGenerationJob(ctx, id, ProjectEndpointAPIVersion) + return ec.evalClient.GetDataGenerationJob(ctx, id, DataGenerationAPIVersion) }, cancel: func(ctx context.Context, ec *evalContext, id string) (*eval_api.GenerationJob, error) { - return ec.evalClient.CancelDataGenerationJob(ctx, id, ProjectEndpointAPIVersion) + return ec.evalClient.CancelDataGenerationJob(ctx, id, DataGenerationAPIVersion) }, remove: func(ctx context.Context, ec *evalContext, id string) error { - return ec.evalClient.DeleteDataGenerationJob(ctx, id, ProjectEndpointAPIVersion) + return ec.evalClient.DeleteDataGenerationJob(ctx, id, DataGenerationAPIVersion) }, } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go b/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go index d1ba676406c..c8a9398efef 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go @@ -20,6 +20,7 @@ import ( "errors" "fmt" "net/http" + "path/filepath" "sort" "strings" "time" @@ -428,6 +429,19 @@ func GenerationFailed(kind string, err error) error { // Two structured failures of the same category stay structured, so an expired // login still arrives as an auth error carrying its suggestion rather than as // a flat string. +// multiError presents several failures as one line while keeping every cause +// reachable through errors.Is and errors.As. +// +// errors.Join would keep the causes but renders them one per line, and this is +// a single error the CLI prints after "ERROR: ". +type multiError struct { + msg string + causes []error +} + +func (m *multiError) Error() string { return m.msg } +func (m *multiError) Unwrap() []error { return m.causes } + func SomeGenerationsFailed(failures []error) error { if len(failures) == 1 { return failures[0] @@ -441,12 +455,12 @@ func SomeGenerationsFailed(failures []error) error { var first *azdext.LocalError if !errors.As(failures[0], &first) { - return errors.New(joined) + return &multiError{msg: joined, causes: failures} } for _, f := range failures[1:] { var other *azdext.LocalError if !errors.As(f, &other) || other.Category != first.Category { - return errors.New(joined) + return &multiError{msg: joined, causes: failures} } } merged := *first @@ -1525,6 +1539,21 @@ func EvaluationLevelInvalid(index int, eval, got, turn, conversation string) err index, eval, got, turn, conversation) } +// TargetNameRequired reports a declared target that names nothing to invoke. +func TargetNameRequired(index int, eval string) error { + return fmt.Errorf( + "evals[%d] (%s): target.name is required; remove the target: to score the "+ + "dataset as it stands", index, eval) +} + +// AmbiguousEvalConfig reports a directory holding both configuration names. +func AmbiguousEvalConfig(current, legacy string) error { + return fmt.Errorf( + "%s and %s are both present, and azure.yaml can reference only one of them. "+ + "Keep %s and delete the other, or point the service's $ref at the one you want", + filepath.ToSlash(current), filepath.ToSlash(legacy), filepath.ToSlash(current)) +} + // ReadingEvalConfig reports a configuration file that would not read. func ReadingEvalConfig(path string, err error) error { return fmt.Errorf("reading eval config %q: %w", path, err) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go index 36602bc9824..f3dc8e313c2 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go @@ -335,10 +335,17 @@ func (c *EvalConfig) validateEval(i int, eval Eval) error { } } - if eval.Target != nil && eval.Target.Type != "" && - eval.Target.Type != TargetTypeAgent && eval.Target.Type != TargetTypeModel { - return messages.TargetTypeUnsupported( - i, eval.Name, eval.Target.Type, TargetTypeAgent, TargetTypeModel) + if eval.Target != nil { + if eval.Target.Type != "" && + eval.Target.Type != TargetTypeAgent && eval.Target.Type != TargetTypeModel { + return messages.TargetTypeUnsupported( + i, eval.Name, eval.Target.Type, TargetTypeAgent, TargetTypeModel) + } + // A target with no name is scored as though nothing were invoked, which + // is a different evaluation from the one that was written down. + if eval.Target.Name == "" { + return messages.TargetNameRequired(i, eval.Name) + } } switch eval.EvaluationLevel { case "", EvaluationLevelTurn, EvaluationLevelConversation: diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_ambiguity_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_ambiguity_test.go new file mode 100644 index 00000000000..cd41c8d87a8 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_ambiguity_test.go @@ -0,0 +1,93 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package project + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func writeFile(t *testing.T, dir, name, body string) { + t.Helper() + require.NoError(t, os.WriteFile(filepath.Join(dir, name), []byte(body), 0o600)) +} + +const oneEvalConfig = "datasets:\n - name: d\n source: ./d.jsonl\n" + +// azure.yaml references one configuration by name. With both files present the +// CLI would edit whichever it preferred while azd up deployed whichever the +// $ref named, and nothing would say so. +func TestOpenEvalConfig_RefusesBothNames(t *testing.T) { + dir := t.TempDir() + writeFile(t, dir, EvalConfigBase, oneEvalConfig) + writeFile(t, dir, LegacyEvalConfigBase, oneEvalConfig) + + _, err := OpenEvalConfig(dir) + + require.Error(t, err) + assert.Contains(t, err.Error(), EvalConfigBase) + assert.Contains(t, err.Error(), LegacyEvalConfigBase) +} + +// The same refusal has to apply on the way out, or generate would append a +// catalog entry to one file while the deployment read the other. +func TestSaveEvalConfig_RefusesBothNames(t *testing.T) { + dir := t.TempDir() + writeFile(t, dir, EvalConfigBase, oneEvalConfig) + writeFile(t, dir, LegacyEvalConfigBase, oneEvalConfig) + + err := SaveEvalConfig(dir, &EvalConfig{}) + + require.Error(t, err) + assert.Contains(t, err.Error(), "only one") +} + +// A project that predates the rename keeps working, and must not silently grow +// a second configuration beside the one it already has. +func TestSaveEvalConfig_WritesBackToTheLegacyFile(t *testing.T) { + dir := t.TempDir() + writeFile(t, dir, LegacyEvalConfigBase, oneEvalConfig) + + require.NoError(t, SaveEvalConfig(dir, &EvalConfig{ + Datasets: []DatasetDecl{{Name: "d", Source: "./d.jsonl"}}, + })) + + assert.FileExists(t, filepath.Join(dir, LegacyEvalConfigBase)) + assert.NoFileExists(t, filepath.Join(dir, EvalConfigBase), + "a legacy project must not grow a second configuration") +} + +// A fresh directory gets the current name. +func TestSaveEvalConfig_WritesTheCurrentNameWhenThereIsNoFile(t *testing.T) { + dir := t.TempDir() + + require.NoError(t, SaveEvalConfig(dir, &EvalConfig{})) + + assert.FileExists(t, filepath.Join(dir, EvalConfigBase)) + assert.NoFileExists(t, filepath.Join(dir, LegacyEvalConfigBase)) +} + +// A target with no name was scored as though nothing were invoked, which is a +// different evaluation from the one that was written down. +func TestValidate_RefusesATargetWithNoName(t *testing.T) { + dir := t.TempDir() + writeFile(t, dir, EvalConfigBase, + "datasets:\n - name: d\n source: ./d.jsonl\n"+ + "evals:\n - name: e\n dataset: d\n"+ + " target:\n type: agent\n"+ + " evaluators:\n - evaluator: builtin.relevance\n") + + cfg, err := OpenEvalConfig(dir) + require.NoError(t, err) + require.NotNil(t, cfg) + + err = cfg.Validate() + + require.Error(t, err) + assert.Contains(t, err.Error(), "target.name is required") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_store.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_store.go index 1b7dd925193..79611caabcd 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_store.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_store.go @@ -55,11 +55,31 @@ func ResolveEvalConfigPath(evalDir string) string { return current } +// checkOneConfig refuses a directory holding both names. +// +// Preferring one silently is the dangerous answer: `azure.yaml` `$ref`s a +// single file by name, so the CLI would edit one configuration while `azd up` +// deployed the other, and nothing would say so. +func checkOneConfig(evalDir string) error { + current := EvalConfigPath(evalDir) + legacy := filepath.Join(evalDir, LegacyEvalConfigBase) + if _, err := os.Stat(current); err != nil { + return nil + } + if _, err := os.Stat(legacy); err != nil { + return nil + } + return messages.AmbiguousEvalConfig(current, legacy) +} + // OpenEvalConfig reads the configuration under evalDir. // // A missing file returns (nil, nil): generate runs before init, so "no // configuration yet" is an ordinary state rather than a failure. func OpenEvalConfig(evalDir string) (*EvalConfig, error) { + if err := checkOneConfig(evalDir); err != nil { + return nil, err + } cfg, err := LoadEvalConfig(ResolveEvalConfigPath(evalDir)) if errors.Is(err, fs.ErrNotExist) { return nil, nil @@ -89,6 +109,9 @@ func LoadEvalConfig(path string) (*EvalConfig, error) { // a generate into an existing project updates the configuration it already // references rather than leaving an inert second one beside it. func SaveEvalConfig(evalDir string, cfg *EvalConfig) error { + if err := checkOneConfig(evalDir); err != nil { + return err + } if err := os.MkdirAll(evalDir, 0o750); err != nil { return messages.Creating(evalDir, err) } From 0680b5327dfc17fa32526e40075501e553ebce77 Mon Sep 17 00:00:00 2001 From: mohessie Date: Mon, 10 Aug 2026 03:10:44 +0300 Subject: [PATCH 152/320] Detect the agent instead of demanding it The spec's first hero command is `azd ai eval init --source traces`, with no target: --target defaults to the project's only agent. init required the flag unconditionally, so the one command a developer runs first did not run. My earlier live check passed --target explicitly, which is exactly why it was not caught. The target is now resolved from the project's agent services: the only one when there is one, a prompt when there are several, and under --no-prompt an error naming the flag and listing the candidates. The extension had never read --no-prompt at all, so it also gains the helper that tells interactive from automated. Verified live: `azd ai eval init --source traces` with no other flags now reproduces the spec transcript. --- .../azure.ai.evaluations/internal/cmd/init.go | 18 ++-- .../internal/cmd/init_target.go | 80 ++++++++++++++ .../internal/cmd/init_target_test.go | 101 ++++++++++++++++++ .../internal/cmd/output.go | 12 +++ .../internal/messages/messages.go | 24 +++++ 5 files changed, 229 insertions(+), 6 deletions(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_target.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_target_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go index c82e2d30028..058bd3dcddc 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go @@ -53,9 +53,6 @@ func newInitCommand() *cobra.Command { RunE: func(cmd *cobra.Command, args []string) error { out := cmd.OutOrStdout() - if target == "" { - return requireFlag("target") - } switch source { case "", initSourceDataset, initSourceTraces: default: @@ -77,9 +74,6 @@ func newInitCommand() *cobra.Command { if path == "" { path = project.DefaultEvalDir } - if evalName == "" { - evalName = defaultEvalName(target, source) - } // Asked before anything is written: the project is the one thing // init cannot supply for itself, and failing after creating @@ -88,6 +82,18 @@ func newInitCommand() *cobra.Command { if err != nil { return err } + + // The target is what the whole scaffold is named and shaped around, + // so it is settled before anything derived from it. + if target == "" { + target, err = resolveAgentTarget(cmd, azdProject) + if err != nil { + return err + } + } + if evalName == "" { + evalName = defaultEvalName(target, source) + } if judgeModel == "" { judgeModel = detectModelDeployment(azdProject) } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_target.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_target.go new file mode 100644 index 00000000000..9a44bae691c --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_target.go @@ -0,0 +1,80 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "sort" + + "azureaieval/internal/messages" + "azureaieval/internal/project" + + "github.com/azure/azure-dev/cli/azd/pkg/azdext" + "github.com/spf13/cobra" +) + +// agentServices names every agent the project declares, sorted so a prompt and +// an error list the same way twice. +func agentServices(proj *azdext.ProjectConfig) []string { + var names []string + for name, svc := range proj.GetServices() { + if svc.GetHost() == project.AgentHost { + names = append(names, name) + } + } + sort.Strings(names) + return names +} + +// resolveAgentTarget settles which agent the scaffold is written for. +// +// The spec makes --target default to the project's only agent, so requiring it +// would put a flag in front of the one command a developer runs first. With +// several agents there is nothing to detect, so it asks; under --no-prompt it +// names the flag rather than guessing which agent someone meant. +// +// It does not announce what it found: init prints that for every target, so +// that an explicit --target and a detected one read the same. +func resolveAgentTarget(cmd *cobra.Command, proj *azdext.ProjectConfig) (string, error) { + agents := agentServices(proj) + switch len(agents) { + case 0: + return "", messages.NoAgentToEvaluate() + case 1: + return agents[0], nil + } + + if noPrompt(cmd) { + return "", messages.AmbiguousAgentTarget(agents) + } + return promptAgentTarget(cmd, agents) +} + +// promptAgentTarget asks which of the project's agents to evaluate. +func promptAgentTarget(cmd *cobra.Command, agents []string) (string, error) { + azdClient, err := azdext.NewAzdClient() + if err != nil { + return "", messages.ConnectingToAzd(err) + } + defer azdClient.Close() + + choices := make([]*azdext.SelectChoice, 0, len(agents)) + for i := range agents { + choices = append(choices, &azdext.SelectChoice{Label: agents[i], Value: agents[i]}) + } + + resp, err := azdClient.Prompt().Select(cmd.Context(), &azdext.SelectRequest{ + Options: &azdext.SelectOptions{ + Message: messages.SelectAgentPrompt(), + Choices: choices, + }, + }) + if err != nil { + return "", messages.SelectingAgent(err) + } + index := int(resp.GetValue()) + if index < 0 || index >= len(agents) { + return "", messages.AmbiguousAgentTarget(agents) + } + return agents[index], nil +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_target_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_target_test.go new file mode 100644 index 00000000000..9cef7d72df4 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_target_test.go @@ -0,0 +1,101 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "testing" + + "azureaieval/internal/project" + + "github.com/azure/azure-dev/cli/azd/pkg/azdext" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// projectWithHosts builds a project whose services carry hosts, which is what +// agent detection keys on. The sibling projectWith helper sets names only. +func projectWithHosts(services map[string]string) *azdext.ProjectConfig { + svcs := map[string]*azdext.ServiceConfig{} + for name, host := range services { + svcs[name] = &azdext.ServiceConfig{Name: name, Host: host} + } + return &azdext.ProjectConfig{Services: svcs} +} + +// The spec's first hero command is `azd ai eval init --source traces`, with no +// target. Requiring the flag put it in front of the one command a developer +// runs first. +func TestAgentServices_FindsTheOnlyAgent(t *testing.T) { + proj := projectWithHosts(map[string]string{ + "ai-project": "azure.ai.project", + "support-agent": project.AgentHost, + }) + + agents := agentServices(proj) + + require.Len(t, agents, 1) + assert.Equal(t, "support-agent", agents[0]) +} + +// Sorted, so a prompt and an error list them the same way twice. +func TestAgentServices_AreSorted(t *testing.T) { + proj := projectWithHosts(map[string]string{ + "zebra-agent": project.AgentHost, + "alpha-agent": project.AgentHost, + "ai-project": "azure.ai.project", + "support-agent": project.AgentHost, + }) + + assert.Equal(t, []string{"alpha-agent", "support-agent", "zebra-agent"}, agentServices(proj)) +} + +func TestAgentServices_NoneWhenTheProjectHasNoAgent(t *testing.T) { + proj := projectWithHosts(map[string]string{"ai-project": "azure.ai.project"}) + + assert.Empty(t, agentServices(proj)) +} + +// A project with no agent cannot be scaffolded, and the error has to say that +// rather than name a flag the developer has nothing to put in. +func TestResolveAgentTarget_NoAgent(t *testing.T) { + cmd := newInitCommand() + + _, err := resolveAgentTarget(cmd, projectWithHosts(map[string]string{ + "ai-project": "azure.ai.project", + })) + + require.Error(t, err) + assert.Contains(t, err.Error(), "no agent service") +} + +// With several agents there is nothing to detect. Under --no-prompt the flag is +// the only way to say which, so the error names it and lists the candidates. +func TestResolveAgentTarget_AmbiguousUnderNoPrompt(t *testing.T) { + cmd := newInitCommand() + // --no-prompt is inherited from the root in the real tree, so the test has + // to supply it the way the root does. + cmd.Flags().Bool("no-prompt", true, "") + + _, err := resolveAgentTarget(cmd, projectWithHosts(map[string]string{ + "one-agent": project.AgentHost, + "two-agent": project.AgentHost, + })) + + require.Error(t, err) + assert.Contains(t, err.Error(), "--target") + assert.Contains(t, err.Error(), "one-agent") + assert.Contains(t, err.Error(), "two-agent") +} + +func TestResolveAgentTarget_DetectsTheSoleAgent(t *testing.T) { + cmd := newInitCommand() + + target, err := resolveAgentTarget(cmd, projectWithHosts(map[string]string{ + "ai-project": "azure.ai.project", + "support-agent": project.AgentHost, + })) + + require.NoError(t, err) + assert.Equal(t, "support-agent", target) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/output.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/output.go index 4a4f6b97446..46e7505acf0 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/output.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/output.go @@ -56,6 +56,18 @@ func isJSON(cmd *cobra.Command) bool { return outputFormat(cmd) == outputJSON } +// noPrompt reports whether the caller asked for no interaction. +// +// JSON output counts: a prompt written into a document nobody is reading is a +// hang, not a question. +func noPrompt(cmd *cobra.Command) bool { + if isJSON(cmd) { + return true + } + value, err := cmd.Flags().GetBool("no-prompt") + return err == nil && value +} + // emitJSON writes v as indented JSON. func emitJSON(w io.Writer, v any) error { enc := json.NewEncoder(w) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go b/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go index c8a9398efef..a4412157282 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go @@ -1323,6 +1323,30 @@ func DetectedTarget(target string) string { return fmt.Sprintf("%s Detected agent target: %s\n", DoneMark, target) } +// NoAgentToEvaluate reports a project declaring no agent service. +func NoAgentToEvaluate() error { + return errors.New( + "this project declares no agent service to evaluate. Add one, or name an " + + "existing agent with --target") +} + +// AmbiguousAgentTarget reports several agents where only one can be scaffolded. +func AmbiguousAgentTarget(agents []string) error { + return fmt.Errorf( + "this project declares more than one agent (%s), so --target says which to "+ + "evaluate", strings.Join(agents, ", ")) +} + +// SelectAgentPrompt asks which agent the eval is for. +func SelectAgentPrompt() string { + return "Select the agent to evaluate:" +} + +// SelectingAgent reports a failed agent prompt. +func SelectingAgent(err error) error { + return fmt.Errorf("selecting an agent to evaluate: %w", err) +} + // UsingTraceSource reports a scaffold that reads production traces. func UsingTraceSource() string { return fmt.Sprintf("%s Using data source: traces (Application Insights)\n", DoneMark) From fe8c31c9dddd748cd17389ccb5786a24aef8d711 Mon Sep 17 00:00:00 2001 From: mohessie Date: Mon, 10 Aug 2026 03:25:48 +0300 Subject: [PATCH 153/320] Name the dataset in the run summary and the CI handoff Scenario 3's summary carries a Dataset line between Eval and Status, and Scenario 5's --no-wait handoff carries dataset and dataset_version. Neither was emitted, so a pipeline recording a pass rate could not say afterwards what the rate was measured against. Both read the run's own metadata rather than the configuration, so they name what this run scored and not what the file says today. The handoff falls back to the declaration for the name, because the create response does not always echo metadata back. --- .../azure.ai.evaluations/internal/cmd/run.go | 26 +++++++++++++++---- .../internal/cmd/run_ops.go | 13 ++++++++++ 2 files changed, 34 insertions(+), 5 deletions(-) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go index bd727ea9306..e2f59ba5841 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go @@ -732,11 +732,15 @@ func (ec *evalContext) pollRun( // API happens to return, so a script reading it would depend on a shape this // extension does not control. type startedRunHandoff struct { - RunID string `json:"run_id"` - EvalID string `json:"eval_id"` - EvalName string `json:"eval_name,omitempty"` - Status string `json:"status,omitempty"` - CreatedAt string `json:"created_at,omitempty"` + RunID string `json:"run_id"` + EvalID string `json:"eval_id"` + EvalName string `json:"eval_name,omitempty"` + // Which rows the run scored. A pipeline that records only a pass rate + // cannot say later what the rate was measured against. + Dataset string `json:"dataset,omitempty"` + DatasetVersion string `json:"dataset_version,omitempty"` + Status string `json:"status,omitempty"` + CreatedAt string `json:"created_at,omitempty"` } // startedRun builds the handoff. @@ -751,9 +755,18 @@ func startedRun( Status: run.Status, CreatedAt: timestampString(run.CreatedAt), } + // Read back from the run rather than the configuration, so the handoff + // names what this run scored and not what the file says today. The create + // response does not always echo metadata, so the declaration is the + // fallback for the name. + handoff.Dataset = run.Metadata[metaDataset] + handoff.DatasetVersion = run.Metadata[metaDatasetVersion] // Absent with --eval-id, where there is no config to take a name from. if group != nil { handoff.EvalName = group.Name + if handoff.Dataset == "" { + handoff.Dataset = group.Dataset + } } return handoff } @@ -871,6 +884,9 @@ func renderRunHeader(out interface{ Write([]byte) (int, error) }, run *eval_api. fmt.Fprintf(out, "%-10s %s\n", "Eval", run.EvalID) } fmt.Fprintf(out, "%-10s %s\n", "Status", run.Status) + if ds := runDatasetLine(run.Metadata); ds != "" { + fmt.Fprintf(out, "%-10s %s\n", "Dataset", ds) + } if c := run.ResultCounts; c != nil && c.Total > 0 { fmt.Fprintf(out, "%-10s %d\n", "Samples", c.Total) } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go index 6b780db08c9..e10b5fda052 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go @@ -334,6 +334,19 @@ func runDataset(metadata map[string]string) string { return name } +// runDatasetLine is the same fact spelled for a detail view, where there is +// room for the whole word. +func runDatasetLine(metadata map[string]string) string { + name := metadata[metaDataset] + if name == "" { + return "" + } + if version := metadata[metaDatasetVersion]; version != "" { + return fmt.Sprintf("%s (version %s)", name, version) + } + return name +} + // sampleCount is how many rows the run scored, which is what makes two rows of // `run list` comparable: a rate over 15 samples and one over 200 are not the // same claim. From 2c8967f4f8b70735ffb16662cd5ae401d12ac268 Mon Sep 17 00:00:00 2001 From: mohessie Date: Mon, 10 Aug 2026 03:29:56 +0300 Subject: [PATCH 154/320] Put the run summary in the order the scenario reads it Scenario 3 names the eval, then the dataset, then the status. The summary showed the eval's id because the name lived only in the eval's own metadata, and a run is read on its own; and the dataset landed after the status. The run now records the eval name alongside the dataset it scored, and the header prints them where the scenario shows them. --- .../extensions/azure.ai.evaluations/internal/cmd/run.go | 9 +++++++-- .../azure.ai.evaluations/internal/cmd/run_ops.go | 3 +++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go index e2f59ba5841..a15cb0584bd 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go @@ -158,6 +158,11 @@ func buildRunCommand(use, short string) *cobra.Command { if lvl := resolveLevel(group); lvl != "" { metadata["evaluation_level"] = lvl } + // The eval carries its name in its own metadata, but a run is read + // on its own, and an id is not what the author called it. + if group != nil && group.Name != "" { + metadata[metaEvalName] = group.Name + } // Recorded per run, not read from the configuration at list time: // comparing two runs is the point of that listing, and the dataset // under an eval can change between them. A source-backed run scored @@ -878,15 +883,15 @@ func renderRun( // because the run carries only an id and the id is not what anyone declared. func renderRunHeader(out interface{ Write([]byte) (int, error) }, run *eval_api.OpenAIEvalRun) { fmt.Fprintf(out, "%-10s %s\n", "Run", run.ID) - if name := run.Metadata["azd_eval"]; name != "" { + if name := run.Metadata[metaEvalName]; name != "" { fmt.Fprintf(out, "%-10s %s\n", "Eval", name) } else if run.EvalID != "" { fmt.Fprintf(out, "%-10s %s\n", "Eval", run.EvalID) } - fmt.Fprintf(out, "%-10s %s\n", "Status", run.Status) if ds := runDatasetLine(run.Metadata); ds != "" { fmt.Fprintf(out, "%-10s %s\n", "Dataset", ds) } + fmt.Fprintf(out, "%-10s %s\n", "Status", run.Status) if c := run.ResultCounts; c != nil && c.Total > 0 { fmt.Fprintf(out, "%-10s %d\n", "Samples", c.Total) } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go index e10b5fda052..35b0a3bd916 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go @@ -315,6 +315,9 @@ func summarizeCounts(counts *eval_api.EvalRunResultCounts) string { const ( metaDataset = "azd_dataset" metaDatasetVersion = "azd_dataset_version" + // metaEvalName is the eval's declared name, recorded on the run because a + // run is read on its own and an id is not what the author called it. + metaEvalName = "azd_eval" ) // runDataset renders the dataset a run scored, versioned when a version was From 5904367780b39e6ca62a6a6a7313fc621ace95b5 Mon Sep 17 00:00:00 2001 From: mohessie Date: Mon, 10 Aug 2026 03:46:22 +0300 Subject: [PATCH 155/320] Show a version list what changed, and accept a file Windows wrote Scenario 4 reads an evaluator's history to see how the rubric changed. The listing showed NAME, VERSION, TYPE -- name and type are the same on every row of a list scoped to one evaluator, so it answered nothing. It now shows the version, when it was published and the description, which is what differs. Timestamps are normalised on the way out. The service returns sub-second precision with an offset on this route and epoch seconds on others, so two listings spelled the same instant differently. Reading a hand-edited file now tolerates a UTF-8 BOM. Notepad, VS Code on Windows and PowerShell all write one, neither encoding/json nor yaml.v3 skips it, and what a developer sees is "invalid character" naming a character rather than the cause. Hit for real while editing a rubric during this scenario. Pass threshold is deliberately not a column: the spec shows it on the local rubric, but neither the generated artifact nor the service's definition carries one, so it would be blank on every row. --- .../internal/cmd/evaluator.go | 26 ++++++- .../internal/cmd/reconciler.go | 2 +- .../azure.ai.evaluations/internal/cmd/run.go | 8 ++- .../internal/pkg/eval_api/evaluators.go | 1 + .../internal/project/eval_config_store.go | 2 +- .../internal/project/readfile.go | 27 +++++++ .../internal/project/readfile_test.go | 70 +++++++++++++++++++ 7 files changed, 130 insertions(+), 6 deletions(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/project/readfile.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/project/readfile_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator.go index 8fcc1d04320..9e5ff2f6785 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator.go @@ -8,11 +8,11 @@ import ( "encoding/json" "fmt" "io" - "os" "strings" "azureaieval/internal/messages" "azureaieval/internal/pkg/eval_api" + "azureaieval/internal/project" "github.com/spf13/cobra" ) @@ -68,7 +68,7 @@ func newEvaluatorWriteCommand(verb, short string) *cobra.Command { return requireFlag("from-file") } - raw, err := os.ReadFile(fromFile) + raw, err := project.ReadFileNoBOM(fromFile) if err != nil { return messages.ReadingEvaluator(fromFile, err) } @@ -273,7 +273,7 @@ func newEvaluatorVersionsListCommand() *cobra.Command { if err != nil { return messages.ListingEvaluatorVersions(name, err) } - return renderEvaluators(cmd, list) + return renderEvaluatorVersions(cmd, list) }, } @@ -296,6 +296,26 @@ func renderEvaluators(cmd *cobra.Command, list *eval_api.EvaluatorListResponse) return emitTable(cmd.OutOrStdout(), []string{"NAME", "VERSION", "TYPE"}, rows) } +// renderEvaluatorVersions lists one evaluator's history. +// +// Name and type are the same on every row here, so they say nothing. What the +// scenario reads a version list for is how the rubric changed, which is the +// date and the description the author left. +func renderEvaluatorVersions(cmd *cobra.Command, list *eval_api.EvaluatorListResponse) error { + if isJSON(cmd) { + return emitJSONList(cmd.OutOrStdout(), list.Value) + } + if len(list.Value) == 0 { + fmt.Fprint(cmd.OutOrStdout(), messages.NoEvaluators()) + return nil + } + rows := make([][]string, 0, len(list.Value)) + for _, e := range list.Value { + rows = append(rows, []string{e.Version, timestampString(e.CreatedAt), e.Description}) + } + return emitTable(cmd.OutOrStdout(), []string{"VERSION", "CREATED AT", "DESCRIPTION"}, rows) +} + func newEvaluatorShowCommand() *cobra.Command { var ( version string diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go index b83ea62b265..e98cbc15987 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go @@ -238,7 +238,7 @@ func (r *evalReconciler) EnsureEvaluator( return "", false, messages.EvaluatorSource(localPath, err) } - raw, err := os.ReadFile(localPath) + raw, err := project.ReadFileNoBOM(localPath) if err != nil { return "", false, messages.EvaluatorSource(localPath, err) } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go index a15cb0584bd..533acc62466 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go @@ -737,7 +737,7 @@ func (ec *evalContext) pollRun( // API happens to return, so a script reading it would depend on a shape this // extension does not control. type startedRunHandoff struct { - RunID string `json:"run_id"` + RunID string `json:"run_id"` EvalID string `json:"eval_id"` EvalName string `json:"eval_name,omitempty"` // Which rows the run scored. A pipeline that records only a pass rate @@ -786,6 +786,12 @@ func timestampString(value any) string { case nil: return "" case string: + // Normalised, not passed through: the service returns sub-second + // precision and an offset here and epoch seconds elsewhere, so two + // listings would otherwise spell the same instant differently. + if parsed, err := time.Parse(time.RFC3339, t); err == nil { + return parsed.UTC().Format(time.RFC3339) + } return t case float64: return time.Unix(int64(t), 0).UTC().Format(time.RFC3339) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/evaluators.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/evaluators.go index dd230ef1f08..a66ac99acb4 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/evaluators.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/evaluators.go @@ -61,6 +61,7 @@ type EvaluatorSummary struct { Name string `json:"name"` Version string `json:"version,omitempty"` Description string `json:"description,omitempty"` + CreatedAt string `json:"created_at,omitempty"` // The listing spells this evaluator_type; `type` is accepted too because // other evaluator payloads use it. diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_store.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_store.go index 79611caabcd..0eac431d8d1 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_store.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_store.go @@ -90,7 +90,7 @@ func OpenEvalConfig(evalDir string) (*EvalConfig, error) { // LoadEvalConfig reads a configuration from an explicit path. The path is used // verbatim, relative to the process working directory — never re-rooted. func LoadEvalConfig(path string) (*EvalConfig, error) { - data, err := os.ReadFile(path) + data, err := ReadFileNoBOM(path) if err != nil { return nil, messages.ReadingEvalConfig(path, err) } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/readfile.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/readfile.go new file mode 100644 index 00000000000..446d3531e11 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/readfile.go @@ -0,0 +1,27 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package project + +import ( + "bytes" + "os" +) + +// utf8BOM is what Windows editors and PowerShell's Set-Content write ahead of +// otherwise valid UTF-8. +var utf8BOM = []byte{0xEF, 0xBB, 0xBF} + +// ReadFileNoBOM reads a file the user may have edited by hand, without the byte +// order mark a Windows editor puts in front of it. +// +// Neither encoding/json nor yaml.v3 skips one, and the error they raise names a +// character rather than the cause: "invalid character 'ï' looking for beginning +// of value" is not something a developer can act on. +func ReadFileNoBOM(path string) ([]byte, error) { + data, err := os.ReadFile(path) //nolint:gosec // a path the caller named + if err != nil { + return nil, err + } + return bytes.TrimPrefix(data, utf8BOM), nil +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/readfile_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/readfile_test.go new file mode 100644 index 00000000000..07ba4497226 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/readfile_test.go @@ -0,0 +1,70 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package project + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// Notepad, VS Code on Windows and PowerShell's Set-Content all write a BOM. +// Neither encoding/json nor yaml.v3 skips one, and what a developer sees is +// "invalid character 'ï' looking for beginning of value" — which names a +// character, not the cause. Hit for real while editing a rubric by hand. +func TestReadFileNoBOM_StripsTheWindowsByteOrderMark(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "rubric.json") + body := append([]byte{0xEF, 0xBB, 0xBF}, []byte(`{"dimensions":[]}`)...) + require.NoError(t, os.WriteFile(path, body, 0o600)) + + data, err := ReadFileNoBOM(path) + + require.NoError(t, err) + assert.Equal(t, `{"dimensions":[]}`, string(data)) +} + +// A file without one is returned byte for byte. +func TestReadFileNoBOM_LeavesOrdinaryContentAlone(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "rubric.json") + require.NoError(t, os.WriteFile(path, []byte(`{"dimensions":[]}`), 0o600)) + + data, err := ReadFileNoBOM(path) + + require.NoError(t, err) + assert.Equal(t, `{"dimensions":[]}`, string(data)) +} + +// Only a leading mark is a BOM. The same bytes inside the content are content. +func TestReadFileNoBOM_OnlyStripsALeadingMark(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "rubric.json") + body := []byte("{\"note\":\"\uFEFF inside\"}") + require.NoError(t, os.WriteFile(path, body, 0o600)) + + data, err := ReadFileNoBOM(path) + + require.NoError(t, err) + assert.Equal(t, string(body), string(data)) +} + +// A configuration saved by a Windows editor has to load, or every command that +// reads it fails at once. +func TestLoadEvalConfig_AcceptsAByteOrderMark(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, EvalConfigBase) + body := append([]byte{0xEF, 0xBB, 0xBF}, + []byte("datasets:\n - name: d\n source: ./d.jsonl\n")...) + require.NoError(t, os.WriteFile(path, body, 0o600)) + + cfg, err := LoadEvalConfig(path) + + require.NoError(t, err) + require.Len(t, cfg.Datasets, 1) + assert.Equal(t, "d", cfg.Datasets[0].Name) +} From 17f0025f0c35719e80358fdb4eedb85a4379346b Mon Sep 17 00:00:00 2001 From: mohessie Date: Mon, 10 Aug 2026 04:44:26 +0300 Subject: [PATCH 156/320] Run the tagged CLI suite and fix what it caught The tests under tests/cli, tests/live and tests/hero carry build tags, so `go test ./...` never compiled them and reported a clean run while 1,822 lines of end-to-end tests sat unexecuted. Running them surfaced seven failures. Two were real product bugs: - `run output export --format jsonl` was refused by a guard that named only json and csv, while the flag's own help offered jsonl and the writer already had a `formatJSONL` branch. The guard now checks the same set the writer switches on. ExportFormatInvalid is dropped; ExportFormatUnsupported names all three. - `evaluator show` had no `--output-file`, though the reconciliation error tells the reader to adopt a remote change by writing it over the local definition with that flag. It now writes the service's document verbatim, since anything the detail view drops would be lost on adoption. One was a missing column. `evaluator versions list` omitted PASS THRESHOLD. An earlier pass had skipped it on the belief that the service does not return a threshold; posting a rubric carrying `"pass_threshold": 0.7` shows it round -trips at definition.pass_threshold, so the value was there and only the display was missing. EvaluatorContract.PassThreshold is a *float64 so an absent threshold, which defers to the criterion's `threshold:` parameter, stays distinguishable from an explicit 0.0, which passes everything. The detail view gained the same field. The other four were stale assertions, re-pinned to the spec: - init was probed with `--generation-model`, which belongs to `generate`; init takes `--judge-model`. The unknown-flag error masked the assertion the test existed for, so it also now asserts it is not testing a typo. - `dataset show` renders a detail view keyed by label, not a one-row table with a header, matching its sibling `show` commands. - `run list` prints RUN, which is the spec's own header; the test pinned an older RUN ID/NAME/RESULTS set. - `evaluator show` renders the detail view and puts the full document behind `-o json`; two tests still expected the bare command to emit JSON. Adds unit coverage for the threshold rendering and live coverage for the jsonl export and the new --output-file. --- .../internal/cmd/detail_test.go | 82 +++++++++++++++++++ .../internal/cmd/evaluator.go | 50 ++++++++++- .../internal/cmd/run_output.go | 12 ++- .../internal/messages/messages.go | 7 +- .../internal/pkg/eval_api/evaluators.go | 6 +- .../tests/cli/dataset_test.go | 7 +- .../tests/cli/evaluator_test.go | 4 +- .../tests/cli/rubric_test.go | 28 +++++-- .../tests/cli/run_ops_test.go | 10 ++- .../tests/cli/run_output_test.go | 23 +++++- 10 files changed, 203 insertions(+), 26 deletions(-) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/detail_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/detail_test.go index 1e5ebd2540d..75986efd373 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/detail_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/detail_test.go @@ -13,6 +13,7 @@ import ( "azureaieval/internal/pkg/eval_api" + "github.com/spf13/cobra" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -160,3 +161,84 @@ func TestPortalEvaluatorURLShape(t *testing.T) { prefix.EvaluatorURL("support-quality", "3"), "/build/evaluations/catalog/support-quality/3")) } + +// The pass mark is what Scenario 4 changes between versions, so a version +// listing that omits it cannot answer the question it is read for. +func TestEvaluatorPassThreshold(t *testing.T) { + threshold := func(v float64) *eval_api.EvaluatorContract { + return &eval_api.EvaluatorContract{PassThreshold: &v} + } + + cases := []struct { + name string + in *eval_api.EvaluatorSummary + want string + }{ + {"absent evaluator", nil, ""}, + {"no definition", &eval_api.EvaluatorSummary{}, ""}, + { + "definition without a threshold", + &eval_api.EvaluatorSummary{Definition: &eval_api.EvaluatorContract{}}, + "", + }, + { + "a threshold of zero is a real threshold, not an absent one", + &eval_api.EvaluatorSummary{Definition: threshold(0)}, + "0.00", + }, + { + "two decimals, because 0.7 and 0.75 pass different samples", + &eval_api.EvaluatorSummary{Definition: threshold(0.75)}, + "0.75", + }, + { + "a trailing zero is kept so the column stays aligned", + &eval_api.EvaluatorSummary{Definition: threshold(0.8)}, + "0.80", + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + assert.Equal(t, tc.want, evaluatorPassThreshold(tc.in)) + }) + } +} + +func TestRenderEvaluatorVersionsShowsTheThreshold(t *testing.T) { + raised, held := 0.80, 0.70 + var buf bytes.Buffer + + cmd := &cobra.Command{} + cmd.SetOut(&buf) + + require.NoError(t, renderEvaluatorVersions(cmd, &eval_api.EvaluatorListResponse{ + Value: []eval_api.EvaluatorSummary{ + { + Version: "3", + CreatedAt: "2026-08-03T11:22:04Z", + Description: "Raised threshold, split cites_policy", + Definition: &eval_api.EvaluatorContract{PassThreshold: &raised}, + }, + { + Version: "2", + CreatedAt: "2026-08-01T14:07:33Z", + Description: "Tightened offers_next_step criteria", + Definition: &eval_api.EvaluatorContract{PassThreshold: &held}, + }, + }, + })) + + out := buf.String() + for _, want := range []string{ + "VERSION", "CREATED AT", "PASS THRESHOLD", "DESCRIPTION", + "0.80", "0.70", + "Raised threshold, split cites_policy", + } { + assert.Contains(t, out, want) + } + + // Name and type are constant down the listing, so printing them would cost + // width and say nothing. + assert.NotContains(t, out, "NAME") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator.go index 9e5ff2f6785..6943ea95ab8 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator.go @@ -8,6 +8,8 @@ import ( "encoding/json" "fmt" "io" + "os" + "strconv" "strings" "azureaieval/internal/messages" @@ -296,11 +298,23 @@ func renderEvaluators(cmd *cobra.Command, list *eval_api.EvaluatorListResponse) return emitTable(cmd.OutOrStdout(), []string{"NAME", "VERSION", "TYPE"}, rows) } +// evaluatorPassThreshold renders the rubric's pass mark for a table cell, +// empty when the evaluator does not carry one. +// +// Two decimals because the scale is normalized 0.0-1.0, where the difference +// between 0.7 and 0.75 is a real change in what passes. +func evaluatorPassThreshold(e *eval_api.EvaluatorSummary) string { + if e == nil || e.Definition == nil || e.Definition.PassThreshold == nil { + return "" + } + return strconv.FormatFloat(*e.Definition.PassThreshold, 'f', 2, 64) +} + // renderEvaluatorVersions lists one evaluator's history. // // Name and type are the same on every row here, so they say nothing. What the // scenario reads a version list for is how the rubric changed, which is the -// date and the description the author left. +// date, the pass mark and the description the author left. func renderEvaluatorVersions(cmd *cobra.Command, list *eval_api.EvaluatorListResponse) error { if isJSON(cmd) { return emitJSONList(cmd.OutOrStdout(), list.Value) @@ -311,14 +325,21 @@ func renderEvaluatorVersions(cmd *cobra.Command, list *eval_api.EvaluatorListRes } rows := make([][]string, 0, len(list.Value)) for _, e := range list.Value { - rows = append(rows, []string{e.Version, timestampString(e.CreatedAt), e.Description}) + rows = append(rows, []string{ + e.Version, + timestampString(e.CreatedAt), + evaluatorPassThreshold(&e), + e.Description, + }) } - return emitTable(cmd.OutOrStdout(), []string{"VERSION", "CREATED AT", "DESCRIPTION"}, rows) + return emitTable(cmd.OutOrStdout(), + []string{"VERSION", "CREATED AT", "PASS THRESHOLD", "DESCRIPTION"}, rows) } func newEvaluatorShowCommand() *cobra.Command { var ( version string + outFile string endpointFlg string ) @@ -344,6 +365,26 @@ func newEvaluatorShowCommand() *cobra.Command { return messages.ReadingEvaluator(name, err) } + // --output-file writes the service's document verbatim, because + // reconciliation points here to adopt a remote change over the local + // definition: anything this view dropped would be lost on adoption. + if outFile != "" { + body := raw + var pretty any + if err := json.Unmarshal(raw, &pretty); err == nil { + if indented, err := json.MarshalIndent(pretty, "", " "); err == nil { + body = append(indented, '\n') + } + } + if err := os.WriteFile(outFile, body, 0o600); err != nil { + return messages.Creating(outFile, err) + } + if !isJSON(cmd) { + fmt.Fprint(cmd.OutOrStdout(), messages.WroteArtifact(outFile)) + } + return nil + } + // -o json answers with the service's document untouched, because a // caller asking for JSON wants the evaluator, not this view of it. if isJSON(cmd) { @@ -367,6 +408,8 @@ func newEvaluatorShowCommand() *cobra.Command { } cmd.Flags().StringVar(&version, "version", "", "Version to show. Omit for the latest.") + cmd.Flags().StringVar(&outFile, "output-file", "", + "Write the evaluator document to this path instead of stdout.") cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") return cmd } @@ -386,6 +429,7 @@ func (ec *evalContext) renderEvaluator( {"Name", e.Name}, {"Version", e.Version}, {"Type", e.Type()}, + {"Pass Threshold", evaluatorPassThreshold(e)}, {"Description", e.Description}, {"Categories", strings.Join(e.Categories, ", ")}, {"Evaluation Levels", strings.Join(e.SupportedEvaluationLevels, ", ")}, diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output.go index 7d5d936d901..26ca0bd3f5b 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output.go @@ -170,12 +170,18 @@ func newRunOutputExportCommand() *cobra.Command { cmd := &cobra.Command{ Use: "export [run]", - Short: "Export run results as JSON or CSV.", + Short: "Export run results as CSV, JSON or JSONL.", Args: cobra.MaximumNArgs(1), RunE: func(cmd *cobra.Command, args []string) error { format = strings.ToLower(format) - if format != "json" && format != "csv" { - return messages.ExportFormatInvalid(format) + // Checked against the same set the writer switches on: this guard + // used to name only json and csv, so --format jsonl was refused by a + // CLI whose own help offered it. + switch format { + case formatCSV, formatJSON, formatJSONL: + default: + return messages.ExportFormatUnsupported( + format, formatCSV, formatJSON, formatJSONL) } ctx := cmd.Context() diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go b/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go index a4412157282..96829bd28b6 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go @@ -342,12 +342,7 @@ func ReportLinkAfterRows(url string) string { return fmt.Sprintf("\nReport: %s\n", url) } -// ExportFormatInvalid reports an --format the export command cannot write. -func ExportFormatInvalid(format string) error { - return fmt.Errorf("--format must be json or csv, got %q", format) -} - -// ExportFormatUnsupported reports an --format that got past the flag check. +// ExportFormatUnsupported reports an --format the export command cannot write. func ExportFormatUnsupported(format, csv, json, jsonl string) error { return fmt.Errorf( "--format %q is not supported; use %s, %s or %s", diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/evaluators.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/evaluators.go index a66ac99acb4..ae70d8b5273 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/evaluators.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/evaluators.go @@ -48,7 +48,11 @@ func (s *JSONSchema) Accepts(name string) bool { // EvaluatorContract is the published input contract for an evaluator: which // data fields it consumes and which initialization parameters it takes. type EvaluatorContract struct { - Type string `json:"type,omitempty"` + Type string `json:"type,omitempty"` + // PassThreshold is a pointer because an absent threshold and a zero one are + // different claims: zero passes every sample, absent defers to the + // `threshold:` init parameter on the criterion that uses this evaluator. + PassThreshold *float64 `json:"pass_threshold,omitempty"` DataSchema *JSONSchema `json:"data_schema,omitempty"` InitParameters *JSONSchema `json:"init_parameters,omitempty"` } diff --git a/cli/azd/extensions/azure.ai.evaluations/tests/cli/dataset_test.go b/cli/azd/extensions/azure.ai.evaluations/tests/cli/dataset_test.go index d82415517fc..efe36c604b6 100644 --- a/cli/azd/extensions/azure.ai.evaluations/tests/cli/dataset_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/tests/cli/dataset_test.go @@ -166,8 +166,11 @@ func TestCLIDatasetShow(t *testing.T) { t.Run("table", func(t *testing.T) { r := requireSuccess(t, run(t, "dataset", "show", ds.Name)) - for _, header := range []string{"NAME", "VERSION", "FORMAT", "URI"} { - require.Containsf(t, r.Stdout, header, "the table lost its %s column", header) + // `show` reads one dataset, so it renders a detail view keyed by label + // rather than a one-row table with a header. The labels are what is + // under test; the column-header form belongs to `dataset list`. + for _, label := range []string{"Name", "Version", "URI"} { + require.Containsf(t, r.Stdout, label, "the detail view lost its %s line", label) } require.Contains(t, r.Stdout, ds.Name) }) diff --git a/cli/azd/extensions/azure.ai.evaluations/tests/cli/evaluator_test.go b/cli/azd/extensions/azure.ai.evaluations/tests/cli/evaluator_test.go index e6c97dffd9e..fbcab7ee126 100644 --- a/cli/azd/extensions/azure.ai.evaluations/tests/cli/evaluator_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/tests/cli/evaluator_test.go @@ -84,9 +84,11 @@ func TestCLIInitNeedsAnAzdProject(t *testing.T) { r := requireFailure(t, runIn(t, dir, "init", "--target", "probe-agent", - "--generation-model", "gpt-4o-mini", + "--judge-model", "gpt-4o-mini", "--no-prompt")) + require.NotContains(t, r.Combined(), "unknown flag", + "the probe must use init's real flags, or it asserts nothing about projects") require.Contains(t, r.Combined(), "azd init", "the refusal must name the command that makes a project") require.NotContains(t, strings.ToLower(r.Combined()), "grpc", diff --git a/cli/azd/extensions/azure.ai.evaluations/tests/cli/rubric_test.go b/cli/azd/extensions/azure.ai.evaluations/tests/cli/rubric_test.go index c104f369e46..755bab67ffc 100644 --- a/cli/azd/extensions/azure.ai.evaluations/tests/cli/rubric_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/tests/cli/rubric_test.go @@ -62,7 +62,7 @@ func TestCLIRubricRoundTrip(t *testing.T) { run(t, "evaluator", "delete", name, "--version", "1") }) - shown := requireSuccess(t, run(t, "evaluator", "show", name)) + shown := requireSuccess(t, run(t, "evaluator", "show", name, "-o", "json")) var doc evaluatorDocument shown.JSON(t, &doc) @@ -102,7 +102,7 @@ func TestCLIRubricRoundTrip(t *testing.T) { // The earlier version stays reachable, which is what makes a published // version safe to reference from a config. - pinned := requireSuccess(t, run(t, "evaluator", "show", name, "--version", "1")) + pinned := requireSuccess(t, run(t, "evaluator", "show", name, "--version", "1", "-o", "json")) var first evaluatorDocument pinned.JSON(t, &first) require.Equal(t, "1", first.Version) @@ -158,9 +158,10 @@ func TestCLIRubricNeedsDimensions(t *testing.T) { require.Contains(t, r.Combined(), "dimensions") } -// TestCLIEvaluatorShowAcceptsAFullDocument proves `evaluator show` emits JSON a -// script can consume, whatever the definition kind. It renders the service's -// body rather than a typed struct, so nothing else pins that it stays parseable. +// TestCLIEvaluatorShowAcceptsAFullDocument proves `evaluator show -o json` +// emits JSON a script can consume, whatever the definition kind. It renders the +// service's body rather than a typed struct, so nothing else pins that it stays +// parseable. The bare command renders the human detail view instead. func TestCLIEvaluatorShowAcceptsAFullDocument(t *testing.T) { name := uniqueName("azdcli_rubricdoc") @@ -176,7 +177,7 @@ func TestCLIEvaluatorShowAcceptsAFullDocument(t *testing.T) { run(t, "evaluator", "delete", name, "--version", "1") }) - shown := requireSuccess(t, run(t, "evaluator", "show", name)) + shown := requireSuccess(t, run(t, "evaluator", "show", name, "-o", "json")) var raw map[string]any require.NoError(t, json.Unmarshal([]byte(shown.Stdout), &raw), "evaluator show must emit parseable JSON:\n%s", shown.Stdout) @@ -185,4 +186,19 @@ func TestCLIEvaluatorShowAcceptsAFullDocument(t *testing.T) { require.Equal(t, name, raw["name"], "--name must decide the evaluator's name, not the document's own field") require.NotContains(t, strings.ToLower(shown.Stdout), `"name": "ignored"`) + + // Reconciliation tells people to adopt a remote change by writing it over + // the local definition with --output-file, so the flag has to exist and has + // to land the same document the service holds. + adopted := filepath.Join(t.TempDir(), "adopted.json") + requireSuccess(t, run(t, "evaluator", "show", name, "--output-file", adopted)) + + body, err := os.ReadFile(adopted) + require.NoError(t, err) + var written map[string]any + require.NoError(t, json.Unmarshal(body, &written), + "--output-file must write parseable JSON:\n%s", string(body)) + require.Equal(t, name, written["name"]) + require.Contains(t, written, "definition", + "adopting a remote change needs the definition, not just its identity") } diff --git a/cli/azd/extensions/azure.ai.evaluations/tests/cli/run_ops_test.go b/cli/azd/extensions/azure.ai.evaluations/tests/cli/run_ops_test.go index 3d68a583083..58faab218e4 100644 --- a/cli/azd/extensions/azure.ai.evaluations/tests/cli/run_ops_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/tests/cli/run_ops_test.go @@ -29,13 +29,17 @@ func TestCLIRunList(t *testing.T) { t.Run("table", func(t *testing.T) { r := requireSuccess(t, run(t, "run", "list", "--eval", f.EvalID)) - for _, header := range []string{"RUN ID", "NAME", "STATUS", "RESULTS"} { + // These are the columns the spec's `run list` sample prints, in the + // spec's own wording. The old RUN ID/NAME/RESULTS set predates it. + for _, header := range []string{ + "RUN", "DATASET", "STARTED", "STATUS", "SAMPLES", "PASS RATE", + } { require.Containsf(t, r.Stdout, header, "the listing lost its %s column", header) } require.Contains(t, r.Stdout, f.FirstRunID) require.Contains(t, r.Stdout, f.SecondRunID) - require.Regexp(t, `\d+ passed, \d+ failed, \d+ errored`, r.Stdout, - "the listing must summarise each run's counts, not just its status") + require.Regexp(t, `\d+\.\d+%`, r.Stdout, + "the listing must summarise each run's pass rate, not just its status") }) t.Run("json", func(t *testing.T) { diff --git a/cli/azd/extensions/azure.ai.evaluations/tests/cli/run_output_test.go b/cli/azd/extensions/azure.ai.evaluations/tests/cli/run_output_test.go index 181cc5df69b..02163578a2d 100644 --- a/cli/azd/extensions/azure.ai.evaluations/tests/cli/run_output_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/tests/cli/run_output_test.go @@ -7,6 +7,7 @@ package cli import ( "encoding/csv" + "encoding/json" "os" "path/filepath" "regexp" @@ -215,7 +216,27 @@ func TestCLIResultsExport(t *testing.T) { t.Run("an unknown format is refused", func(t *testing.T) { r := requireFailure(t, run(t, "run", "output", "export", f.FirstRunID, "--eval", f.EvalID, "--format", "xml")) - require.Contains(t, r.Combined(), "json or csv") + require.Contains(t, r.Combined(), `--format "xml" is not supported`) + // The refusal has to name jsonl too, or it repeats the bug where the + // guard advertised a narrower set than the exporter can write. + require.Contains(t, r.Combined(), "use csv, json or jsonl") + }) + + t.Run("jsonl is accepted, not just advertised", func(t *testing.T) { + path := filepath.Join(t.TempDir(), "results.jsonl") + requireSuccess(t, run(t, "run", "output", "export", f.FirstRunID, + "--eval", f.EvalID, "--format", "jsonl", "--output-file", path)) + + body, err := os.ReadFile(path) + require.NoError(t, err) + lines := strings.Split(strings.TrimSpace(string(body)), "\n") + require.NotEmpty(t, lines) + // Every line has to stand alone as an object, otherwise it is JSON + // wearing a .jsonl name. + for _, line := range lines { + var row map[string]any + require.NoError(t, json.Unmarshal([]byte(line), &row)) + } }) } From bf0176f7a4d9a688a578c02132d67d5550ee499e Mon Sep 17 00:00:00 2001 From: mohessie Date: Mon, 10 Aug 2026 05:16:32 +0300 Subject: [PATCH 157/320] Match the spec's run output renderings, and review findings `run output show` was the last `show` emitting raw JSON whatever was asked for, which made the command a person reaches for after a failing listing the hardest one in the CLI to read. It now renders a detail view and keeps the document behind `-o json`, the same split `evaluator show` already uses. Results are grouped by evaluator because a rubric reports one result per dimension, all carrying the evaluator's name; printed flat they read as several evaluators that happen to share one. Verifying this against the live service turned up that it also echoes the evaluator's name in `metric` for a single-score evaluator, so the flat form is used unless a result names a dimension distinct from its evaluator. The listing truncates the reason to a cell, so this view prints it whole. Two more renderings now match the spec: - The overall pass rate prints the bare fraction, `93.3% (14/15)`. Spelling out "samples passed every evaluator" on every run read as a caveat on the number rather than a definition of it; that belongs in the doc comment, where it now is. The arithmetic is unchanged. - `run output list` heads its last column `REASON`, not `REASON (first failure)`. Review findings, both from a pass over the recent changes: - `renderOutputItem` read the item before checking it was there, so an empty service response would have panicked rather than reported. - `evaluator show --output-file` used os.WriteFile, which truncates before writing. The spec points that flag at a definition the developer is still working with, so a failed write would have destroyed the local copy it was meant to update. It now writes through a temporary file and renames. Writing the test for that last one found a bug in the fix: removing the destination before renaming happily removed a *directory* at that path. It now refuses anything that is not a regular file. --- .../internal/cmd/evaluator.go | 3 +- .../internal/cmd/output.go | 45 ++++++++++ .../internal/cmd/output_test.go | 42 +++++++++ .../internal/cmd/run_output.go | 86 ++++++++++++++++++- .../internal/cmd/run_render_test.go | 84 +++++++++++++++++- .../internal/messages/messages.go | 37 +++++++- .../tests/cli/run_output_test.go | 2 +- 7 files changed, 292 insertions(+), 7 deletions(-) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator.go index 6943ea95ab8..b622f367eef 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator.go @@ -8,7 +8,6 @@ import ( "encoding/json" "fmt" "io" - "os" "strconv" "strings" @@ -376,7 +375,7 @@ func newEvaluatorShowCommand() *cobra.Command { body = append(indented, '\n') } } - if err := os.WriteFile(outFile, body, 0o600); err != nil { + if err := writeFileAtomic(outFile, body); err != nil { return messages.Creating(outFile, err) } if !isJSON(cmd) { diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/output.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/output.go index 46e7505acf0..c90a5c3ecdc 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/output.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/output.go @@ -5,8 +5,11 @@ package cmd import ( "encoding/json" + "errors" "fmt" "io" + "os" + "path/filepath" "strings" "text/tabwriter" @@ -147,3 +150,45 @@ func emitDetail(w io.Writer, fields []field) error { func requireFlag(name string) error { return messages.FlagRequired(name) } + +// writeFileAtomic replaces a file's contents in one step. +// +// The caller is usually overwriting a definition the developer already has and +// wants to keep working with, so a half-written file is worse than no write at +// all: os.WriteFile truncates first, and a failure after that leaves the good +// local copy destroyed. +func writeFileAtomic(path string, body []byte) error { + // The rename below needs the destination gone on Windows, so refuse + // anything that is not a regular file rather than removing it: pointed at a + // directory, this would otherwise delete it. + switch info, err := os.Stat(path); { + case err == nil && !info.Mode().IsRegular(): + return messages.NotARegularFile(path) + case err != nil && !errors.Is(err, os.ErrNotExist): + return err + } + + dir := filepath.Dir(path) + tmp, err := os.CreateTemp(dir, ".azd-eval-*") + if err != nil { + return err + } + tmpName := tmp.Name() + defer os.Remove(tmpName) + + if _, err := tmp.Write(body); err != nil { + tmp.Close() + return err + } + if err := tmp.Chmod(0o600); err != nil { + tmp.Close() + return err + } + if err := tmp.Close(); err != nil { + return err + } + if err := os.Remove(path); err != nil && !errors.Is(err, os.ErrNotExist) { + return err + } + return os.Rename(tmpName, path) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/output_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/output_test.go index 2914e5c3fbd..38b9e3824ac 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/output_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/output_test.go @@ -5,6 +5,8 @@ package cmd import ( "bytes" + "os" + "path/filepath" "testing" "github.com/stretchr/testify/assert" @@ -29,3 +31,43 @@ func TestEmitJSONList_NilBecomesEmptyArray(t *testing.T) { require.NoError(t, emitJSONList(&buf, none)) assert.Equal(t, "[]\n", buf.String()) } + +// `evaluator show --output-file` is pointed at a definition the developer is +// still working with, so a write that cannot complete must leave the old one +// intact rather than truncate it. +func TestWriteFileAtomic(t *testing.T) { + t.Run("replaces an existing file", func(t *testing.T) { + path := filepath.Join(t.TempDir(), "evaluator.json") + require.NoError(t, os.WriteFile(path, []byte("old"), 0o600)) + require.NoError(t, writeFileAtomic(path, []byte("new"))) + + body, err := os.ReadFile(path) + require.NoError(t, err) + assert.Equal(t, "new", string(body)) + }) + + t.Run("creates a file that was not there", func(t *testing.T) { + path := filepath.Join(t.TempDir(), "evaluator.json") + require.NoError(t, writeFileAtomic(path, []byte("new"))) + + body, err := os.ReadFile(path) + require.NoError(t, err) + assert.Equal(t, "new", string(body)) + }) + + t.Run("a directory is refused, not removed", func(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "evaluator.json") + require.NoError(t, os.Mkdir(path, 0o750)) + + require.Error(t, writeFileAtomic(path, []byte("new"))) + + info, err := os.Stat(path) + require.NoError(t, err, "the directory must survive") + assert.True(t, info.IsDir()) + + entries, err := os.ReadDir(dir) + require.NoError(t, err) + assert.Len(t, entries, 1, "no temporary file may be left behind") + }) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output.go index 26ca0bd3f5b..7e349295f76 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output.go @@ -150,7 +150,10 @@ func newRunOutputShowCommand() *cobra.Command { } return messages.ReadingOutputItem(itemID, err) } - return emitJSON(cmd.OutOrStdout(), item) + if isJSON(cmd) { + return emitJSON(cmd.OutOrStdout(), item) + } + return renderOutputItem(cmd.OutOrStdout(), item) }, } @@ -324,6 +327,83 @@ func (ec *evalContext) latestOrNamedRun( return &list.Data[0], nil } +// renderOutputItem is the detail view for one evaluated row. +// +// This was the one `show` that emitted raw JSON whatever was asked for, which +// made the command a person reaches for after a failing listing the hardest one +// to read. The listing truncates the reason to a cell; this is where the whole +// of it lives, so the reasons are printed in full rather than wrapped or cut. +// +// Results are grouped by evaluator: a rubric reports one result per dimension, +// all carrying the evaluator's name, and printing them flat would read as +// several evaluators that happen to share a name. +func renderOutputItem(w io.Writer, item *eval_api.OutputItem) error { + if item == nil { + return messages.OutputItemEmpty() + } + if err := emitDetail(w, []field{ + {"Item", item.ID}, + {"Run", item.RunID}, + {"Status", item.Status}, + }); err != nil { + return err + } + + order := make([]string, 0, len(item.Results)) + byName := make(map[string][]eval_api.OutputResult, len(item.Results)) + for _, r := range item.Results { + if _, seen := byName[r.Name]; !seen { + order = append(order, r.Name) + } + byName[r.Name] = append(byName[r.Name], r) + } + + for _, name := range order { + results := byName[name] + fmt.Fprintln(w) + + // The service repeats the evaluator's name in `metric` for a + // single-score evaluator, so a group is only worth nesting when its + // results name dimensions of their own. + if len(results) == 1 && (results[0].Metric == "" || results[0].Metric == name) { + r := results[0] + fmt.Fprint(w, messages.OutputItemVerdict( + name, formatScore(r.Score), verdictWord(r.Passed))) + if r.Reason != "" { + fmt.Fprint(w, messages.OutputItemReason(r.Reason)) + } + continue + } + + fmt.Fprint(w, messages.OutputItemEvaluator(name)) + for _, r := range results { + label := r.Metric + if label == "" { + label = r.Name + } + fmt.Fprint(w, messages.OutputItemMetric( + label, formatScore(r.Score), verdictWord(r.Passed))) + if r.Reason != "" { + fmt.Fprint(w, messages.OutputItemReason(r.Reason)) + } + } + } + return nil +} + +// verdictWord spells a boolean the way the rest of the output does. +func verdictWord(passed bool) string { + if passed { + return "pass" + } + return "fail" +} + +// formatScore prints a judge's score at the two decimals the scale carries. +func formatScore(score eval_api.LenientFloat) string { + return strconv.FormatFloat(float64(score), 'f', 2, 64) +} + func renderResults( w io.Writer, run *eval_api.OpenAIEvalRun, @@ -395,8 +475,10 @@ func renderResults( truncate(reason, 44), }) } + // Only the first failure's reason fits a cell; `run output show` has + // the rest. if err := emitTable(w, - []string{"ITEM", "SAMPLE", "FAILED EVALUATORS", "REASON (first failure)"}, + []string{"ITEM", "SAMPLE", "FAILED EVALUATORS", "REASON"}, rows); err != nil { return err } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_render_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_render_test.go index cce1c662156..6dcf14c3c9d 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_render_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_render_test.go @@ -46,7 +46,7 @@ func TestRenderResultsIsOneRowPerSample(t *testing.T) { assert.Equal(t, 1, strings.Count(text, "oi_2"), "a sample that failed two evaluators must still be one row:\n%s", text) - for _, header := range []string{"ITEM", "SAMPLE", "FAILED EVALUATORS", "REASON (first failure)"} { + for _, header := range []string{"ITEM", "SAMPLE", "FAILED EVALUATORS", "REASON"} { assert.Containsf(t, text, header, "the listing lost its %s column", header) } } @@ -143,3 +143,85 @@ func TestRenderRunOmitsTheScoreColumnWithoutMeans(t *testing.T) { assert.Contains(t, with.String(), "MEAN SCORE") assert.Contains(t, with.String(), "4.2", "the mean is shown to one decimal") } + +// `run output show` is what a person opens after a failing listing, so it has +// to be readable. It used to emit raw JSON whatever was asked for. +func TestRenderOutputItemIsNotJSON(t *testing.T) { + var out bytes.Buffer + require.NoError(t, renderOutputItem(&out, &eval_api.OutputItem{ + ID: "oi_01JQZY7K3R", + RunID: "evalrun_1", + Status: "fail", + Results: []eval_api.OutputResult{{ + Name: "builtin.task_adherence", + Score: 0.35, + Passed: false, + Reason: "Task abandoned after the first clarifying question.", + }}, + })) + + text := out.String() + assert.NotContains(t, text, `"results"`, "the detail view must not be JSON") + for _, want := range []string{ + "Item", "oi_01JQZY7K3R", + "Status", "fail", + "builtin.task_adherence", "0.35", + "Task abandoned after the first clarifying question.", + } { + assert.Contains(t, text, want) + } +} + +// The listing truncates the reason to a cell, so this view exists to carry the +// whole of it. Cutting it here would leave it readable nowhere. +func TestRenderOutputItemKeepsTheWholeReason(t *testing.T) { + reason := strings.Repeat("a reason that runs well past any column width. ", 8) + + var out bytes.Buffer + require.NoError(t, renderOutputItem(&out, &eval_api.OutputItem{ + ID: "oi_1", + Status: "fail", + Results: []eval_api.OutputResult{{Name: "relevance", Passed: false, Reason: reason}}, + })) + + assert.Contains(t, out.String(), reason) +} + +// A rubric reports one result per dimension, all carrying the evaluator's +// name. Printed flat they read as several evaluators that share a name. +func TestRenderOutputItemGroupsARubricsDimensions(t *testing.T) { + var out bytes.Buffer + require.NoError(t, renderOutputItem(&out, &eval_api.OutputItem{ + ID: "oi_1", + Status: "fail", + Results: []eval_api.OutputResult{ + {Name: "support-agent-quality", Metric: "resolves_issue", Score: 1, Passed: false}, + {Name: "support-agent-quality", Metric: "cites_policy", Score: 5, Passed: true}, + {Name: "builtin.task_adherence", Score: 0.35, Passed: false}, + }, + })) + + text := out.String() + assert.Equal(t, 1, strings.Count(text, "support-agent-quality"), + "the evaluator is named once, above its dimensions:\n%s", text) + for _, want := range []string{"resolves_issue", "cites_policy", "builtin.task_adherence"} { + assert.Contains(t, text, want) + } +} + +// The service echoes the evaluator's name in `metric` for a single-score +// evaluator. Nesting that reads as a dimension that happens to share its +// evaluator's name. +func TestRenderOutputItemDoesNotNestASelfNamedMetric(t *testing.T) { + var out bytes.Buffer + require.NoError(t, renderOutputItem(&out, &eval_api.OutputItem{ + ID: "oi_1", + Status: "completed", + Results: []eval_api.OutputResult{ + {Name: "task_adherence", Metric: "task_adherence", Score: 1, Passed: true}, + }, + })) + + assert.Equal(t, 1, strings.Count(out.String(), "task_adherence"), + "the evaluator must be named once:\n%s", out.String()) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go b/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go index 96829bd28b6..b5cdb4ed163 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go @@ -163,8 +163,12 @@ func RunFinishedWithStatus(runID, status string) error { } // OverallPassRate reports the share of samples that passed every evaluator. +// +// The parenthetical is the bare fraction the spec prints. Spelling out "samples +// passed every evaluator" on every run reads as a caveat on the number rather +// than a definition of it; the doc comment above is where that belongs. func OverallPassRate(rate string, passed, total int) string { - return fmt.Sprintf("\nOverall pass rate: %s (%d/%d samples passed every evaluator)\n", + return fmt.Sprintf("\nOverall pass rate: %s (%d/%d)\n", rate, passed, total) } @@ -337,6 +341,37 @@ func SamplesFailedAtLeastOne(samples int) string { return fmt.Sprintf("\n%d sample(s) failed at least one evaluator.\n", samples) } +// OutputItemEmpty reports a row the service acknowledged but returned nothing +// for, which is a service fault rather than a missing item. +func OutputItemEmpty() error { + return errors.New("the service returned no content for this output item") +} + +// NotARegularFile reports an --output-file that names a directory or a device. +func NotARegularFile(path string) error { + return fmt.Errorf("%s is not a regular file, so it will not be overwritten", path) +} + +// OutputItemVerdict is one evaluator's line in `run output show`. +func OutputItemVerdict(evaluator, score, verdict string) string { + return fmt.Sprintf("%s %s %s\n", evaluator, score, verdict) +} + +// OutputItemEvaluator heads the dimensions of a rubric that scored per metric. +func OutputItemEvaluator(evaluator string) string { + return evaluator + "\n" +} + +// OutputItemMetric is one scored dimension under its evaluator. +func OutputItemMetric(metric, score, verdict string) string { + return fmt.Sprintf(" %s %s %s\n", metric, score, verdict) +} + +// OutputItemReason is the judge's explanation, indented under its verdict. +func OutputItemReason(reason string) string { + return fmt.Sprintf(" %s\n", reason) +} + // ReportLinkAfterRows closes a per-sample listing with the service's report. func ReportLinkAfterRows(url string) string { return fmt.Sprintf("\nReport: %s\n", url) diff --git a/cli/azd/extensions/azure.ai.evaluations/tests/cli/run_output_test.go b/cli/azd/extensions/azure.ai.evaluations/tests/cli/run_output_test.go index 02163578a2d..2ba5ab5264b 100644 --- a/cli/azd/extensions/azure.ai.evaluations/tests/cli/run_output_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/tests/cli/run_output_test.go @@ -66,7 +66,7 @@ func TestCLIResultsShowRendersTheRows(t *testing.T) { // One row per evaluated sample, which is what makes "how many should I go // and look at" answerable by counting lines. - for _, header := range []string{"ITEM", "SAMPLE", "FAILED EVALUATORS", "REASON (first failure)"} { + for _, header := range []string{"ITEM", "SAMPLE", "FAILED EVALUATORS", "REASON"} { require.Containsf(t, r.Stdout, header, "the listing lost its %s column", header) } require.NotContains(t, r.Stdout, "EVALUATOR ", From e67f0653a9b5e060aabe37c82e68fad34c567152 Mon Sep 17 00:00:00 2001 From: mohessie Date: Mon, 10 Aug 2026 05:50:43 +0300 Subject: [PATCH 158/320] Answer three not-found cases as answers, not as service noise MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Found by publishing both extensions to the local registry and running the CLI off its happy path. `evaluator versions list ` printed a twelve-line HTTP 404 body, complete with the request URL and the raw error envelope, to say one thing: there is no such evaluator. It now says that, using the same guard `evaluator show` already had. `--output-file` pointed into a directory that does not exist named the temporary file the writer had chosen, which the caller never asked for and cannot act on. writeFileAtomic now owns every error it returns and names only the path the caller passed; a missing directory is reported as missing. The job-group hint read "if it generated a evaluator". Reworded so the kind never follows an article. Not changed, deliberately: `dataset versions list` on an unknown name still lists nothing and exits 0, while the dataset extension answers the same question with an error. I changed the eval side to match and it broke two tests whose names are the reasoning — "an unknown name lists nothing rather than failing" and "deleting an unregistered dataset is idempotent, not an error", the second because delete idempotence is verified by listing what is left. A list is a filter, not a lookup: empty and successful is right, and `-o json` callers range over the array. Which extension should move is a product call, so it is raised rather than settled here. --- .../internal/cmd/dataset.go | 3 +++ .../internal/cmd/evaluator.go | 7 ++++++- .../internal/cmd/output.go | 20 ++++++++++++------- .../internal/messages/messages.go | 18 +++++++++++++++-- 4 files changed, 38 insertions(+), 10 deletions(-) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset.go index 6c9ae110b4d..93ca2b65db0 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset.go @@ -208,6 +208,9 @@ func newDatasetVersionsListCommand() *cobra.Command { if err != nil { return messages.ListingDatasetVersions(name, err) } + // An unknown name lists nothing and succeeds; it is not an error. + // `-o json` callers range over the array, and `dataset delete` is + // checked for idempotence by listing what is left. return renderDatasets(cmd, list) }, } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator.go index b622f367eef..39b3330aea5 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator.go @@ -272,6 +272,11 @@ func newEvaluatorVersionsListCommand() *cobra.Command { list, err := ec.evalClient.ListEvaluatorVersions(ctx, name, ProjectEndpointAPIVersion) if err != nil { + // A name nobody published is the ordinary way to get here, and + // it does not need the whole 404 body to explain it. + if eval_api.IsNotFound(err) { + return messages.EvaluatorNotFound(name) + } return messages.ListingEvaluatorVersions(name, err) } return renderEvaluatorVersions(cmd, list) @@ -376,7 +381,7 @@ func newEvaluatorShowCommand() *cobra.Command { } } if err := writeFileAtomic(outFile, body); err != nil { - return messages.Creating(outFile, err) + return err } if !isJSON(cmd) { fmt.Fprint(cmd.OutOrStdout(), messages.WroteArtifact(outFile)) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/output.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/output.go index c90a5c3ecdc..dfa295440c4 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/output.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/output.go @@ -157,6 +157,9 @@ func requireFlag(name string) error { // wants to keep working with, so a half-written file is worse than no write at // all: os.WriteFile truncates first, and a failure after that leaves the good // local copy destroyed. +// +// Every error names the path the caller passed. The temporary file is this +// function's business and appears nowhere the caller asked for. func writeFileAtomic(path string, body []byte) error { // The rename below needs the destination gone on Windows, so refuse // anything that is not a regular file rather than removing it: pointed at a @@ -165,30 +168,33 @@ func writeFileAtomic(path string, body []byte) error { case err == nil && !info.Mode().IsRegular(): return messages.NotARegularFile(path) case err != nil && !errors.Is(err, os.ErrNotExist): - return err + return messages.Creating(path, err) } dir := filepath.Dir(path) tmp, err := os.CreateTemp(dir, ".azd-eval-*") if err != nil { - return err + return messages.CannotWriteInDirectory(dir, err) } tmpName := tmp.Name() defer os.Remove(tmpName) if _, err := tmp.Write(body); err != nil { tmp.Close() - return err + return messages.Creating(path, err) } if err := tmp.Chmod(0o600); err != nil { tmp.Close() - return err + return messages.Creating(path, err) } if err := tmp.Close(); err != nil { - return err + return messages.Creating(path, err) } if err := os.Remove(path); err != nil && !errors.Is(err, os.ErrNotExist) { - return err + return messages.Creating(path, err) + } + if err := os.Rename(tmpName, path); err != nil { + return messages.Creating(path, err) } - return os.Rename(tmpName, path) + return nil } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go b/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go index b5cdb4ed163..1e7fd21fc1f 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go @@ -19,6 +19,7 @@ package messages import ( "errors" "fmt" + "io/fs" "net/http" "path/filepath" "sort" @@ -352,6 +353,16 @@ func NotARegularFile(path string) error { return fmt.Errorf("%s is not a regular file, so it will not be overwritten", path) } +// CannotWriteInDirectory reports a destination directory that cannot be written +// to. A missing directory is reported as such: the wrapped error names the +// temporary file the writer chose, which the caller never asked for. +func CannotWriteInDirectory(dir string, err error) error { + if errors.Is(err, fs.ErrNotExist) { + return fmt.Errorf("%s does not exist", dir) + } + return fmt.Errorf("cannot write in %s: %w", dir, err) +} + // OutputItemVerdict is one evaluator's line in `run output show`. func OutputItemVerdict(evaluator, score, verdict string) string { return fmt.Sprintf("%s %s %s\n", evaluator, score, verdict) @@ -706,10 +717,13 @@ func JobDeleted(kind, jobID string) string { } // JobNotFound reports a job id that is not in this group, naming the other one. +// +// Phrased to avoid an article before the kind: "a evaluator" is what the +// obvious wording produces. func JobNotFound(kind, jobID, other string) error { return fmt.Errorf( - "no %s generation job %q in this project; if it generated a %s, "+ - "use the %s job group instead", kind, jobID, other, other) + "no %s generation job %q in this project; try the %s job group", + kind, jobID, other) } // ReadingJob reports a failure to read a generation job. From ec5161a60211d3c982aa5f5e35dd1da0be2d2aea Mon Sep 17 00:00:00 2001 From: mohessie Date: Mon, 10 Aug 2026 06:03:59 +0300 Subject: [PATCH 159/320] Report a downloaded artifact the way every other step reports The line read ' wrote ' while its neighbours, and the spec's own generate transcript, use '(1) Done: Downloaded '. Same marker now. --- .../azure.ai.evaluations/internal/messages/messages.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go b/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go index 1e7fd21fc1f..fd8f4629a50 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go @@ -666,7 +666,7 @@ func ReattachToJob(selector, jobID string) string { // WroteArtifact reports where a generated artifact landed. func WroteArtifact(path string) string { - return fmt.Sprintf(" wrote %s\n", path) + return fmt.Sprintf("%s Downloaded %s\n", DoneMark, path) } // ArtifactExists reports a generation that would overwrite a checked-in file. From 91d136aa3e27397f787df2c6b7475a785a0ddefe Mon Sep 17 00:00:00 2001 From: mohessie Date: Mon, 10 Aug 2026 06:24:09 +0300 Subject: [PATCH 160/320] Stop promising an exit code azd cannot deliver --fail-on read 'Exit 2 when the run misses this threshold'. A pipeline author reads that and writes a check on 2, which never fires: azd reports 1 for any extension failure regardless of what the extension exited with. The flag now says what the caller will observe. --target gained the default its neighbours all document. --- .../extensions/azure.ai.evaluations/internal/cmd/gating.go | 5 ++++- cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go | 3 ++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/gating.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/gating.go index db3eea7fa03..c49856b3fb1 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/gating.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/gating.go @@ -120,6 +120,9 @@ func applyGate(cmd *cobra.Command, g gate, run *eval_api.OpenAIEvalRun) { } func addFailOnFlag(cmd *cobra.Command, target *string) { + // The caveat is here because a pipeline author reads this line and then + // writes `if ($LASTEXITCODE -eq 2)`, which does not fire today. cmd.Flags().StringVar(target, "fail-on", "", - "Exit 2 when the run misses this threshold: any-failure, or pass-rate=<0..1>.") + "Fail when the run misses this threshold: any-failure, or pass-rate=<0..1>. "+ + "Exits 2, which azd currently reports as 1.") } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go index 058bd3dcddc..2b0c315a555 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go @@ -195,7 +195,8 @@ func newInitCommand() *cobra.Command { cmd.Flags().StringVar(&evalName, "name", "", "Name of the eval. Defaults to -eval, or -trace-eval under --source traces.") - cmd.Flags().StringVar(&target, "target", "", "Name of the agent to evaluate.") + cmd.Flags().StringVar(&target, "target", "", + "Name of the agent to evaluate. Detected when the project has one agent; prompts when it has several.") cmd.Flags().StringVar(&source, "source", "", "Where rows come from: dataset or traces. Defaults to dataset.") cmd.Flags().StringVar(&dataset, "dataset", "", From 8ed8354850f79dcd39293a420b245a962bd17b36 Mon Sep 17 00:00:00 2001 From: mohessie Date: Mon, 10 Aug 2026 07:08:00 +0300 Subject: [PATCH 161/320] Upload the dataset that was declared, not its neighbour MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review of the reconciliation path turned up three defects that share a shape: something is keyed or resolved by a lossy proxy for the thing itself. **The declared file was resolved to its directory before upload.** The upload helper then took whichever `.jsonl` sorted first. One file per dataset under `./evals` is the ordinary layout, so a project with two datasets registered one of them under the other's name — while the reconciler recorded the fingerprint of the file it *meant* to send. The two then agree on every subsequent deploy, so nothing ever reports drift and the eval scores rows nobody chose. A named file is now read as itself; a directory still scans, which is what pointing at a directory means. The same defect was in the dataset extension's `--from-file`, and a test there pinned it in place: *"a file resolves to the directory the upload scans"*. Fixing it surfaced a second inconsistency: this extension did not strip a UTF-8 BOM on the dataset upload path, though the dataset extension does. The same file registered differently depending on which command sent it. Both now share one `jsonlContent` guard. **Environment keys collided.** `FingerprintKey` mapped every character outside `[A-Z0-9]` to an underscore, so `quality-a`, `quality_a`, `quality a` and `qualityéa` were one key — and `versionKey` and `idKey` both derive from it. Two artifacts sharing a key overwrite each other's recorded fingerprint, version and id, which republishes both on every deploy and, per the comment already on `idKey`, can hand one eval another's id. The key now ends in a digest of the original name. `envkeys_test.go` asserted this collision as intended behaviour — while its own doc comment demanded "distinct, stable keys" and the test above it warned that a shared key makes "group A silently score group B's criteria". The assertion contradicted its documentation; it now matches it. **A generated name became a path component unchecked.** `--dataset-name ../../x` wrote outside the directory generation was pointed at, and `--force` would overwrite whatever was there. Checked at `generatedName`, the one place names are chosen. Only the filesystem's objections are enforced — the service polices its own character set, and duplicating it here would refuse names it accepts. --- .../internal/cmd/envkeys_test.go | 12 +++- .../internal/cmd/generate_composite.go | 34 ++++++++-- .../internal/cmd/generate_composite_test.go | 43 ++++++++++++ .../internal/cmd/reconciler.go | 8 +-- .../internal/messages/messages.go | 8 +++ .../pkg/dataset_api/jsonl_read_test.go | 67 +++++++++++++++++++ .../internal/pkg/dataset_api/models.go | 40 +++++++++-- .../internal/project/fingerprint_test.go | 41 +++++++++++- .../internal/project/service_target_eval.go | 12 +++- 9 files changed, 242 insertions(+), 23 deletions(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/jsonl_read_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/envkeys_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/envkeys_test.go index ccfe9719aa8..de109689e41 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/envkeys_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/envkeys_test.go @@ -28,11 +28,19 @@ func TestIDKey_IsPerName(t *testing.T) { // Names that are not valid env identifiers still have to produce distinct, // stable keys. +// +// The readable half of the key cannot tell "my group" from "my-group" — both +// sanitize to MY_GROUP. Letting them share a key is the collision the test +// above describes: the second declaration finds the first's id cached and +// scores the wrong group. func TestIDKey_NormalizesNames(t *testing.T) { - assert.Equal(t, idKey("eval", "my group"), idKey("eval", "my-group"), - "characters that cannot appear in an env name normalize the same way") + assert.NotEqual(t, idKey("eval", "my group"), idKey("eval", "my-group"), + "names that sanitize alike are still different names") assert.NotEqual(t, idKey("eval", "a"), idKey("dataset", "a"), "the kind keeps different resources apart") + + assert.Equal(t, idKey("eval", "my group"), idKey("eval", "my group"), + "the same name must key the same way on every deploy") } // The id and version keys for the same declaration must not collide. diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_composite.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_composite.go index 8b42d488d89..f2d1bf290cc 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_composite.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_composite.go @@ -6,6 +6,7 @@ package cmd import ( "bytes" "fmt" + "path/filepath" "strings" "sync" @@ -200,14 +201,37 @@ func buildGeneratePlans(req generateRequest) ([]generationPlan, error) { } // generatedName is the explicit name, or one derived from the target. +// +// The name becomes a filename as well as a service asset name, so it is +// checked here: `--dataset-name ../../x` would otherwise write outside the +// directory the caller pointed generation at, and `--force` would overwrite +// whatever is there. func generatedName(explicit, target, suffix string) (string, error) { - if explicit != "" { - return explicit, nil + name := explicit + if name == "" { + if target == "" { + return "", messages.GeneratedNameNeedsATarget(suffix) + } + name = target + "-" + suffix + } + if !nameIsAPathComponent(name) { + return "", messages.GeneratedNameNotAFileName(suffix, name) + } + return name, nil +} + +// nameIsAPathComponent reports whether a name stays where it is put. +// +// Only the filesystem's objections are checked. The service enforces its own +// character set, and duplicating it here would refuse names it accepts. +func nameIsAPathComponent(name string) bool { + if name == "" || name == "." || name == ".." { + return false } - if target == "" { - return "", messages.GeneratedNameNeedsATarget(suffix) + if strings.ContainsAny(name, `/\:`) || filepath.IsAbs(name) { + return false } - return target + "-" + suffix, nil + return true } type generationOutcome struct { diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_composite_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_composite_test.go index bae850aba24..5ea3ec5ae42 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_composite_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_composite_test.go @@ -93,3 +93,46 @@ func TestBuildGeneratePlans_NarrowedToOne(t *testing.T) { require.Len(t, plans, 1) assert.Equal(t, generateKindDataset, plans[0].Kind) } + +// The name becomes a filename, so one carrying a separator would write outside +// the directory generation was pointed at, and --force would overwrite it. +func TestGeneratedName_RefusesANameThatWouldLeaveTheDirectory(t *testing.T) { + escapes := []string{ + "../outside", + "..\\outside", + "sub/dir", + "sub\\dir", + "..", + ".", + "C:\\Windows\\System32\\drivers\\etc\\hosts", + "/etc/passwd", + } + + for _, name := range escapes { + t.Run(name, func(t *testing.T) { + _, err := generatedName(name, "support-agent", "dataset") + + require.Errorf(t, err, "%q must not be accepted as a file name", name) + assert.Contains(t, err.Error(), "file name") + }) + } +} + +// The service decides its own character set. Refusing everything it might +// accept would block names that work. +func TestGeneratedName_AllowsOrdinaryNames(t *testing.T) { + for _, name := range []string{ + "golden", + "support-agent-dataset", + "support_agent.v2", + "caf\u00e9-dataset", + "dataset 2", + } { + t.Run(name, func(t *testing.T) { + got, err := generatedName(name, "support-agent", "dataset") + + require.NoError(t, err) + assert.Equal(t, name, got) + }) + } +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go index e98cbc15987..f369df4ff9c 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go @@ -8,7 +8,6 @@ import ( "context" "encoding/json" "os" - "path/filepath" "reflect" "strconv" "strings" @@ -97,11 +96,10 @@ func (r *evalReconciler) EnsureDataset( } } - // The upload helper scans a directory for the first .jsonl. + // Uploaded by the path the author declared. Collapsing a file to its + // directory would upload whichever .jsonl sorts first, while the + // fingerprint below still describes the declared one. dir := localPath - if info, err := os.Stat(localPath); err == nil && !info.IsDir() { - dir = filepath.Dir(localPath) - } // A declared version is the version to publish, not one to count from. // Reaching here means the content differs from what that version holds, so diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go b/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go index fd8f4629a50..f03f2958d90 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go @@ -342,6 +342,14 @@ func SamplesFailedAtLeastOne(samples int) string { return fmt.Sprintf("\n%d sample(s) failed at least one evaluator.\n", samples) } +// GeneratedNameNotAFileName reports a generated artifact name that would not +// stay inside the output directory. +func GeneratedNameNotAFileName(kind, name string) error { + return fmt.Errorf( + "%s name %q cannot be used as a file name: remove any of / \\ : and do not use . or ..", + kind, name) +} + // OutputItemEmpty reports a row the service acknowledged but returned nothing // for, which is a service fault rather than a missing item. func OutputItemEmpty() error { diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/jsonl_read_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/jsonl_read_test.go new file mode 100644 index 00000000000..5e1e80a48b0 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/jsonl_read_test.go @@ -0,0 +1,67 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package dataset_api + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// One .jsonl per dataset in one folder is the ordinary layout under ./evals. +// Scanning the directory instead of reading the declared file registers the +// rows of whichever sorts first under the other one's name, while the +// reconciler records the fingerprint of the declared file — so the two agree +// forever and the eval scores data nobody chose. +func TestReadFirstJSONLFile_ReadsTheNamedFileNotItsNeighbour(t *testing.T) { + dir := t.TempDir() + named := filepath.Join(dir, "zebra.jsonl") + require.NoError(t, os.WriteFile(named, []byte("{\"pick\":\"me\"}\n"), 0o600)) + require.NoError(t, os.WriteFile( + filepath.Join(dir, "alpha.jsonl"), []byte("{\"pick\":\"not me\"}\n"), 0o600)) + + content, err := ReadFirstJSONLFile(named) + require.NoError(t, err) + assert.Contains(t, content, `"me"`) + assert.NotContains(t, content, "not me") + + content, err = ReadFirstJSONLFile(dir) + require.NoError(t, err) + assert.Contains(t, content, "not me", "the directory form takes the first .jsonl") +} + +// The dataset extension strips the BOM before upload; this path uploads the +// same rows under `azd up` and has to agree, or the same file registers +// differently depending on which command sent it. +func TestReadFirstJSONLFile_StripsTheByteOrderMark(t *testing.T) { + dir := t.TempDir() + body := append([]byte{0xEF, 0xBB, 0xBF}, []byte("{\"query\":\"q\"}\n")...) + + named := filepath.Join(dir, "d.jsonl") + require.NoError(t, os.WriteFile(named, body, 0o600)) + + for _, path := range []string{named, dir} { + content, err := ReadFirstJSONLFile(path) + require.NoError(t, err) + assert.Truef(t, strings.HasPrefix(content, `{"query"`), + "the first row has to start with its own first key, got %q", content) + } +} + +// A file holding only a BOM has no rows, and registering it succeeds — the +// failure would surface at the run that scores it instead. +func TestReadFirstJSONLFile_RefusesAnEmptyFile(t *testing.T) { + dir := t.TempDir() + named := filepath.Join(dir, "d.jsonl") + require.NoError(t, os.WriteFile(named, []byte{0xEF, 0xBB, 0xBF}, 0o600)) + + _, err := ReadFirstJSONLFile(named) + + require.Error(t, err) + assert.Contains(t, err.Error(), "no rows") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/models.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/models.go index c579bca1490..3cfbd1dba4a 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/models.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/models.go @@ -4,6 +4,7 @@ package dataset_api import ( + "bytes" "math" "os" "path/filepath" @@ -189,12 +190,27 @@ func NextVersion(current string) string { return current + ".1" } -// ReadFirstJSONLFile finds and reads the first .jsonl file in a directory. +// ReadFirstJSONLFile reads the rows to upload from a .jsonl file, or from the +// first .jsonl in a directory. +// +// A file path is read as itself. Resolving it to its directory and scanning +// would upload whichever .jsonl sorts first, so a project with one file per +// dataset would register the wrong rows under a name while recording the +// fingerprint of the declared file — the two would then agree forever. // // An empty file is refused here rather than uploaded: registering it succeeds, // and the failure then surfaces at the run that tries to score it, which is a // long way from the command that caused it. -func ReadFirstJSONLFile(dir string) (string, error) { +func ReadFirstJSONLFile(path string) (string, error) { + if info, err := os.Stat(path); err == nil && !info.IsDir() { + data, err := os.ReadFile(path) //nolint:gosec // local artifact path + if err != nil { + return "", messages.ReadingPath(path, err) + } + return jsonlContent(filepath.Base(path), data) + } + + dir := path entries, err := os.ReadDir(dir) if err != nil { return "", messages.ReadingDatasetDirectory(err) @@ -208,11 +224,23 @@ func ReadFirstJSONLFile(dir string) (string, error) { if err != nil { return "", messages.ReadingPath(e.Name(), err) } - if strings.TrimSpace(string(data)) == "" { - return "", messages.DatasetFileHasNoRows(e.Name()) - } - return string(data), nil + return jsonlContent(e.Name(), data) } } return "", messages.NoJSONLInDirectory(dir) } + +// utf8BOM is what Windows editors and PowerShell's Set-Content write ahead of +// otherwise valid UTF-8. +var utf8BOM = []byte{0xEF, 0xBB, 0xBF} + +// jsonlContent prepares one file's bytes for upload. +func jsonlContent(name string, data []byte) (string, error) { + // Uploaded as-is a BOM becomes part of the first row's first key, so every + // consumer of the dataset sees one malformed record. + data = bytes.TrimPrefix(data, utf8BOM) + if strings.TrimSpace(string(data)) == "" { + return "", messages.DatasetFileHasNoRows(name) + } + return string(data), nil +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/fingerprint_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/fingerprint_test.go index 2cf1fe8e889..1eb481bf18c 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/fingerprint_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/fingerprint_test.go @@ -54,7 +54,7 @@ func TestFingerprint_MissingFileNamesIt(t *testing.T) { // every deploy. func TestFingerprintKey_IsAValidEnvironmentKey(t *testing.T) { tests := []struct { - kind, name, want string + kind, name, readable string }{ {"dataset", "support-regression", "DATASET_SUPPORT_REGRESSION"}, {"evaluator", "quality.v2", "EVALUATOR_QUALITY_V2"}, @@ -65,10 +65,12 @@ func TestFingerprintKey_IsAValidEnvironmentKey(t *testing.T) { } for _, tt := range tests { - t.Run(tt.want, func(t *testing.T) { + t.Run(tt.readable, func(t *testing.T) { key := FingerprintKey(tt.kind, tt.name) - assert.Equal(t, EnvKeyFingerprintPrefix+tt.want, key) + assert.True(t, + strings.HasPrefix(key, EnvKeyFingerprintPrefix+tt.readable+"_"), + "the key stays readable: %q", key) for _, r := range strings.TrimPrefix(key, EnvKeyFingerprintPrefix) { assert.Truef(t, (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9') || r == '_', @@ -85,3 +87,36 @@ func TestFingerprintKey_KindSeparatesTheNamespaces(t *testing.T) { FingerprintKey("dataset", "quality"), FingerprintKey("evaluator", "quality")) } + +// The readable half of the key maps every character outside [A-Z0-9] to an +// underscore, so these names are indistinguishable in it. Sharing a key means +// sharing a recorded fingerprint, version and id: both artifacts then look +// changed on every deploy and republish forever. +func TestFingerprintKey_NamesThatSanitizeAlikeStillDiffer(t *testing.T) { + collidingNames := []string{ + "quality-a", + "quality_a", + "quality a", + "quality.a", + "quality/a", + "quality\u00e9a", + "qualityXa", + } + + seen := make(map[string]string, len(collidingNames)) + for _, name := range collidingNames { + key := FingerprintKey("evaluator", name) + if previous, clash := seen[key]; clash { + t.Fatalf("%q and %q share the key %q", previous, name, key) + } + seen[key] = name + } +} + +// The digest covers the kind and the name separately, so moving a character +// across the boundary is not the same artifact. +func TestFingerprintKey_TheKindBoundaryIsNotAmbiguous(t *testing.T) { + assert.NotEqual(t, + FingerprintKey("dataset_a", "b"), + FingerprintKey("dataset", "a_b")) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go index 82ee8406ade..d04144ec5ac 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go @@ -344,8 +344,14 @@ func FingerprintGroup(group Eval) (string, error) { } // FingerprintKey is the azd environment key holding an artifact's fingerprint. +// +// The readable half is lossy: everything outside [A-Z0-9] becomes an +// underscore, so `quality-a`, `quality_a` and `quality a` all sanitize alike, +// as does any pair of names differing only outside ASCII. Two artifacts sharing +// a key overwrite each other's recorded fingerprint, version and id, which +// makes every deploy republish both. The trailing digest keeps them apart. func FingerprintKey(kind, name string) string { - safe := strings.Map(func(r rune) rune { + readable := strings.Map(func(r rune) rune { switch { case r >= 'A' && r <= 'Z', r >= '0' && r <= '9': return r @@ -355,5 +361,7 @@ func FingerprintKey(kind, name string) string { return '_' } }, kind+"_"+name) - return EnvKeyFingerprintPrefix + safe + + sum := sha256.Sum256([]byte(kind + "\x00" + name)) + return EnvKeyFingerprintPrefix + readable + "_" + strings.ToUpper(hex.EncodeToString(sum[:4])) } From 6a27a74932a5c6e52b4dbf40f822ecc1a7992f6b Mon Sep 17 00:00:00 2001 From: mohessie Date: Mon, 10 Aug 2026 07:36:14 +0300 Subject: [PATCH 162/320] Notice a rubric field the author deleted MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `azd up` decided whether to publish an evaluator by comparing the local file to the one on the service. That comparison walks the authored keys and looks for each one on the service — it has to, because the service enriches a stored definition with `data_schema`, `init_parameters` and `metrics` that nobody wrote, and comparing whole documents would republish on every deploy. A key the author *deleted* is not among the authored keys. Its survival on the service therefore goes unnoticed, the two definitions are called equal, and nothing publishes. Deleting a `pass_threshold` — Scenario 4's edit in reverse — left the old threshold grading every subsequent run, with `azd up` reporting the evaluator as unchanged. Raising a threshold always worked, which is presumably why this went unseen. No smarter comparison fixes it: against a service that adds fields of its own, "the author removed this" and "the service added this" are the same observation. Telling them apart needs history. So the evaluator now records a digest of the author's file, which is what datasets have always done a few lines above. When the digest differs from the last deploy the skip path is not taken, whatever the comparison thinks. `sameDefinition` stays as the first-deploy fallback, for the case where no digest has been recorded yet. The test for the blind spot asserts it as a blind spot rather than as a property, so the next reader does not mistake it for intent. --- .../internal/cmd/reconciler.go | 17 ++++++++++++++++- .../internal/cmd/reconciler_test.go | 18 ++++++++++++++++++ 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go index f369df4ff9c..d8697acc36d 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go @@ -246,19 +246,33 @@ func (r *evalReconciler) EnsureEvaluator( return "", false, messages.EvaluatorProblem(decl.Name, err) } + // The author's own file decides whether there is anything to publish. + // Comparing against the service cannot: it enriches a definition with + // fields nobody authored, so sameDefinition only looks for authored keys on + // the service and a key the author *deleted* — a pass_threshold, say — is + // still there to be found, and the deletion never publishes. + digest, err := project.Fingerprint(localPath) + if err != nil { + return "", false, messages.EvaluatorSource(localPath, err) + } + digestKey := project.FingerprintKey("evaluator", decl.Name) + prior := r.ec.getEnvValue(ctx, digestKey) + authorEdited := prior != "" && prior != digest + // Compare against the definition already on the service. var known json.RawMessage if existing, err := r.ec.evalClient.GetEvaluatorRaw( ctx, decl.Name, "", ProjectEndpointAPIVersion, ); err == nil { remote := versionFromRaw(existing, "") - if sameDefinition(existing, body) { + if !authorEdited && sameDefinition(existing, body) { // Nothing to publish, but the version is still worth recording: // it is what a later deploy compares against to notice that // someone moved the evaluator on from here. if remote != "" { _ = r.ec.setEnvValue(ctx, versionKey("evaluator", decl.Name), remote) } + _ = r.ec.setEnvValue(ctx, digestKey, digest) return versionFromRaw(existing, decl.Version), false, nil } @@ -286,6 +300,7 @@ func (r *evalReconciler) EnsureEvaluator( } r.awaitEvaluatorReadable(ctx, decl.Name, created.Version) _ = r.ec.setEnvValue(ctx, versionKey("evaluator", decl.Name), created.Version) + _ = r.ec.setEnvValue(ctx, digestKey, digest) return created.Version, true, nil } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler_test.go index 8086fd96c43..54ced469be9 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler_test.go @@ -61,3 +61,21 @@ func TestSameDefinitionRejectsMalformed(t *testing.T) { require.False(t, sameDefinition(good, []byte(`not json`))) require.False(t, sameDefinition([]byte(`{"no":"definition"}`), good)) } + +// What sameDefinition cannot see, and why EnsureEvaluator digests the author's +// file instead of relying on it. +// +// The comparison walks the authored keys and looks for each on the service. A +// key the author *deleted* is not among them, so its survival on the service +// goes unnoticed and the definitions are called equal. Deleting a +// pass_threshold — the spec's own Scenario 4 edit, in reverse — would publish +// nothing and leave the old threshold grading every run. +func TestSameDefinitionCannotSeeARemovedField(t *testing.T) { + authored := []byte(`{"definition":{"type":"rubric","dimensions":[{"id":"a","weight":5}]}}`) + onService := []byte( + `{"definition":{"type":"rubric","pass_threshold":0.7,` + + `"dimensions":[{"id":"a","weight":5}]}}`) + + require.True(t, sameDefinition(onService, authored), + "this is the blind spot the digest exists to cover, not a property to rely on") +} From c82553ddbe38227998f536f3c738890c286833e3 Mon Sep 17 00:00:00 2001 From: mohessie Date: Mon, 10 Aug 2026 07:47:25 +0300 Subject: [PATCH 163/320] Read every page of a run's rows MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `ListOutputItems` made one request, and `OutputItemList` modelled only `data` — so the client could not observe whether the service had more to give. A run answered a page at a time would have had its first page reported as the whole run: `run output list` would show those rows, `--failed-only` would search only those rows, `export` would write only those rows, and the mean score per evaluator would be averaged over a sample of the run rather than the run. None of it would look wrong on screen. I could not confirm the envelope against the live service — the extension does not log response bodies, and no run in the test project is large enough to paginate. So the change is written to be correct either way: `has_more` and `last_id` are read, never required, and the cursor is followed only if the service actually sets one. A service that returns neither behaves exactly as before, which is the behaviour every existing test pins. Both paths are proved against a fake server rather than a live guess: follows the cursor, stops without one, honours `--limit` as a cap on rows across pages, and does not loop on a page that claims more but carries nothing. --- .../internal/pkg/eval_api/evaluators.go | 36 ++++- .../internal/pkg/eval_api/models.go | 5 + .../pkg/eval_api/output_items_paging_test.go | 134 ++++++++++++++++++ 3 files changed, 169 insertions(+), 6 deletions(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/output_items_paging_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/evaluators.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/evaluators.go index ae70d8b5273..006189c4d89 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/evaluators.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/evaluators.go @@ -243,16 +243,40 @@ func (c *EvalClient) ListOutputItems( evalID, runID string, limit int, ) (*OutputItemList, error) { - query := map[string]string{} - if limit > 0 { - query["limit"] = strconv.Itoa(limit) - } - path := fmt.Sprintf( "%s/%s/runs/%s/output_items", pathOpenAIEvals, url.PathEscape(evalID), url.PathEscape(runID), ) - return doRequestTyped[OutputItemList](c, ctx, http.MethodGet, path, query, nil, "") + + // Pages are followed only when the service says there are more. A run of + // 200 samples answered one page at a time would otherwise be reported as + // however many rows fit in the first, and the mean scores computed from + // them would be a sample of the run rather than the run. + all := &OutputItemList{} + after := "" + for { + query := map[string]string{} + if limit > 0 { + query["limit"] = strconv.Itoa(limit - len(all.Data)) + } + if after != "" { + query["after"] = after + } + + page, err := doRequestTyped[OutputItemList](c, ctx, http.MethodGet, path, query, nil, "") + if err != nil { + return nil, err + } + all.Data = append(all.Data, page.Data...) + + if !page.HasMore || page.LastID == "" || len(page.Data) == 0 { + return all, nil + } + if limit > 0 && len(all.Data) >= limit { + return all, nil + } + after = page.LastID + } } // GetOutputItem reads a single evaluated row. diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/models.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/models.go index 033fcff35ff..a5f13b905c1 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/models.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/models.go @@ -556,6 +556,11 @@ type OpenAIEvalRunList struct { // OutputItemList is a page of a run's per-sample results. type OutputItemList struct { Data []OutputItem `json:"data"` + // HasMore and LastID are the OpenAI list envelope's cursor. They are only + // read, never required: a service that returns neither yields one page, + // which is what this client did before it could see them at all. + HasMore bool `json:"has_more"` + LastID string `json:"last_id"` } // OutputItem is one evaluated row: the dataset item, and every evaluator's diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/output_items_paging_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/output_items_paging_test.go new file mode 100644 index 00000000000..1503c252539 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/output_items_paging_test.go @@ -0,0 +1,134 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package eval_api + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "sync/atomic" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// A run's rows are what `run output list`, `--failed-only` and `export` all +// read, and what the mean score per evaluator is averaged over. Stopping at +// the first page would report a sample of the run as the run, with nothing on +// screen to say rows were missing. +func TestListOutputItemsFollowsTheCursor(t *testing.T) { + var requests atomic.Int32 + + client := newRecordingClient(t, func(w http.ResponseWriter, r *http.Request) { + n := requests.Add(1) + after := r.URL.Query().Get("after") + + switch n { + case 1: + require.Empty(t, after, "the first page is not asked for by cursor") + writeJSON(t, w, map[string]any{ + "data": []map[string]any{ + {"id": "a", "status": "completed"}, + {"id": "b", "status": "completed"}, + }, + "has_more": true, + "last_id": "b", + }) + case 2: + require.Equal(t, "b", after, "the next page is asked for from the last id") + writeJSON(t, w, map[string]any{ + "data": []map[string]any{{"id": "c", "status": "completed"}}, + "has_more": false, + }) + default: + t.Fatalf("asked for a page after the service said there were none") + } + }) + + list, err := client.ListOutputItems(context.Background(), "eval_1", "run_1", 0) + + require.NoError(t, err) + require.Len(t, list.Data, 3, "every page's rows belong to the run") + assert.Equal(t, []string{"a", "b", "c"}, ids(list.Data)) + assert.EqualValues(t, 2, requests.Load()) +} + +// A service that answers one page and says nothing about more is the shape +// this client was written against, and must still work exactly as before. +func TestListOutputItemsStopsWithoutACursor(t *testing.T) { + var requests atomic.Int32 + + client := newRecordingClient(t, func(w http.ResponseWriter, r *http.Request) { + requests.Add(1) + writeJSON(t, w, map[string]any{ + "data": []map[string]any{{"id": "only", "status": "completed"}}, + }) + }) + + list, err := client.ListOutputItems(context.Background(), "eval_1", "run_1", 0) + + require.NoError(t, err) + require.Len(t, list.Data, 1) + assert.EqualValues(t, 1, requests.Load(), "one page, one request") +} + +// --limit is a cap on rows, not on requests: fetching past it would spend the +// caller's time on rows they said they did not want. +func TestListOutputItemsHonoursTheLimitAcrossPages(t *testing.T) { + var requests atomic.Int32 + + client := newRecordingClient(t, func(w http.ResponseWriter, r *http.Request) { + requests.Add(1) + writeJSON(t, w, map[string]any{ + "data": []map[string]any{ + {"id": fmt.Sprintf("row-%d", requests.Load()), "status": "completed"}, + }, + "has_more": true, + "last_id": fmt.Sprintf("row-%d", requests.Load()), + }) + }) + + list, err := client.ListOutputItems(context.Background(), "eval_1", "run_1", 2) + + require.NoError(t, err) + require.Len(t, list.Data, 2) + assert.EqualValues(t, 2, requests.Load(), "the cap stops the paging") +} + +// A page that claims more but carries nothing would otherwise loop forever. +func TestListOutputItemsStopsOnAnEmptyPage(t *testing.T) { + var requests atomic.Int32 + + client := newRecordingClient(t, func(w http.ResponseWriter, r *http.Request) { + requests.Add(1) + require.Less(t, requests.Load(), int32(5), "the client is looping") + writeJSON(t, w, map[string]any{ + "data": []map[string]any{}, + "has_more": true, + "last_id": "x", + }) + }) + + list, err := client.ListOutputItems(context.Background(), "eval_1", "run_1", 0) + + require.NoError(t, err) + assert.Empty(t, list.Data) + assert.EqualValues(t, 1, requests.Load()) +} + +func writeJSON(t *testing.T, w http.ResponseWriter, body any) { + t.Helper() + w.Header().Set("Content-Type", "application/json") + require.NoError(t, json.NewEncoder(w).Encode(body)) +} + +func ids(items []OutputItem) []string { + out := make([]string, 0, len(items)) + for _, i := range items { + out = append(out, i.ID) + } + return out +} From 13d94b06718c6b4ff6fc77f02676cfdc535d11a7 Mon Sep 17 00:00:00 2001 From: mohessie Date: Mon, 10 Aug 2026 07:58:51 +0300 Subject: [PATCH 164/320] Refuse a configuration key rather than read it as nothing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `azure.eval.yaml` is meant to be hand-edited — the spec's Scenario 4 opens it in an editor — and a typed YAML unmarshal ignores every key it does not recognise. Writing this under an eval: target: agent: support-agent instead of `type:` and `name:` parsed without complaint and produced a target with two empty strings. The run then failed several steps later saying the eval named no target, which is true but is not what went wrong. The configuration is now decoded with `KnownFields(true)`, so an unrecognised key is reported where it was written. An empty file is still an empty configuration rather than a parse failure: `generate` writes one before it has anything to record. Not addressed here: writing the configuration back marshals the typed struct, so comments and key order in a hand-edited file are lost whenever `init` or `generate` rewrite it. Preserving them means editing the YAML tree rather than round-tripping through the struct, which is a larger change than this one and is better made deliberately. The round-trip test that found this first asserted the wrong thing — it used `target: {agent: ...}` and appeared to show the rewrite destroying targets. It now uses the real shape and checks that an eval, its target and its datasets survive being written back. --- .../project/eval_config_roundtrip_test.go | 104 ++++++++++++++++++ .../internal/project/eval_config_store.go | 17 ++- 2 files changed, 120 insertions(+), 1 deletion(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_roundtrip_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_roundtrip_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_roundtrip_test.go new file mode 100644 index 00000000000..27cbcc1fe27 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_roundtrip_test.go @@ -0,0 +1,104 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package project + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// `init` and `generate` both read the configuration, add an entry and write the +// whole file back. Anything the round trip cannot carry is deleted from a file +// the developer wrote and is expected to keep editing. +func TestEvalConfigRoundTripKeepsWhatTheAuthorWrote(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, EvalConfigBase) + + authored := `evals: + - name: support-agent-eval + target: + type: agent + name: support-agent + dataset: golden + evaluators: + - evaluator: builtin.task_adherence +datasets: + - name: golden + source: ./datasets/golden.jsonl +` + require.NoError(t, os.WriteFile(path, []byte(authored), 0o600)) + + cfg, err := LoadEvalConfig(path) + require.NoError(t, err) + require.NoError(t, SaveEvalConfigTo(path, cfg)) + + reloaded, err := LoadEvalConfig(path) + require.NoError(t, err) + + require.Len(t, reloaded.Evals, 1) + assert.Equal(t, "support-agent-eval", reloaded.Evals[0].Name) + require.NotNil(t, reloaded.Evals[0].Target, "the target survived the rewrite") + assert.Equal(t, "agent", reloaded.Evals[0].Target.Type) + assert.Equal(t, "support-agent", reloaded.Evals[0].Target.Name) + require.Len(t, reloaded.Datasets, 1) + assert.Equal(t, "golden", reloaded.Datasets[0].Name) +} + +// A hand-edited configuration is the normal way to use this file, and a +// misspelled key used to be read as nothing at all: `agent: support-agent` +// under `target:` left an empty target, and the run failed later with a message +// about the target rather than about the typo that caused it. +func TestEvalConfigRefusesAKeyItDoesNotKnow(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, EvalConfigBase) + + require.NoError(t, os.WriteFile(path, []byte(`evals: + - name: support-agent-eval + target: + agent: support-agent +`), 0o600)) + + _, err := LoadEvalConfig(path) + + require.Error(t, err, "a key the extension does not know is a typo, not a no-op") + assert.Contains(t, err.Error(), "agent") +} + +// The keys the extension does know must still load, or strictness would break +// every configuration it writes itself. +func TestEvalConfigAcceptsEveryKeyItWrites(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, EvalConfigBase) + + require.NoError(t, os.WriteFile(path, []byte(`datasets: + - name: golden + source: ./datasets/golden.jsonl + version: "2" +evaluators: + - name: quality + source: ./evaluators/quality.json +evals: + - name: e + id: eval_1 + description: grades support answers + dataset: golden + evaluation_level: turn + max_samples: 15 + evaluators: + - evaluator: builtin.task_adherence + target: + type: agent + name: support-agent +`), 0o600)) + + cfg, err := LoadEvalConfig(path) + + require.NoError(t, err) + require.Len(t, cfg.Evals, 1) + assert.Equal(t, 15, cfg.Evals[0].MaxSamples) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_store.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_store.go index 0eac431d8d1..979587913bc 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_store.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_store.go @@ -4,7 +4,9 @@ package project import ( + "bytes" "errors" + "io" "io/fs" "os" "path/filepath" @@ -89,14 +91,27 @@ func OpenEvalConfig(evalDir string) (*EvalConfig, error) { // LoadEvalConfig reads a configuration from an explicit path. The path is used // verbatim, relative to the process working directory — never re-rooted. +// +// Decoded strictly: a key this extension does not know is a typo, and reading +// it as nothing leaves a configuration that looks fine and fails later +// somewhere else. `agent:` written under `target:` instead of `type:`/`name:` +// used to produce an empty target and a run that complained about the target. func LoadEvalConfig(path string) (*EvalConfig, error) { data, err := ReadFileNoBOM(path) if err != nil { return nil, messages.ReadingEvalConfig(path, err) } + decoder := yaml.NewDecoder(bytes.NewReader(data)) + decoder.KnownFields(true) + var cfg EvalConfig - if err := yaml.Unmarshal(data, &cfg); err != nil { + if err := decoder.Decode(&cfg); err != nil { + // An empty file is a configuration with nothing in it, not a parse + // failure: `generate` writes one before it has anything to record. + if errors.Is(err, io.EOF) { + return &cfg, nil + } return nil, messages.ParsingEvalConfig(path, err) } return &cfg, nil From 00a276032c1c9ea551ece9f8bd5a80bff948bbd1 Mon Sep 17 00:00:00 2001 From: mohessie Date: Mon, 10 Aug 2026 08:13:41 +0300 Subject: [PATCH 165/320] Recreate the eval whose artifact changed, not every eval in the file MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reconciliation accumulated one `anyChanged` flag across every dataset and every evaluator, then handed it to each eval in turn. Evals are immutable, so being told something changed means being recreated under a new id — and the run history hangs off the old one. A configuration with two evals therefore lost both histories whenever either one's dataset gained a row. That is the loop Scenario 3 exists for: change the agent, run again, see whether the results improved. Comparing this run to the last one stops working the moment an unrelated sibling is edited. Changes are now tracked per artifact, and an eval is recreated only when its own dataset or one of its own evaluators was republished. Built-ins are skipped — the service owns those, this configuration never publishes one, so they can never be the reason to discard an eval. --- .../internal/project/change_scope_test.go | 77 +++++++++++++++++++ .../internal/project/service_target_eval.go | 47 ++++++++++- 2 files changed, 120 insertions(+), 4 deletions(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/project/change_scope_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/change_scope_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/change_scope_test.go new file mode 100644 index 00000000000..bc825c5332c --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/change_scope_test.go @@ -0,0 +1,77 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package project + +import ( + "testing" + + "azureaieval/internal/pkg/evalcore" + + "github.com/stretchr/testify/assert" +) + +func evalNamed(name, dataset string, evaluators ...string) Eval { + refs := evalcore.EvaluatorList{} + for _, e := range evaluators { + refs = append(refs, evalcore.EvaluatorRef{Evaluator: e}) + } + return Eval{Name: name, Dataset: dataset, Evaluators: refs} +} + +// An eval is immutable, so recreating it abandons the id its runs are recorded +// against. Scenario 3 is a developer comparing this run to the last one under +// that id, and one flag for the whole file used to break that for every eval +// because one unrelated dataset gained a row. +func TestChangeSetOnlyReachesWhatAnEvalNames(t *testing.T) { + changed := changeSet{ + datasets: map[string]bool{"support-golden": true, "billing-golden": false}, + evaluators: map[string]bool{"support-quality": false, "billing-quality": false}, + } + + support := evalNamed("support-eval", "support-golden", + "builtin.task_adherence", "support-quality") + billing := evalNamed("billing-eval", "billing-golden", + "builtin.task_adherence", "billing-quality") + + assert.True(t, changed.reaches(support), "its own dataset was republished") + assert.False(t, changed.reaches(billing), + "a sibling's dataset changing is not a reason to discard this eval's history") +} + +func TestChangeSetReachesThroughAnEvaluator(t *testing.T) { + changed := changeSet{ + datasets: map[string]bool{"golden": false}, + evaluators: map[string]bool{"quality": true}, + } + + assert.True(t, + changed.reaches(evalNamed("e", "golden", "quality")), + "an evaluator this eval runs was republished") + assert.False(t, + changed.reaches(evalNamed("other", "golden", "builtin.task_adherence")), + "this eval does not run that evaluator") +} + +// Built-ins belong to the service. This configuration never publishes one, so +// a name that merely looks like a catalog entry must not recreate anything. +func TestChangeSetIgnoresBuiltins(t *testing.T) { + changed := changeSet{ + datasets: map[string]bool{}, + evaluators: map[string]bool{"builtin.task_adherence": true}, + } + + assert.False(t, changed.reaches(evalNamed("e", "golden", "builtin.task_adherence"))) +} + +// A configuration whose datasets and evaluators are all already registered +// publishes nothing, and must leave every eval where it is. +func TestChangeSetReachesNothingWhenNothingChanged(t *testing.T) { + changed := changeSet{ + datasets: map[string]bool{"golden": false}, + evaluators: map[string]bool{"quality": false}, + } + + assert.False(t, changed.reaches(evalNamed("e", "golden", "quality"))) + assert.False(t, changed.reaches(evalNamed("no-dataset", ""))) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go index d04144ec5ac..a98a12e4ea7 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go @@ -13,6 +13,7 @@ import ( "strings" "azureaieval/internal/messages" + "azureaieval/internal/pkg/evalcore" "github.com/azure/azure-dev/cli/azd/pkg/azdext" "github.com/azure/azure-dev/cli/azd/pkg/foundry" @@ -148,7 +149,15 @@ func (p *EvalServiceTargetProvider) Deploy( // 1. Datasets the configuration owns. Paths are kept so an eval that names // one can derive its columns without reading the blob back. - anyChanged := false + // + // What changed is tracked per artifact, not as one flag for the file. Evals + // are immutable, so recreating one discards the id its run history hangs + // off; a single flag would do that to every eval in the file because one + // unrelated dataset gained a row. + changedArtifacts := changeSet{ + datasets: map[string]bool{}, + evaluators: map[string]bool{}, + } datasetPaths := map[string]string{} for _, decl := range cfg.Datasets { if decl.Source == "" { @@ -161,7 +170,7 @@ func (p *EvalServiceTargetProvider) Deploy( if err != nil { return nil, messages.DatasetProblem(decl.Name, err) } - anyChanged = anyChanged || changed + changedArtifacts.datasets[decl.Name] = changed report(progress, describeResult("dataset", decl.Name, version, changed)) } @@ -174,7 +183,7 @@ func (p *EvalServiceTargetProvider) Deploy( if err != nil { return nil, messages.EvaluatorProblem(decl.Name, err) } - anyChanged = anyChanged || changed + changedArtifacts.evaluators[decl.Name] = changed report(progress, describeResult("evaluator", decl.Name, version, changed)) } @@ -183,7 +192,8 @@ func (p *EvalServiceTargetProvider) Deploy( for i := range cfg.Evals { eval := cfg.Evals[i] report(progress, messages.ReconcilingEval(eval.Name)) - id, err := reconciler.EnsureEval(ctx, eval, datasetPaths[eval.Dataset], anyChanged) + id, err := reconciler.EnsureEval( + ctx, eval, datasetPaths[eval.Dataset], changedArtifacts.reaches(eval)) if err != nil { return nil, messages.EvalProblem(eval.Name, err) } @@ -193,6 +203,35 @@ func (p *EvalServiceTargetProvider) Deploy( return &azdext.ServiceDeployResult{}, nil } +// changeSet records which artifacts this deploy republished. +type changeSet struct { + datasets map[string]bool + evaluators map[string]bool +} + +// reaches reports whether anything this eval is built from was republished. +// +// Only what the eval names counts. An eval is immutable, so recreating it +// abandons the id its runs are recorded against, and Scenario 3 is a developer +// comparing this run to the last one under that id — a sibling eval gaining a +// row is not a reason to break that. +func (c changeSet) reaches(eval Eval) bool { + if c.datasets[eval.Dataset] { + return true + } + for _, ref := range eval.Evaluators { + // Built-ins are the service's, so this configuration never republishes + // one and never has to recreate an eval for it. + if strings.HasPrefix(ref.Evaluator, evalcore.BuiltinPrefix) { + continue + } + if c.evaluators[ref.Evaluator] { + return true + } + } + return false +} + // projectRoot is the directory `$ref` paths resolve against. It is the // directory holding azure.yaml, which only azd can report. func (p *EvalServiceTargetProvider) projectRoot(ctx context.Context) string { From f5d920be3ca7e6c2cded9d42992ace8df89b96d3 Mon Sep 17 00:00:00 2001 From: mohessie Date: Mon, 10 Aug 2026 08:41:15 +0300 Subject: [PATCH 166/320] Reconcile an eval against what it declares, not what it resolves to Four behaviours the spec asks for that the reconciler did not have. An eval was recreated whenever an artifact it referenced was republished. The spec scopes the comparison to the entry's own declaration -- "an evaluator tracking latest that publishes a new version therefore leaves the eval alone" -- and Scenario 4 shows the evaluator going to version 3 while the eval that runs it is skipped as unchanged. Evals are immutable, so recreating one strands every run recorded against its id, which is the history that scenario exists to compare across a rubric edit. The declaration digest already answers the question, so the upstream flag is removed rather than narrowed, and the change tracking it needed goes with it. Name and description were excluded from that digest, correctly, because they are what the service treats as mutable and must not split a history. But nothing then pushed them, so an edit confined to either was dropped in silence. They are reconciled now through UpdateEvalParametersBody, on the reuse path as well as the rename path, and the eval keeps its id and every run under it. Metadata is merged rather than replaced so keys this extension does not own survive the update. A dataset entry with no source: was skipped before it reached the reconciler, which left EnsureDataset's own "already registered; just confirm it" branch unreachable. resolveSource already returns an empty path in that case, so the guard was redundant as well as wrong: a misspelled name now fails the deploy rather than a later run. A generated dataset recorded no fingerprint, so the first azd up after generate read the downloaded copy as new and published a second version identical to the one the generation job had just registered. The spec asks for the opposite: the job registered the version and the CLI holds a copy, so that deploy publishes nothing and only a local edit produces version 2. --- .../internal/cmd/build.go | 6 +- .../internal/cmd/eval_group.go | 2 +- .../internal/cmd/generate.go | 6 + .../internal/cmd/mutable_metadata_test.go | 145 ++++++++++++++++++ .../internal/cmd/reconciler.go | 103 +++++++++++-- .../internal/cmd/run_ops.go | 5 + .../internal/project/artifacts.go | 6 +- .../internal/project/change_scope_test.go | 77 ---------- .../internal/project/service_target_eval.go | 59 ++----- 9 files changed, 263 insertions(+), 146 deletions(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/mutable_metadata_test.go delete mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/project/change_scope_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build.go index fcbb52d689d..a181c676c21 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build.go @@ -293,18 +293,18 @@ func buildEvalRequest( hasTarget := group.Target != nil && group.Target.Name != "" targetType := "" if hasTarget { - metadata["azd_agent"] = group.Target.Name + metadata[metaAgent] = group.Target.Name targetType = group.Target.Type if targetType == "" { targetType = project.TargetTypeAgent } } targetBindings := sampleBindingsFor(targetType) - metadata["azd_eval"] = group.Name + metadata[metaEvalName] = group.Name // The create request has no description field, so the group's own // description rides in metadata rather than being dropped. if group.Description != "" { - metadata["azd_description"] = group.Description + metadata[metaDescription] = group.Description } level := group.EvaluationLevel diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/eval_group.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/eval_group.go index 03ffccd65c6..7af0b46cf8a 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/eval_group.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/eval_group.go @@ -70,7 +70,7 @@ func newEvalCreateCommand() *cobra.Command { } reconciler := &evalReconciler{ec: ec} - id, err := reconciler.EnsureEval(ctx, *eval, datasetPath, false) + id, err := reconciler.EnsureEval(ctx, *eval, datasetPath) if err != nil { return err } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go index ab91da7a10f..fde26ef1a92 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go @@ -365,6 +365,12 @@ func (ec *evalContext) generateDataset( } fmt.Fprint(out, messages.WroteArtifact(path)) + // The job registered the version and this file is a copy of it, so the + // state a deploy would have left behind is recorded now. Without it the + // next `azd up` finds no fingerprint for this dataset, reads the file as + // new, and publishes a second version identical to the one just generated. + ec.recordDeployedDataset(ctx, plan.Name, path, version) + return &project.ArtifactRef{ Name: plan.Name, Source: relativeSource(plan.BaseDir, path), diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/mutable_metadata_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/mutable_metadata_test.go new file mode 100644 index 00000000000..9a0b4142b2c --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/mutable_metadata_test.go @@ -0,0 +1,145 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "context" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "testing" + + "azureaieval/internal/pkg/eval_api" + "azureaieval/internal/project" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// recordedUpdate is the body of an update the reconciler pushed, or nil when it +// pushed nothing. +type recordedUpdate struct { + body *eval_api.UpdateOpenAIEvalRequest +} + +// reconcilerHoldingEval builds a reconciler whose service holds this eval, and +// records any update pushed to it. +func reconcilerHoldingEval( + t *testing.T, + held eval_api.OpenAIEval, +) (*evalReconciler, *recordedUpdate) { + t.Helper() + seen := &recordedUpdate{} + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodPost { + raw, err := io.ReadAll(r.Body) + require.NoError(t, err) + var body eval_api.UpdateOpenAIEvalRequest + require.NoError(t, json.Unmarshal(raw, &body)) + seen.body = &body + } + w.Header().Set("Content-Type", "application/json") + require.NoError(t, json.NewEncoder(w).Encode(held)) + })) + t.Cleanup(srv.Close) + + pipeline := runtime.NewPipeline("test", "v1", runtime.PipelineOptions{}, + &policy.ClientOptions{Retry: policy.RetryOptions{MaxRetries: -1}}) + return &evalReconciler{ec: &evalContext{ + evalClient: eval_api.NewEvalClientFromPipeline(srv.URL, pipeline), + }}, seen +} + +// A description is excluded from the fingerprint so that editing it does not +// fork the run history. Excluding it from the digest is not the same as +// ignoring it: the edit still has to reach the service, and this reuse path is +// the only place it can. +func TestPushMutableSendsAnEditedDescription(t *testing.T) { + r, seen := reconcilerHoldingEval(t, eval_api.OpenAIEval{ + ID: "eval-1", + Name: "support-gate", + Metadata: map[string]string{metaEvalName: "support-gate", metaDescription: "old wording"}, + }) + + r.pushMutable(context.Background(), "eval-1", project.Eval{ + Name: "support-gate", + Description: "new wording", + }, &eval_api.OpenAIEval{ + ID: "eval-1", + Name: "support-gate", + Metadata: map[string]string{metaEvalName: "support-gate", metaDescription: "old wording"}, + }) + + require.NotNil(t, seen.body, "an edited description must be pushed") + assert.Equal(t, "new wording", seen.body.Metadata[metaDescription]) + assert.Equal(t, "support-gate", seen.body.Name, "the name rides along unchanged") +} + +// A rename is applied in place rather than forking the history. +func TestPushMutableSendsARename(t *testing.T) { + held := eval_api.OpenAIEval{ID: "eval-1", Name: "old-name"} + r, seen := reconcilerHoldingEval(t, held) + + r.pushMutable(context.Background(), "eval-1", + project.Eval{Name: "new-name"}, &held) + + require.NotNil(t, seen.body) + assert.Equal(t, "new-name", seen.body.Name) +} + +// Every deploy walks this path, so an unchanged declaration must stay silent. +// Pushing regardless would write to the service on every `azd up`. +func TestPushMutableIsSilentWhenNothingChanged(t *testing.T) { + held := eval_api.OpenAIEval{ + ID: "eval-1", + Name: "support-gate", + Metadata: map[string]string{metaEvalName: "support-gate", metaDescription: "wording"}, + } + r, seen := reconcilerHoldingEval(t, held) + + r.pushMutable(context.Background(), "eval-1", project.Eval{ + Name: "support-gate", + Description: "wording", + }, &held) + + assert.Nil(t, seen.body, "an unchanged eval must not be written to") +} + +// Deleting the line from the config is an edit like any other. +func TestPushMutableClearsARemovedDescription(t *testing.T) { + held := eval_api.OpenAIEval{ + ID: "eval-1", + Name: "support-gate", + Metadata: map[string]string{metaDescription: "wording that was deleted"}, + } + r, seen := reconcilerHoldingEval(t, held) + + r.pushMutable(context.Background(), "eval-1", + project.Eval{Name: "support-gate"}, &held) + + require.NotNil(t, seen.body) + assert.NotContains(t, seen.body.Metadata, metaDescription) +} + +// The update replaces metadata rather than merging it, so anything the service +// or another writer put there has to be carried across or it is dropped. +func TestWithDescriptionKeepsMetadataItDoesNotOwn(t *testing.T) { + held := map[string]string{ + metaEvalName: "support-gate", + metaAgent: "support-agent", + "service_added": "keep me", + } + + merged := withDescription(held, "new wording") + + assert.Equal(t, "new wording", merged[metaDescription]) + assert.Equal(t, "keep me", merged["service_added"]) + assert.Equal(t, "support-agent", merged[metaAgent]) + assert.NotContains(t, held, metaDescription, "the held map must not be mutated") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go index d8697acc36d..bbfb4c0b42b 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go @@ -7,6 +7,7 @@ import ( "bufio" "context" "encoding/json" + "maps" "os" "reflect" "strconv" @@ -406,23 +407,27 @@ func (r *evalReconciler) evaluatorVersionResolvable( return false } -// EnsureEval creates the group when it has never been deployed, or when an -// upstream artifact changed. Groups are immutable, so a change means a new -// group and a new id. +// EnsureEval creates the eval when it has never been deployed, or when its own +// declaration changed. Evals are immutable, so a declaration change means a new +// eval and a new id. +// +// What an eval's references *resolve to* is deliberately not a reason to +// recreate it: an evaluator tracking latest that publishes a new version leaves +// every eval that runs it alone, which is what keeps a rubric edit comparable +// against the runs taken before it. func (r *evalReconciler) EnsureEval( ctx context.Context, group project.Eval, datasetPath string, - recreate bool, ) (string, error) { if group.ID != "" { return group.ID, nil } // Evals are immutable, so a change to the eval's own substance — evaluators, - // dataset, source, target, level — needs a new eval just as much as a change - // to an upstream artifact does. Name and description are excluded from the - // digest and pushed in place instead. + // dataset, source, target, level — needs a new eval. Name and description are + // excluded from the digest and pushed in place instead. + recreate := false digest, err := project.FingerprintGroup(group) if err != nil { return "", err @@ -443,7 +448,13 @@ func (r *evalReconciler) EnsureEval( } } if cached != "" && !recreate { - if _, err := r.ec.evalClient.GetOpenAIEval(ctx, cached); err == nil { + if remote, err := r.ec.evalClient.GetOpenAIEval(ctx, cached); err == nil { + // Reusing the eval is not the same as leaving it alone: name and + // description are excluded from the digest because they must not + // split a history, which makes this the only place an edit to + // either of them can reach the service. + r.pushMutable(ctx, cached, group, remote) + // Record the digest on reuse as well, otherwise an eval deployed // before fingerprinting existed never establishes a baseline and // later edits go undetected. @@ -480,9 +491,7 @@ func (r *evalReconciler) EnsureEval( // rename keeps the id and every run under it rather than forking the history. // // The name is what UpdateEvalParametersBody reaches, so the new one is pushed -// to the service. A failure there is not fatal: the eval is still the right one -// and the declaration still resolves, it just reads under its old name in the -// portal until the next deploy. +// to the service. func (r *evalReconciler) adoptRenamed( ctx context.Context, group project.Eval, @@ -496,13 +505,52 @@ func (r *evalReconciler) adoptRenamed( if err != nil { return "" } - if remote.Name == group.Name { - return id + r.pushMutable(ctx, id, group, remote) + return id +} + +// pushMutable sends the half of a declaration the service treats as mutable. +// +// Substance never travels this way — an edit that touches it is a new eval. +// Name and description are left out of the fingerprint precisely because they +// cost nothing to change and must not split a run history, so they are +// reconciled here rather than ignored, and the eval keeps its id and every run +// under it. +// +// A failure is not fatal. The eval is still the right one and the declaration +// still resolves; it just reads under its old wording in the portal until the +// next deploy. +func (r *evalReconciler) pushMutable( + ctx context.Context, + id string, + group project.Eval, + remote *eval_api.OpenAIEval, +) { + if remote == nil { + return + } + desired := withDescription(remote.Metadata, group.Description) + if remote.Name == group.Name && maps.Equal(remote.Metadata, desired) { + return } _, _ = r.ec.evalClient.UpdateOpenAIEval(ctx, id, &eval_api.UpdateOpenAIEvalRequest{ - Name: group.Name, + Name: group.Name, + Metadata: desired, }) - return id +} + +// withDescription applies the declaration's description to the metadata the +// service already holds, leaving every other key alone — including any the +// service added itself, which a replacing update would otherwise drop. +func withDescription(held map[string]string, description string) map[string]string { + merged := make(map[string]string, len(held)+1) + maps.Copy(merged, held) + if description == "" { + delete(merged, metaDescription) + } else { + merged[metaDescription] = description + } + return merged } // sameDefinition reports whether the locally authored definition already @@ -572,6 +620,31 @@ func versionKey(kind, name string) string { return project.FingerprintKey(kind, name) + "_VERSION" } +// recordDeployedDataset records the state a deploy would have left behind for a +// dataset that the service has already registered and that already has a local +// copy. +// +// `azd up` decides whether to publish by comparing the local file against a +// fingerprint held in the environment. A generated dataset arrives with no such +// fingerprint, so without this the first deploy after `generate` reads the file +// as new and publishes a second version identical to the one the job just +// registered. Only a local edit should produce version 2. +// +// Best effort: failing to record costs a redundant version, not correctness. +func (ec *evalContext) recordDeployedDataset( + ctx context.Context, + name, localPath, version string, +) { + digest, err := project.Fingerprint(localPath) + if err != nil { + return + } + _ = ec.setEnvValue(ctx, project.FingerprintKey("dataset", name), digest) + if version != "" { + _ = ec.setEnvValue(ctx, versionKey("dataset", name), version) + } +} + // idKey names the env entry holding a resolved id. // // Ids are per declaration. A single shared key works only while a config has diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go index 35b0a3bd916..c9d09387cdd 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go @@ -318,6 +318,11 @@ const ( // metaEvalName is the eval's declared name, recorded on the run because a // run is read on its own and an id is not what the author called it. metaEvalName = "azd_eval" + // metaAgent is the agent an eval targets. + metaAgent = "azd_agent" + // metaDescription carries an eval's description: the create request has no + // field of its own for it. + metaDescription = "azd_description" ) // runDataset renders the dataset a run scored, versioned when a version was diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/artifacts.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/artifacts.go index 6159261fc1b..1a7277c390a 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/artifacts.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/artifacts.go @@ -21,8 +21,10 @@ const ( type ArtifactRef struct { Name string `json:"name"` Source string `json:"source"` - // Version is what the generation job published, which the catalog line - // names so a reader can pin it without going to look. + // Version is what the generation job published. It is reported to the + // author rather than written into the catalog: an evaluator cannot carry + // both a `source:` and a `version:`, and pinning a generated dataset would + // freeze it against the very edit it exists to be the starting point for. Version string `json:"version,omitempty"` } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/change_scope_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/change_scope_test.go deleted file mode 100644 index bc825c5332c..00000000000 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/change_scope_test.go +++ /dev/null @@ -1,77 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package project - -import ( - "testing" - - "azureaieval/internal/pkg/evalcore" - - "github.com/stretchr/testify/assert" -) - -func evalNamed(name, dataset string, evaluators ...string) Eval { - refs := evalcore.EvaluatorList{} - for _, e := range evaluators { - refs = append(refs, evalcore.EvaluatorRef{Evaluator: e}) - } - return Eval{Name: name, Dataset: dataset, Evaluators: refs} -} - -// An eval is immutable, so recreating it abandons the id its runs are recorded -// against. Scenario 3 is a developer comparing this run to the last one under -// that id, and one flag for the whole file used to break that for every eval -// because one unrelated dataset gained a row. -func TestChangeSetOnlyReachesWhatAnEvalNames(t *testing.T) { - changed := changeSet{ - datasets: map[string]bool{"support-golden": true, "billing-golden": false}, - evaluators: map[string]bool{"support-quality": false, "billing-quality": false}, - } - - support := evalNamed("support-eval", "support-golden", - "builtin.task_adherence", "support-quality") - billing := evalNamed("billing-eval", "billing-golden", - "builtin.task_adherence", "billing-quality") - - assert.True(t, changed.reaches(support), "its own dataset was republished") - assert.False(t, changed.reaches(billing), - "a sibling's dataset changing is not a reason to discard this eval's history") -} - -func TestChangeSetReachesThroughAnEvaluator(t *testing.T) { - changed := changeSet{ - datasets: map[string]bool{"golden": false}, - evaluators: map[string]bool{"quality": true}, - } - - assert.True(t, - changed.reaches(evalNamed("e", "golden", "quality")), - "an evaluator this eval runs was republished") - assert.False(t, - changed.reaches(evalNamed("other", "golden", "builtin.task_adherence")), - "this eval does not run that evaluator") -} - -// Built-ins belong to the service. This configuration never publishes one, so -// a name that merely looks like a catalog entry must not recreate anything. -func TestChangeSetIgnoresBuiltins(t *testing.T) { - changed := changeSet{ - datasets: map[string]bool{}, - evaluators: map[string]bool{"builtin.task_adherence": true}, - } - - assert.False(t, changed.reaches(evalNamed("e", "golden", "builtin.task_adherence"))) -} - -// A configuration whose datasets and evaluators are all already registered -// publishes nothing, and must leave every eval where it is. -func TestChangeSetReachesNothingWhenNothingChanged(t *testing.T) { - changed := changeSet{ - datasets: map[string]bool{"golden": false}, - evaluators: map[string]bool{"quality": false}, - } - - assert.False(t, changed.reaches(evalNamed("e", "golden", "quality"))) - assert.False(t, changed.reaches(evalNamed("no-dataset", ""))) -} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go index a98a12e4ea7..0552542d880 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go @@ -13,7 +13,6 @@ import ( "strings" "azureaieval/internal/messages" - "azureaieval/internal/pkg/evalcore" "github.com/azure/azure-dev/cli/azd/pkg/azdext" "github.com/azure/azure-dev/cli/azd/pkg/foundry" @@ -43,7 +42,7 @@ type Reconciler interface { // evaluators or options changed, returning its id. datasetPath is the local // dataset backing the group, or empty when it is already registered; it lets // the reconciler bind criteria to the columns that actually exist. - EnsureEval(ctx context.Context, group Eval, datasetPath string, recreate bool) (id string, err error) + EnsureEval(ctx context.Context, group Eval, datasetPath string) (id string, err error) } // EvalServiceTargetProvider deploys eval resources during `azd up`. azd owns @@ -150,19 +149,12 @@ func (p *EvalServiceTargetProvider) Deploy( // 1. Datasets the configuration owns. Paths are kept so an eval that names // one can derive its columns without reading the blob back. // - // What changed is tracked per artifact, not as one flag for the file. Evals - // are immutable, so recreating one discards the id its run history hangs - // off; a single flag would do that to every eval in the file because one - // unrelated dataset gained a row. - changedArtifacts := changeSet{ - datasets: map[string]bool{}, - evaluators: map[string]bool{}, - } + // A declaration with no `source:` is included rather than skipped: it names + // a dataset that is already registered, and reconciling it is what confirms + // it is really there and settles which version a `version:` pin selected. + // Skipping it would leave a misspelled name to surface as a failed run. datasetPaths := map[string]string{} for _, decl := range cfg.Datasets { - if decl.Source == "" { - continue - } report(progress, messages.ReconcilingDataset(decl.Name)) localPath := resolveSource(baseDir, decl.Source) datasetPaths[decl.Name] = localPath @@ -170,7 +162,6 @@ func (p *EvalServiceTargetProvider) Deploy( if err != nil { return nil, messages.DatasetProblem(decl.Name, err) } - changedArtifacts.datasets[decl.Name] = changed report(progress, describeResult("dataset", decl.Name, version, changed)) } @@ -183,17 +174,18 @@ func (p *EvalServiceTargetProvider) Deploy( if err != nil { return nil, messages.EvaluatorProblem(decl.Name, err) } - changedArtifacts.evaluators[decl.Name] = changed report(progress, describeResult("evaluator", decl.Name, version, changed)) } - // 3. The evals. Evals are immutable, so a change upstream means a new one - // must be created and the stored id replaced. + // 3. The evals. An eval is recreated only when its own declaration changed: + // the comparison covers what the entry declares, not what its references + // resolve to. An evaluator tracking latest that publishes a new version + // leaves every eval that runs it alone, which is what keeps a rubric edit + // comparable against the runs before it. for i := range cfg.Evals { eval := cfg.Evals[i] report(progress, messages.ReconcilingEval(eval.Name)) - id, err := reconciler.EnsureEval( - ctx, eval, datasetPaths[eval.Dataset], changedArtifacts.reaches(eval)) + id, err := reconciler.EnsureEval(ctx, eval, datasetPaths[eval.Dataset]) if err != nil { return nil, messages.EvalProblem(eval.Name, err) } @@ -203,35 +195,6 @@ func (p *EvalServiceTargetProvider) Deploy( return &azdext.ServiceDeployResult{}, nil } -// changeSet records which artifacts this deploy republished. -type changeSet struct { - datasets map[string]bool - evaluators map[string]bool -} - -// reaches reports whether anything this eval is built from was republished. -// -// Only what the eval names counts. An eval is immutable, so recreating it -// abandons the id its runs are recorded against, and Scenario 3 is a developer -// comparing this run to the last one under that id — a sibling eval gaining a -// row is not a reason to break that. -func (c changeSet) reaches(eval Eval) bool { - if c.datasets[eval.Dataset] { - return true - } - for _, ref := range eval.Evaluators { - // Built-ins are the service's, so this configuration never republishes - // one and never has to recreate an eval for it. - if strings.HasPrefix(ref.Evaluator, evalcore.BuiltinPrefix) { - continue - } - if c.evaluators[ref.Evaluator] { - return true - } - } - return false -} - // projectRoot is the directory `$ref` paths resolve against. It is the // directory holding azure.yaml, which only azd can report. func (p *EvalServiceTargetProvider) projectRoot(ctx context.Context) string { From 2890575fb18a93b14701bba7f6633d00336fb73e Mon Sep 17 00:00:00 2001 From: mohessie Date: Mon, 10 Aug 2026 08:41:30 +0300 Subject: [PATCH 167/320] Find gotestsum on the shell Windows actually runs Two lines of ci-test.ps1 assumed PowerShell 6 or later. $IsWindows does not exist on Windows PowerShell 5.1, so the .exe suffix was never appended; and Join-Path there takes a single child path, so the three-argument call failed outright. The script then found no gotestsum, printed the fallback notice, and ran bare `go test` with no JUnit report -- while still exiting 0, which is why it went unnoticed. --- cli/azd/extensions/azure.ai.evaluations/ci-test.ps1 | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/cli/azd/extensions/azure.ai.evaluations/ci-test.ps1 b/cli/azd/extensions/azure.ai.evaluations/ci-test.ps1 index 6175585d318..5415bd107a8 100644 --- a/cli/azd/extensions/azure.ai.evaluations/ci-test.ps1 +++ b/cli/azd/extensions/azure.ai.evaluations/ci-test.ps1 @@ -18,10 +18,15 @@ $gopath = go env GOPATH $gotestsumBinary = "gotestsum" -if ($IsWindows) { +# $IsWindows only exists on PowerShell 6 and later. On Windows PowerShell 5.1 it +# is undefined, so the suffix was never appended, the binary was never found, +# and the run silently fell back to `go test` with no JUnit report. +if ($env:OS -eq "Windows_NT") { $gotestsumBinary += ".exe" } -$gotestsum = Join-Path $gopath "bin" $gotestsumBinary +# Windows PowerShell 5.1 takes a single child path, so the three-argument form +# fails outright there. Nesting is what every version accepts. +$gotestsum = Join-Path (Join-Path $gopath "bin") $gotestsumBinary Write-Host "Running unit tests..." From d9e4e49bf4f1919ccf18e14be9b649308d225f37 Mon Sep 17 00:00:00 2001 From: mohessie Date: Mon, 10 Aug 2026 17:32:47 +0300 Subject: [PATCH 168/320] Validate on the path a deploy actually takes, and honour a pinned version Findings from two more model reviews. Hoisting `buildEvalRequest` above the reuse decision closes a gap the previous commit opened. Building the request is also what checks each evaluator's required fields against the dataset's columns, and it only ran when an eval was being created. Now that an eval is reused whenever its own declaration is unchanged, a dataset that loses a column an evaluator needs reaches a run unreported. The spec asks for the opposite: `azd up` validates locally and names the offending column. Memoizing the evaluator schema index keeps that hoist from costing two more list calls per eval, and removes the same cost from the create path, which was already paying it once per eval. Only a complete read is cached: a transient failure on either listing would otherwise be frozen in for the rest of the process, and every eval after it would validate against legacyInputs, which accepts fields the evaluator never declared. A dataset pinned with `version:` and no `source:` was resolved at deploy and then ignored at run time. The version was returned but never recorded, so the run fell back to latest and scored rows the author had pinned away from. It is now confirmed against the service -- a pin naming a version that is not there is a mistake worth reporting at deploy -- and recorded under the key a run already reads. `dataset list -o json` answered an absent list with the sentence a human would read, which is not JSON, so a caller piping it into a parser got a syntax error instead of an empty array. The other eight empty-state renderers already decided output format first; this one did not. --- .../internal/cmd/build.go | 13 +++++++- .../internal/cmd/context.go | 3 ++ .../internal/cmd/dataset.go | 5 ++-- .../internal/cmd/reconciler.go | 30 ++++++++++++++----- 4 files changed, 40 insertions(+), 11 deletions(-) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build.go index a181c676c21..0989c4b50f5 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build.go @@ -28,11 +28,16 @@ import ( // query/response and so looks right for the common evaluators while quietly // dropping the fields anything else needs. func (ec *evalContext) evaluatorSchemas(ctx context.Context) map[string]*eval_api.EvaluatorSummary { - index := map[string]*eval_api.EvaluatorSummary{} + if ec.schemas != nil { + return ec.schemas + } + index := map[string]*eval_api.EvaluatorSummary{} + complete := true for _, filter := range []string{"", eval_api.EvaluatorTypeBuiltin} { list, err := ec.evalClient.ListEvaluators(ctx, filter, ProjectEndpointAPIVersion) if err != nil { + complete = false continue } maps.Copy(index, list.ByName()) @@ -40,6 +45,12 @@ func (ec *evalContext) evaluatorSchemas(ctx context.Context) map[string]*eval_ap if len(index) == 0 { return nil } + // Only a complete read is worth keeping. Caching a half of it would leave + // every later eval validating against legacyInputs, which accepts fields + // the evaluator never declared. + if complete { + ec.schemas = index + } return index } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/context.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/context.go index 10e7ef85dab..5dbb7b59c28 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/context.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/context.go @@ -32,6 +32,9 @@ type evalContext struct { evalClient *eval_api.EvalClient datasetClient *dataset_api.DatasetClient + + // Held only once both listings succeed; a partial read is not reusable. + schemas map[string]*eval_api.EvaluatorSummary } // newEvalContext resolves the project endpoint and builds the data-plane diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset.go index 93ca2b65db0..fcf12465434 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset.go @@ -220,9 +220,10 @@ func newDatasetVersionsListCommand() *cobra.Command { } func renderDatasets(cmd *cobra.Command, list *dataset_api.DatasetList) error { + // JSON is decided before emptiness: a caller piping this into a parser needs + // an empty array, not the sentence a human would read. if list == nil { - fmt.Fprint(cmd.OutOrStdout(), messages.NoDatasets()) - return nil + list = &dataset_api.DatasetList{} } if isJSON(cmd) { return emitJSONList(cmd.OutOrStdout(), list.Value) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go index bbfb4c0b42b..b66f2334883 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go @@ -60,7 +60,16 @@ func (r *evalReconciler) EnsureDataset( return "", false, messages.DatasetNotLocalNorRegistered(decl.Name) } version = dataset_api.LatestVersion(list.Value) + } else if _, err := r.ec.datasetClient.GetDataset( + ctx, decl.Name, version, ProjectEndpointAPIVersion, + ); err != nil { + return "", false, messages.DatasetVersionNotFoundWithHint(decl.Name, version) } + + // Recorded so a run reads the version reconciliation settled on. Without + // this a pin is honoured at deploy and then ignored at run time, which + // scores different rows than the ones the author asked for. + _ = r.ec.setEnvValue(ctx, versionKey("dataset", decl.Name), version) return version, false, nil } @@ -437,6 +446,19 @@ func (r *evalReconciler) EnsureEval( recreate = true } + // Building the request is also what checks the declaration against the + // dataset's columns, so it happens before the reuse decision: a dataset can + // lose a column an evaluator needs without the eval's own declaration + // changing, and reusing the eval would let that reach a run unreported. + req, err := buildEvalRequest( + &group, + r.ec.evaluatorSchemas(ctx), + datasetColumnsFromPath(datasetPath), + ) + if err != nil { + return "", err + } + cached := r.ec.getEnvValue(ctx, idKey("eval", group.Name)) if cached == "" && !recreate { // Nothing recorded under this name, but the substance may already be @@ -466,14 +488,6 @@ func (r *evalReconciler) EnsureEval( } } - req, err := buildEvalRequest( - &group, - r.ec.evaluatorSchemas(ctx), - datasetColumnsFromPath(datasetPath), - ) - if err != nil { - return "", err - } created, err := r.ec.evalClient.CreateOpenAIEval(ctx, req) if err != nil { return "", err From 42dc70dca8ad741ec0e118f684885344b1e11cc5 Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 11 Aug 2026 02:44:42 +0300 Subject: [PATCH 169/320] Reconcile artifacts in `eval create`, and resolve `eval show` by name Two defects the bug bash surfaced by running the hero scenarios end to end. `eval create` called only EnsureEval, so the dataset and evaluators the eval names were never published. The service resolves those names when the eval is created, so the command failed against a project where they did not already exist. Reconciliation lived solely in the `azd up` path, which is unreachable here: an eval project has no infra/, so `azd up` fails compiling bicep that does not exist and `azd deploy` reports nothing provisioned. `eval create` now publishes what the eval refers to before creating it, which is also what lets a rubric edit reach the service without a full deploy. `eval show` accepted only the service id, and so refused the very name that `eval list` prints. It now falls back to a lookup by name when the direct read returns not-found. Also records, as a TODO in init.go, that the scaffold's next-steps text still suggests `azd up` for a project that has no infrastructure to provision. --- .../azure.ai.evaluations/extension.yaml | 2 +- .../internal/cmd/eval_group.go | 35 +++++++++++++++++++ .../internal/cmd/evalref.go | 18 ++++++++++ .../azure.ai.evaluations/internal/cmd/init.go | 2 ++ .../azure.ai.evaluations/version.txt | 2 +- 5 files changed, 57 insertions(+), 2 deletions(-) diff --git a/cli/azd/extensions/azure.ai.evaluations/extension.yaml b/cli/azd/extensions/azure.ai.evaluations/extension.yaml index 825f01ed341..ae1b95bac0b 100644 --- a/cli/azd/extensions/azure.ai.evaluations/extension.yaml +++ b/cli/azd/extensions/azure.ai.evaluations/extension.yaml @@ -5,7 +5,7 @@ displayName: Foundry evaluations (Beta) description: Define and run Foundry evaluations from your terminal. (Beta) usage: azd ai eval [options] # NOTE: Make sure version.txt is in sync with this version. -version: 1.0.0-beta.1 +version: 1.0.0-beta.2 requiredAzdVersion: ">=1.27.1" language: go capabilities: diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/eval_group.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/eval_group.go index 7af0b46cf8a..ba8dcb55694 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/eval_group.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/eval_group.go @@ -70,6 +70,33 @@ func newEvalCreateCommand() *cobra.Command { } reconciler := &evalReconciler{ec: ec} + + // The eval names its dataset and evaluators, and the service resolves + // those names when the eval is created, so they have to be published + // first. `azd up` reconciles the whole file; this reconciles only what + // this eval refers to, which is also what makes a rubric edit reach + // the service without a full deploy. + baseDir := filepath.Dir(path) + if decl, ok := cfg.DatasetDeclaration(eval.Dataset); ok { + if _, _, err := reconciler.EnsureDataset(ctx, *decl, datasetPath); err != nil { + return messages.DatasetProblem(decl.Name, err) + } + } + for _, ref := range eval.Evaluators { + decl, ok := cfg.EvaluatorDeclaration(ref.Evaluator) + // A built-in, or one already registered, has nothing local to publish. + if !ok || decl.Source == "" { + continue + } + local := decl.Source + if !filepath.IsAbs(local) { + local = filepath.Join(baseDir, local) + } + if _, _, err := reconciler.EnsureEvaluator(ctx, *decl, local); err != nil { + return messages.EvaluatorProblem(decl.Name, err) + } + } + id, err := reconciler.EnsureEval(ctx, *eval, datasetPath) if err != nil { return err @@ -154,6 +181,14 @@ func newEvalShowCommand() *cobra.Command { defer ec.Close() group, err := ec.evalClient.GetOpenAIEval(ctx, evalID) + if err != nil && eval_api.IsNotFound(err) { + // The argument reads as an id. `list` reports names, and this + // refused the very name it points the reader at, so a name is + // resolved before giving up. + if resolved := ec.evalIDNamed(ctx, evalID); resolved != "" { + group, err = ec.evalClient.GetOpenAIEval(ctx, resolved) + } + } if err != nil { if eval_api.IsNotFound(err) { return messages.EvalNotFound(evalID) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evalref.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evalref.go index ad0ff0068b4..98d04c16870 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evalref.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evalref.go @@ -76,3 +76,21 @@ func (ec *evalContext) resolveEvalRef( func (ec *evalContext) recordedEvalID(ctx context.Context, evalName string) string { return ec.getEnvValue(ctx, idKey("eval", evalName)) } + +// evalIDNamed finds the id of the eval the service lists under this name. +// +// Evals are addressed by id, but every listing reports a name, so a name is +// what a reader has to hand. Returns empty when nothing matches, leaving the +// caller's not-found reporting alone. +func (ec *evalContext) evalIDNamed(ctx context.Context, name string) string { + list, err := ec.evalClient.ListOpenAIEvals(ctx, 0) + if err != nil || list == nil { + return "" + } + for _, e := range list.Data { + if e.Name == name { + return e.ID + } + } + return "" +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go index 2b0c315a555..cb613c56c79 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go @@ -406,6 +406,8 @@ func (s scaffold) nextSteps() []string { "azd ai eval generate --evaluator --evaluator-name "+s.rubricName) } if len(steps) == 0 { + // TODO: suggest `azd deploy`, not `azd up`. Eval resources are data-plane + // only, so a project with no infra/ fails provision before reaching us. steps = append(steps, "azd up", "azd ai eval run start") } return steps diff --git a/cli/azd/extensions/azure.ai.evaluations/version.txt b/cli/azd/extensions/azure.ai.evaluations/version.txt index ffbc9939864..7e0b23109f3 100644 --- a/cli/azd/extensions/azure.ai.evaluations/version.txt +++ b/cli/azd/extensions/azure.ai.evaluations/version.txt @@ -1 +1 @@ -1.0.0-beta.1 +1.0.0-beta.2 From a8a3d8e3c89121257e1bb95d3bb04472d83eacd0 Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 11 Aug 2026 03:12:12 +0300 Subject: [PATCH 170/320] Make `init` say it has no judge model instead of writing one without `init` scaffolded a config whose evaluators carried no deployment_name, exited 0, and said nothing. The judging built-ins declare the deployment as required, so the failure surfaced much later, from the service, far from the command that caused it. Bug 5511012. The cause is that detection never worked. detectModelDeployment looks for a service whose host is `azure.ai.model`, and no such host exists: it is not in azure.yaml's serviceHost enum and appears nowhere else in azd. Deployments are declared under the Foundry project service's `deployments:`, which detection never read, so it returned empty for every real project and the empty result was then treated as acceptable. Deployments are now read from where they actually live. One is used, several prompt, and none, or several under --no-prompt, is an error naming --judge-model. That matches what the spec's flag table already says the command should do. The old host lookup is kept ahead of the new one so a project that somehow does declare it is unaffected. --- .../azure.ai.evaluations/internal/cmd/init.go | 8 +- .../internal/cmd/init_model.go | 116 ++++++++++++++++++ .../internal/cmd/init_model_test.go | 83 +++++++++++++ .../internal/messages/messages.go | 29 +++++ 4 files changed, 234 insertions(+), 2 deletions(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_model.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_model_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go index cb613c56c79..7d80347ac04 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go @@ -95,7 +95,10 @@ func newInitCommand() *cobra.Command { evalName = defaultEvalName(target, source) } if judgeModel == "" { - judgeModel = detectModelDeployment(azdProject) + judgeModel, err = resolveJudgeModel(cmd, azdProject) + if err != nil { + return err + } } configPath := project.ResolveEvalConfigPath(path) @@ -478,7 +481,8 @@ const aiModelHost = "azure.ai.model" // the project already declares. // // `init` makes no service calls, so detection is limited to the project file. -// Coming back empty is not a failure: --judge-model supplies it. +// Coming back empty leaves it to resolveJudgeModel, which reads the Foundry +// project's deployments: and then asks or names --judge-model. func detectModelDeployment(proj *azdext.ProjectConfig) string { for name, svc := range proj.GetServices() { if svc.GetHost() != aiModelHost { diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_model.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_model.go new file mode 100644 index 00000000000..2060e5ce9d5 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_model.go @@ -0,0 +1,116 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "sort" + + "azureaieval/internal/messages" + + "github.com/azure/azure-dev/cli/azd/pkg/azdext" + "github.com/spf13/cobra" +) + +// modelDeployments names every model deployment the project declares, sorted so +// a prompt and an error list the same way twice. +// +// They live under the Foundry project service's deployments:, which is where +// the sibling extensions put them, so reading them is a file read rather than a +// service call. An entry given as a $ref is skipped rather than followed: +// resolving includes is the project extension's job, and a scaffold that cannot +// see a deployment falls back to naming --judge-model, which is a worse message +// but never a wrong one. +func modelDeployments(proj *azdext.ProjectConfig) []string { + seen := map[string]bool{} + var names []string + + for _, svc := range proj.GetServices() { + if svc.GetHost() != aiProjectHost { + continue + } + props := svc.GetAdditionalProperties() + if props == nil || len(props.GetFields()) == 0 { + props = svc.GetConfig() + } + if props == nil { + continue + } + declared, ok := props.AsMap()["deployments"].([]any) + if !ok { + continue + } + for _, entry := range declared { + fields, ok := entry.(map[string]any) + if !ok { + continue + } + name, ok := fields["name"].(string) + if !ok || name == "" || seen[name] { + continue + } + seen[name] = true + names = append(names, name) + } + } + + sort.Strings(names) + return names +} + +// resolveJudgeModel settles the deployment the graders judge with. +// +// The judging built-ins declare the deployment as required, so an eval written +// without one is rejected by the service long after the command that wrote it. +// That is why coming back empty is a failure here rather than something left +// for later: `init` would otherwise exit 0 having written a configuration that +// cannot be deployed. +func resolveJudgeModel(cmd *cobra.Command, proj *azdext.ProjectConfig) (string, error) { + if model := detectModelDeployment(proj); model != "" { + return model, nil + } + + deployments := modelDeployments(proj) + switch len(deployments) { + case 0: + return "", messages.JudgeModelRequired() + case 1: + return deployments[0], nil + } + + if noPrompt(cmd) { + return "", messages.AmbiguousJudgeModel(deployments) + } + return promptJudgeModel(cmd, deployments) +} + +// promptJudgeModel asks which of the project's deployments to judge with. +func promptJudgeModel(cmd *cobra.Command, deployments []string) (string, error) { + azdClient, err := azdext.NewAzdClient() + if err != nil { + return "", messages.ConnectingToAzd(err) + } + defer azdClient.Close() + + choices := make([]*azdext.SelectChoice, 0, len(deployments)) + for i := range deployments { + choices = append(choices, &azdext.SelectChoice{ + Label: deployments[i], Value: deployments[i], + }) + } + + resp, err := azdClient.Prompt().Select(cmd.Context(), &azdext.SelectRequest{ + Options: &azdext.SelectOptions{ + Message: messages.SelectJudgeModelPrompt(), + Choices: choices, + }, + }) + if err != nil { + return "", messages.SelectingJudgeModel(err) + } + index := int(resp.GetValue()) + if index < 0 || index >= len(deployments) { + return "", messages.AmbiguousJudgeModel(deployments) + } + return deployments[index], nil +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_model_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_model_test.go new file mode 100644 index 00000000000..39d6612ac00 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_model_test.go @@ -0,0 +1,83 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "testing" + + "github.com/azure/azure-dev/cli/azd/pkg/azdext" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// projectWithDeployments builds a project whose Foundry project service +// declares the given deployments, which is the shape azd hands the extension +// for a `deployments:` list in azure.yaml. +func projectWithDeployments(t *testing.T, names ...string) *azdext.ProjectConfig { + t.Helper() + declared := make([]any, 0, len(names)) + for _, n := range names { + declared = append(declared, map[string]any{ + "name": n, + "model": map[string]any{"name": n, "format": "OpenAI", "version": "1"}, + }) + } + proj := projectWith() + proj.Services["ai-project"] = &azdext.ServiceConfig{ + Name: "ai-project", + Host: aiProjectHost, + AdditionalProperties: mustStruct(t, map[string]any{ + "deployments": declared, + }), + } + return proj +} + +// The deployments live under the Foundry project service, which is where the +// sibling extensions put them. Reading them is a file read, so `init` keeps +// making no service calls. +func TestModelDeployments_ReadFromTheProjectService(t *testing.T) { + assert.Empty(t, modelDeployments(projectWith("api", "web")), + "a project with no Foundry project service declares no deployments") + + assert.Equal(t, []string{"gpt-4.1-nano", "gpt-4o-mini"}, + modelDeployments(projectWithDeployments(t, "gpt-4o-mini", "gpt-4.1-nano")), + "sorted, so a prompt and an error list them the same way twice") +} + +// A single declared deployment is the one to judge with, so the common project +// needs no flag at all. +func TestResolveJudgeModel_DetectsTheOnlyDeployment(t *testing.T) { + model, err := resolveJudgeModel(newInitCommand(), + projectWithDeployments(t, "gpt-4.1-nano")) + + require.NoError(t, err) + assert.Equal(t, "gpt-4.1-nano", model) +} + +// The judging built-ins declare the deployment as required, so a project that +// declares none has to say which to use. Failing here is the point: `init` used +// to exit 0 having written a configuration the service later rejects. +func TestResolveJudgeModel_NoDeploymentsNamesTheFlag(t *testing.T) { + _, err := resolveJudgeModel(newInitCommand(), projectWith("api", "web")) + + require.Error(t, err) + assert.Contains(t, err.Error(), "--judge-model") +} + +// With several there is nothing to detect. Under --no-prompt the flag is the +// only way to say which, so the error names it and lists the candidates. +func TestResolveJudgeModel_AmbiguousUnderNoPrompt(t *testing.T) { + cmd := newInitCommand() + // --no-prompt is inherited from the root in the real tree, so the test has + // to supply it the way the root does. + cmd.Flags().Bool("no-prompt", true, "") + + _, err := resolveJudgeModel(cmd, projectWithDeployments(t, "gpt-4.1-nano", "gpt-4o-mini")) + + require.Error(t, err) + assert.Contains(t, err.Error(), "--judge-model") + assert.Contains(t, err.Error(), "gpt-4.1-nano") + assert.Contains(t, err.Error(), "gpt-4o-mini") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go b/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go index f03f2958d90..56b02daa8aa 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go @@ -1399,6 +1399,35 @@ func SelectingAgent(err error) error { return fmt.Errorf("selecting an agent to evaluate: %w", err) } +// JudgeModelRequired reports a scaffold that has no deployment to judge with. +// +// Reached when the project declares no model deployment to read one from, so +// the flag is the whole of the way out. Failing here is deliberate: the judging +// built-ins declare the deployment as required, so a config written without one +// is rejected by the service later, far from the command that wrote it. +func JudgeModelRequired() error { + return errors.New( + "a model deployment is required to judge with: pass --judge-model. " + + "This project declares no deployments: to read one from") +} + +// AmbiguousJudgeModel reports several deployments where only one can be used. +func AmbiguousJudgeModel(models []string) error { + return fmt.Errorf( + "this project declares more than one model deployment (%s), so "+ + "--judge-model says which the graders judge with", strings.Join(models, ", ")) +} + +// SelectJudgeModelPrompt asks which deployment the graders judge with. +func SelectJudgeModelPrompt() string { + return "Select the model deployment the graders judge with:" +} + +// SelectingJudgeModel reports a failed judge model prompt. +func SelectingJudgeModel(err error) error { + return fmt.Errorf("selecting a judge model deployment: %w", err) +} + // UsingTraceSource reports a scaffold that reads production traces. func UsingTraceSource() string { return fmt.Sprintf("%s Using data source: traces (Application Insights)\n", DoneMark) From 67f49a61457ff0b9b254ff97b3b039e60bb81899 Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 11 Aug 2026 03:27:42 +0300 Subject: [PATCH 171/320] Release 1.0.0-beta.3 --- cli/azd/extensions/azure.ai.evaluations/extension.yaml | 2 +- cli/azd/extensions/azure.ai.evaluations/version.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cli/azd/extensions/azure.ai.evaluations/extension.yaml b/cli/azd/extensions/azure.ai.evaluations/extension.yaml index ae1b95bac0b..7b5c94df867 100644 --- a/cli/azd/extensions/azure.ai.evaluations/extension.yaml +++ b/cli/azd/extensions/azure.ai.evaluations/extension.yaml @@ -5,7 +5,7 @@ displayName: Foundry evaluations (Beta) description: Define and run Foundry evaluations from your terminal. (Beta) usage: azd ai eval [options] # NOTE: Make sure version.txt is in sync with this version. -version: 1.0.0-beta.2 +version: 1.0.0-beta.3 requiredAzdVersion: ">=1.27.1" language: go capabilities: diff --git a/cli/azd/extensions/azure.ai.evaluations/version.txt b/cli/azd/extensions/azure.ai.evaluations/version.txt index 7e0b23109f3..b0a2ffd0f5f 100644 --- a/cli/azd/extensions/azure.ai.evaluations/version.txt +++ b/cli/azd/extensions/azure.ai.evaluations/version.txt @@ -1 +1 @@ -1.0.0-beta.2 +1.0.0-beta.3 From 9af1729b3593ef7c870e85195452df7f8d960bb1 Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 11 Aug 2026 04:14:43 +0300 Subject: [PATCH 172/320] Send `--from traces` a request the service accepts Generating from traces failed for everyone with `Atleast Prompt or prompt agent_name is required`. A traces source names the agent whose conversations to read, but the service also requires a prompt or an agent beside it, and `--from traces` sent the traces source alone. Probed against the service to settle what it wants rather than guess: traces alone is a 400, traces plus agent is accepted, traces plus prompt is accepted. The agent and its instructions now travel with traces exactly as they already do for `--from agent`. The instructions matter for a second reason: agent-seeded generation is failing server-side for every agent, and the prompt is what the existing retry falls back to, so without it a traces run had no way through. Verified live end to end: `generate --dataset --from traces` now exits 0 and downloads the dataset, taking the documented prompt-only retry on the way. Also refuses `--from traces` with no agent locally, where it previously built a source naming nothing to read. --- .../internal/pkg/eval_api/generation.go | 18 ++++++-- .../internal/pkg/eval_api/generation_test.go | 43 ++++++++++++++----- 2 files changed, 48 insertions(+), 13 deletions(-) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/generation.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/generation.go index 235f58884a0..142183e8883 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/generation.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/generation.go @@ -72,10 +72,17 @@ func BuildGenerationSources( // which is what decides whether an empty-handed source is an error. asked := func(kind string) bool { return want[kind] } + // A traces source names the agent whose traces to read, but the service + // still requires a prompt or an agent beside it, so traces on their own are + // refused for every agent. The agent travels with them for the same reason + // it travels with a prompt below. Verified against the service: traces alone + // is a 400, traces plus agent is accepted. + tracesNeedTheAgent := asked("traces") && agentName != "" + // The agent is settled first because whether it was built decides whether // its instructions have anything to be the instructions of. var agentSource *GenerationSource - if selected("agent") { + if selected("agent") || tracesNeedTheAgent { switch { case agentName != "": agentSource = &GenerationSource{Type: "agent", AgentName: agentName} @@ -91,8 +98,9 @@ func BuildGenerationSources( // travel with it as a prompt. That is also the only shape the service // currently honours: the agent source alone fails for every agent, and the // prompt is what the retry in generateDataset falls back to. Without this, - // `--from agent` would be a request that always fails. - promptCarriesTheAgent := agentSource != nil && asked("agent") + // `--from agent` would be a request that always fails, and `--from traces` + // would have nothing to fall back to when agent seeding fails. + promptCarriesTheAgent := agentSource != nil && (asked("agent") || tracesNeedTheAgent) if selected("prompt") || promptCarriesTheAgent { switch { case instruction != "": @@ -113,6 +121,10 @@ func BuildGenerationSources( // A window narrows the request; it does not authorize it. Asking for // traces without one means every trace the agent has. switch { + case agentName == "" && asked("traces"): + // Without an agent the source names nothing to read and carries + // nothing the service accepts beside it. + unbuildable = append(unbuildable, "traces") case traces != nil && traces.Days > 0: sources = append(sources, GenerationSource{ Type: "traces", diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/generation_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/generation_test.go index 6a5c993f820..298347a8e19 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/generation_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/generation_test.go @@ -23,14 +23,18 @@ func kindsOf(sources []GenerationSource) []string { // Naming a source is a request to send that one, not a hint. Everything the // plan could otherwise have offered stays out of the request. +// +// Demonstrated with prompt because it stands alone. Agent and traces are both +// refused by the service unless a prompt or an agent accompanies them, so each +// carries one; that carve-out is pinned in their own tests. func TestBuildGenerationSources_SendsOnlyWhatFromNamed(t *testing.T) { sources, unbuildable := BuildGenerationSources( - []string{"traces"}, + []string{"prompt"}, "support-agent", "3", "answer support questions", &TraceOptions{Days: 7}, ) - assert.Equal(t, []string{"traces"}, kindsOf(sources)) + assert.Equal(t, []string{"prompt"}, kindsOf(sources)) assert.Empty(t, unbuildable) } @@ -61,13 +65,32 @@ func TestBuildGenerationSources_InstructionsDoNotSubstituteForTheAgent(t *testin // The agent name travels with the traces source: it is what scopes the query // to this agent's conversations rather than the whole project's. +// +// The agent and its instructions are also sent as sources of their own, because +// the service refuses a request carrying neither a prompt nor an agent. +// Verified against it: traces alone is a 400 naming that requirement, traces +// plus agent is accepted. The prompt is what the agent-seeding retry falls back +// to, so without it a traces run has no way through that failure. func TestBuildGenerationSources_TracesCarryTheAgent(t *testing.T) { sources, _ := BuildGenerationSources( - []string{"traces"}, "support-agent", "", "", &TraceOptions{Days: 7}, + []string{"traces"}, "support-agent", "", "answer support questions", + &TraceOptions{Days: 7}, ) - require.Len(t, sources, 1) - assert.Equal(t, "support-agent", sources[0].AgentName) + assert.Equal(t, []string{"prompt", "agent", "traces"}, kindsOf(sources)) + assert.True(t, HasPromptSource(WithoutAgentSource(sources)), + "dropping the agent must leave something the service still accepts") +} + +// Without an agent the traces source names nothing to read, and nothing the +// service accepts can accompany it, so it is refused here rather than sent. +func TestBuildGenerationSources_TracesNeedAnAgent(t *testing.T) { + sources, unbuildable := BuildGenerationSources( + []string{"traces"}, "", "", "", nil, + ) + + assert.Empty(t, sources) + assert.Equal(t, []string{"traces"}, unbuildable) } // A day window narrows the trace query; it is not what authorizes it. The @@ -78,9 +101,9 @@ func TestBuildGenerationSources_TracesWithoutAWindowAreUnbounded(t *testing.T) { []string{"traces"}, "support-agent", "", "", nil, ) - require.Len(t, sources, 1) - assert.Equal(t, "traces", sources[0].Type) - assert.Zero(t, sources[0].StartTime, + require.Len(t, sources, 2) + assert.Equal(t, "traces", sources[1].Type) + assert.Zero(t, sources[1].StartTime, "an absent window must leave start_time off the wire, not pin it to now") assert.Empty(t, unbuildable) } @@ -90,9 +113,9 @@ func TestBuildGenerationSources_TraceWindowBecomesAStartTime(t *testing.T) { []string{"traces"}, "support-agent", "", "", &TraceOptions{Days: 7}, ) - require.Len(t, sources, 1) + require.Len(t, sources, 2) want := time.Now().AddDate(0, 0, -7).Unix() - assert.InDelta(t, want, sources[0].StartTime, 60) + assert.InDelta(t, want, sources[1].StartTime, 60) } // No --from is no preference, so the plan sends everything it happens to have. From f006f1c5c5a4dd6339acbf391ca41cc2155049c8 Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 11 Aug 2026 04:22:29 +0300 Subject: [PATCH 173/320] Say a rubric has not been generated yet, rather than naming a syscall `init` declares the rubric it plans and names the command that writes it, so running `eval create` before `generate` is an ordering mistake. It read as a broken configuration instead: ERROR: evaluator "support-agent-quality": evaluator source "evals\\evaluators\\support-agent-quality.json": GetFileAttributesEx evals\evaluators\support-agent-quality.json: The system cannot find the file specified. A Windows syscall name, the path twice, once with doubled separators, and nothing about what to do. A missing definition now says what is missing and names the command that produces it. Every other stat failure keeps the old message, because a permission error is not an ordering mistake. --- .../internal/cmd/reconciler.go | 5 +++ .../messages/evaluator_missing_test.go | 37 +++++++++++++++++++ .../internal/messages/messages.go | 18 +++++++++ 3 files changed, 60 insertions(+) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/messages/evaluator_missing_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go index b66f2334883..c4877c617ed 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go @@ -7,6 +7,8 @@ import ( "bufio" "context" "encoding/json" + "errors" + "io/fs" "maps" "os" "reflect" @@ -243,6 +245,9 @@ func (r *evalReconciler) EnsureEvaluator( } if _, err := os.Stat(localPath); err != nil { + if errors.Is(err, fs.ErrNotExist) { + return "", false, messages.EvaluatorNotGeneratedYet(decl.Name, localPath) + } return "", false, messages.EvaluatorSource(localPath, err) } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/messages/evaluator_missing_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/messages/evaluator_missing_test.go new file mode 100644 index 00000000000..15cc1ca92d3 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/messages/evaluator_missing_test.go @@ -0,0 +1,37 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package messages + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +// The bare stat failure underneath this is a Windows syscall name and a path +// with doubled separators, which says nothing about what to do next. Both +// callers wrap it with EvaluatorProblem, so the evaluator is named once. +func TestEvaluatorNotGeneratedYet(t *testing.T) { + err := EvaluatorProblem("support-agent-quality", + EvaluatorNotGeneratedYet("support-agent-quality", `evals\evaluators\support-agent-quality.json`)) + + got := err.Error() + assert.Equal(t, 1, countOccurrences(got, `"support-agent-quality"`), + "the wrapper names the evaluator, so the message must not name it again") + assert.Contains(t, got, "evals/evaluators/support-agent-quality.json", + "the path reads as a path, not as an escaped Windows literal") + assert.NotContains(t, got, `\\`) + assert.Contains(t, got, "azd ai eval generate --evaluator --evaluator-name support-agent-quality", + "the way out is the command that writes the definition") +} + +func countOccurrences(s, sub string) int { + n := 0 + for i := 0; i+len(sub) <= len(s); i++ { + if s[i:i+len(sub)] == sub { + n++ + } + } + return n +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go b/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go index 56b02daa8aa..1a8975506f3 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go @@ -1014,6 +1014,24 @@ func EvaluatorSource(path string, err error) error { return fmt.Errorf("evaluator source %q: %w", path, err) } +// EvaluatorNotGeneratedYet reports a declared evaluator whose definition has +// not been written yet. +// +// `init` declares the rubric it plans and names the command that produces it, +// so reaching a deploy without one is an ordering mistake rather than a broken +// configuration. Said plainly, because the bare stat failure underneath is a +// Windows syscall name and a path with doubled separators. +// +// Both callers wrap this with EvaluatorProblem, which names the evaluator, so +// this does not name it again. +func EvaluatorNotGeneratedYet(evaluator, path string) error { + return fmt.Errorf( + "its definition %s has not been generated yet. "+ + "Run `azd ai eval generate --evaluator --evaluator-name %s` to write it, "+ + "or drop the evaluator from azure.eval.yaml", + filepath.ToSlash(path), evaluator) +} + // CheckingEvaluatorExists reports a failure to tell create from update. func CheckingEvaluatorExists(evaluator string, err error) error { return fmt.Errorf("checking whether evaluator %q exists: %w", evaluator, err) From aff1fa31108e0e5568ce52a2c0ef4e93729dcce3 Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 11 Aug 2026 04:29:15 +0300 Subject: [PATCH 174/320] Answer the way the sibling extension does: TYPE, absent, and not-found Three inconsistencies the bug bash surfaced, each one the eval extension disagreeing with `azd ai dataset` about the same thing. `eval dataset list` printed a FORMAT column that was blank on every row. The service populates `type` (`uri_file`) and leaves `format` empty; the dataset extension learned this already and renamed the column. `show` had the same field wrong. `eval dataset show ` said the dataset "has no versions", which reads as though it exists and is empty. The service answers an unknown name with an empty list rather than a 404, and a dataset cannot exist with no versions, so that list means absent. It now says so, and names the command that lists what is there. DatasetHasNoVersions had no other caller and is removed rather than left as a second way to say the same thing. `run start` against an unknown eval printed the whole 404 that discovered it: the request URL, a rule of eighty dashes, the response block and an error code. `eval show` already answers that input in one line, and this now does too. --- .../azure.ai.evaluations/internal/cmd/dataset.go | 13 +++++++++---- .../azure.ai.evaluations/internal/cmd/run.go | 5 +++++ .../internal/messages/messages.go | 12 +++++++++--- 3 files changed, 23 insertions(+), 7 deletions(-) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset.go index fcf12465434..38e63d7ab71 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset.go @@ -230,13 +230,15 @@ func renderDatasets(cmd *cobra.Command, list *dataset_api.DatasetList) error { } rows := make([][]string, 0, len(list.Value)) for _, d := range list.Value { - rows = append(rows, []string{d.Name, d.Version, d.Format}) + rows = append(rows, []string{d.Name, d.Version, d.Type}) } if len(rows) == 0 { fmt.Fprint(cmd.OutOrStdout(), messages.NoDatasets()) return nil } - return emitTable(cmd.OutOrStdout(), []string{"NAME", "VERSION", "FORMAT"}, rows) + // TYPE, not FORMAT: the service populates type (`uri_file`) and leaves + // format empty, so the column was blank on every row. + return emitTable(cmd.OutOrStdout(), []string{"NAME", "VERSION", "TYPE"}, rows) } func newDatasetShowCommand() *cobra.Command { @@ -265,7 +267,10 @@ func newDatasetShowCommand() *cobra.Command { return messages.ResolvingLatestDatasetVersion(name, err) } if len(list.Value) == 0 { - return messages.DatasetHasNoVersions(name) + // The service answers an unknown name with an empty list + // rather than a 404, and a dataset cannot exist with no + // versions, so this is what "no such dataset" looks like. + return messages.DatasetNotFound(name) } version = dataset_api.LatestVersion(list.Value) } @@ -284,7 +289,7 @@ func newDatasetShowCommand() *cobra.Command { if err := emitDetail(cmd.OutOrStdout(), []field{ {"Name", ds.Name}, {"Version", ds.Version}, - {"Format", ds.Format}, + {"Type", ds.Type}, {"URI", ds.ResolvedBlobURI()}, }); err != nil { return err diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go index 533acc62466..683a4226a77 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go @@ -397,6 +397,11 @@ func (ec *evalContext) reuseDataSourceFromLastRun( ) (*eval_api.EvalRunDataSource, error) { list, err := ec.evalClient.ListOpenAIEvalRuns(ctx, evalID, 1) if err != nil { + if eval_api.IsNotFound(err) { + // The eval itself is missing, which is worth saying plainly rather + // than as forty lines of the 404 that discovered it. + return nil, messages.EvalNotFound(evalID) + } return nil, messages.ReadingPreviousRuns(evalID, err) } if list == nil || len(list.Data) == 0 || list.Data[0].DataSource == nil { diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go b/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go index 1a8975506f3..1cc686e40d1 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go @@ -847,9 +847,15 @@ func ResolvingLatestDatasetVersion(dataset string, err error) error { return fmt.Errorf("resolving the latest version of %q: %w", dataset, err) } -// DatasetHasNoVersions reports a dataset nothing was ever published under. -func DatasetHasNoVersions(dataset string) error { - return fmt.Errorf("dataset %q has no versions", dataset) +// DatasetNotFound reports a name the project does not hold. +// +// The service answers an unknown name with an empty version list rather than a +// 404, and a dataset cannot exist with no versions, so an empty list means the +// dataset is absent rather than empty. +func DatasetNotFound(dataset string) error { + return fmt.Errorf( + "no dataset %q in this project; "+ + "`azd ai eval dataset list` shows the ones there are", dataset) } // DatasetVersionNotFoundWithHint reports a dataset version the project does not hold. From e1180f79e3099d9b361c1aaf9bccb1b9a00e79d6 Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 11 Aug 2026 04:51:54 +0300 Subject: [PATCH 175/320] Release 1.0.0-beta.4 --- cli/azd/extensions/azure.ai.evaluations/extension.yaml | 2 +- cli/azd/extensions/azure.ai.evaluations/version.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cli/azd/extensions/azure.ai.evaluations/extension.yaml b/cli/azd/extensions/azure.ai.evaluations/extension.yaml index 7b5c94df867..d19e01d1aef 100644 --- a/cli/azd/extensions/azure.ai.evaluations/extension.yaml +++ b/cli/azd/extensions/azure.ai.evaluations/extension.yaml @@ -5,7 +5,7 @@ displayName: Foundry evaluations (Beta) description: Define and run Foundry evaluations from your terminal. (Beta) usage: azd ai eval [options] # NOTE: Make sure version.txt is in sync with this version. -version: 1.0.0-beta.3 +version: 1.0.0-beta.4 requiredAzdVersion: ">=1.27.1" language: go capabilities: diff --git a/cli/azd/extensions/azure.ai.evaluations/version.txt b/cli/azd/extensions/azure.ai.evaluations/version.txt index b0a2ffd0f5f..1bdfe3ec1a2 100644 --- a/cli/azd/extensions/azure.ai.evaluations/version.txt +++ b/cli/azd/extensions/azure.ai.evaluations/version.txt @@ -1 +1 @@ -1.0.0-beta.3 +1.0.0-beta.4 From cfe3aae9cf1d10d6a615fafd290ea1316249e69f Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 11 Aug 2026 05:04:29 +0300 Subject: [PATCH 176/320] Print a path someone can copy back `%q` escapes a Windows separator, so every message naming a file printed it with the separators doubled: ERROR: reading eval config "evals\\azure.eval.yaml": ... A reader who copies that into a shell gets a path that does not exist, which is the opposite of what naming the file is for. Paths now go through filepath.ToSlash before they are quoted; forward slashes are accepted by the Windows APIs and by PowerShell, so the quoted form stays usable. Applied to every constructor that names a filesystem path, and written down in the package's conventions so the next one follows it. The URL path in InvalidRequestPath is left alone, being a URL rather than a file. --- .../internal/messages/messages.go | 32 ++++++++------- .../internal/messages/paths_test.go | 40 +++++++++++++++++++ 2 files changed, 58 insertions(+), 14 deletions(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/messages/paths_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go b/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go index 1cc686e40d1..74fcc969a7e 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go @@ -11,6 +11,9 @@ // Lowercase, no trailing period: azd renders them after "ERROR: ". // - A name the user chose is quoted with %q; an identifier the service // assigned is not, because it is already unmistakable. +// - A filesystem path goes through filepath.ToSlash first. %q escapes a +// Windows separator, so `evals\eval.yaml` prints as "evals\\eval.yaml" and +// a reader who copies it back gets a path that does not exist. // - Progress and success lines are sentences with a capital and no period. // - A printed line carries its own newlines, so a call site is a bare Fprint. // - Nothing here decides *whether* to print. That stays at the call site. @@ -53,7 +56,7 @@ func EvalHasNoDataset(eval string) error { // DatasetFileEmpty reports a local dataset file that parsed but held no rows. func DatasetFileEmpty(path string) error { - return fmt.Errorf("dataset file %q has no rows", path) + return fmt.Errorf("dataset file %q has no rows", filepath.ToSlash(path)) } // TracesNeedAgentName reports a trace-backed eval that does not say whose @@ -533,12 +536,12 @@ func GenerationModelRequired() error { // ReadingInstructionFile reports an --agent-instruction-file that would not read. func ReadingInstructionFile(path string, err error) error { - return fmt.Errorf("reading --agent-instruction-file %q: %w", path, err) + return fmt.Errorf("reading --agent-instruction-file %q: %w", filepath.ToSlash(path), err) } // InstructionFileEmpty reports an --agent-instruction-file with nothing in it. func InstructionFileEmpty(path string) error { - return fmt.Errorf("--agent-instruction-file %q is empty", path) + return fmt.Errorf("--agent-instruction-file %q is empty", filepath.ToSlash(path)) } // ReadingInstructions reports a declared instructions file that would not read. @@ -808,13 +811,14 @@ func JSONLNoRows(path string) error { // ReadingFromFile reports a --from-file that would not stat. func ReadingFromFile(path string, err error) error { - return fmt.Errorf("reading --from-file %q: %w", path, err) + return fmt.Errorf("reading --from-file %q: %w", filepath.ToSlash(path), err) } // FromFileMustBeJSONL reports a --from-file that is not a dataset. func FromFileMustBeJSONL(path string) error { return fmt.Errorf( - "--from-file must be a .jsonl file or a directory containing one, got %q", path) + "--from-file must be a .jsonl file or a directory containing one, got %q", + filepath.ToSlash(path)) } // RegisteringDataset reports the service refusing to publish the dataset. @@ -887,7 +891,7 @@ func DatasetProblem(dataset string, err error) error { // DatasetSource reports a declared source that is not on disk. func DatasetSource(path string, err error) error { - return fmt.Errorf("dataset source %q: %w", path, err) + return fmt.Errorf("dataset source %q: %w", filepath.ToSlash(path), err) } // DatasetNotLocalNorFound reports a source-less dataset the project rejected. @@ -1017,7 +1021,7 @@ func EvaluatorProblem(evaluator string, err error) error { // EvaluatorSource reports a declared source that is not on disk. func EvaluatorSource(path string, err error) error { - return fmt.Errorf("evaluator source %q: %w", path, err) + return fmt.Errorf("evaluator source %q: %w", filepath.ToSlash(path), err) } // EvaluatorNotGeneratedYet reports a declared evaluator whose definition has @@ -1303,7 +1307,7 @@ func EvalDeleted(evalID string) string { // Hashing reports a local artifact that could not be fingerprinted. func Hashing(path string, err error) error { - return fmt.Errorf("hashing %q: %w", path, err) + return fmt.Errorf("hashing %q: %w", filepath.ToSlash(path), err) } // HashingEval reports an eval declaration that could not be fingerprinted. @@ -1685,12 +1689,12 @@ func AmbiguousEvalConfig(current, legacy string) error { // ReadingEvalConfig reports a configuration file that would not read. func ReadingEvalConfig(path string, err error) error { - return fmt.Errorf("reading eval config %q: %w", path, err) + return fmt.Errorf("reading eval config %q: %w", filepath.ToSlash(path), err) } // ParsingEvalConfig reports a configuration file that would not parse. func ParsingEvalConfig(path string, err error) error { - return fmt.Errorf("parsing eval config %q: %w", path, err) + return fmt.Errorf("parsing eval config %q: %w", filepath.ToSlash(path), err) } // SerializingEvalConfig reports a configuration that would not serialize. @@ -1700,7 +1704,7 @@ func SerializingEvalConfig(err error) error { // WritingEvalConfig reports a configuration file that would not be written. func WritingEvalConfig(path string, err error) error { - return fmt.Errorf("writing eval config %q: %w", path, err) + return fmt.Errorf("writing eval config %q: %w", filepath.ToSlash(path), err) } // ErrAmbiguousAgentService reports that a target name matched more than one service. @@ -1833,17 +1837,17 @@ func FlagRequired(name string) error { // Creating reports a directory or file that could not be created. func Creating(path string, err error) error { - return fmt.Errorf("creating %q: %w", path, err) + return fmt.Errorf("creating %q: %w", filepath.ToSlash(path), err) } // Serializing reports a value that could not be written out. func Serializing(path string, err error) error { - return fmt.Errorf("serializing %q: %w", path, err) + return fmt.Errorf("serializing %q: %w", filepath.ToSlash(path), err) } // Writing reports a file that could not be written. func Writing(path string, err error) error { - return fmt.Errorf("writing %q: %w", path, err) + return fmt.Errorf("writing %q: %w", filepath.ToSlash(path), err) } // ReadingPath reports a file or directory that could not be read. diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/messages/paths_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/messages/paths_test.go new file mode 100644 index 00000000000..1cc641126a7 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/messages/paths_test.go @@ -0,0 +1,40 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package messages + +import ( + "errors" + "strings" + "testing" + + "github.com/stretchr/testify/assert" +) + +// %q escapes a Windows separator, so a path printed straight through comes back +// as "evals\\azure.eval.yaml". A reader who copies that into a shell gets a path +// that does not exist, which is the opposite of what naming the file is for. +func TestPathsInMessagesStayCopyable(t *testing.T) { + boom := errors.New("no such file") + + for _, err := range []error{ + ReadingEvalConfig(`evals\azure.eval.yaml`, boom), + ParsingEvalConfig(`evals\azure.eval.yaml`, boom), + WritingEvalConfig(`evals\azure.eval.yaml`, boom), + ReadingFromFile(`evals\datasets\d.jsonl`, boom), + FromFileMustBeJSONL(`evals\datasets`), + DatasetSource(`evals\datasets\d.jsonl`, boom), + EvaluatorSource(`evals\evaluators\e.json`, boom), + DatasetFileEmpty(`evals\datasets\d.jsonl`), + ReadingInstructionFile(`docs\instructions.md`, boom), + InstructionFileEmpty(`docs\instructions.md`), + Hashing(`evals\evaluators\e.json`, boom), + Creating(`evals\out`, boom), + Serializing(`evals\out\x.json`, boom), + Writing(`evals\out\x.json`, boom), + } { + got := err.Error() + assert.NotContains(t, got, `\\`, "a doubled separator is not a path anyone can use: %s", got) + assert.Truef(t, strings.Contains(got, "/"), "the path should read with forward slashes: %s", got) + } +} From 6993e6a54780ea39e57adabeaaf02e064d2e5299 Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 11 Aug 2026 05:08:53 +0300 Subject: [PATCH 177/320] Name a mistyped key, not the Go type it was not found in A misspelled key in azure.eval.yaml reported: parsing eval config "evals\\azure.eval.yaml": yaml: unmarshal errors: line 7: field evaulators not found in type project.Eval `project.Eval` is an implementation detail, and hand-editing the file is the documented way to use one, so the reader is being shown a Go type to explain their own typo. It now reads: parsing eval config "evals/azure.eval.yaml": line 7: unknown key "evaulators"; did you mean "evaluators"? The candidates come from the declarations' own yaml tags, so they cannot drift from what the file actually accepts. A suggestion is only offered within a third of the key's length, because a confidently wrong one is worse than none. Anything that is not an unknown-key failure passes through untouched. --- .../internal/project/eval_config_keys.go | 113 ++++++++++++++++++ .../internal/project/eval_config_keys_test.go | 46 +++++++ .../internal/project/eval_config_store.go | 2 +- 3 files changed, 160 insertions(+), 1 deletion(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_keys.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_keys_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_keys.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_keys.go new file mode 100644 index 00000000000..9b8c2156c63 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_keys.go @@ -0,0 +1,113 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package project + +import ( + "fmt" + "reflect" + "regexp" + "strings" +) + +// goTypeInField matches what yaml.KnownFields reports for an unrecognised key: +// `line 7: field evaulators not found in type project.Eval`. The Go type is an +// implementation detail, and the reader is editing a YAML file by hand, which +// is the documented way to use one. +var goTypeInField = regexp.MustCompile(`field (\S+) not found in type (\S+)`) + +// explainUnknownKeys rewrites a decode failure into the file's own vocabulary, +// naming the near-miss when there is one. +func explainUnknownKeys(err error) error { + text := err.Error() + if !goTypeInField.MatchString(text) { + return err + } + + lines := make([]string, 0, 4) + for _, line := range strings.Split(text, "\n") { + m := goTypeInField.FindStringSubmatch(line) + if m == nil { + continue + } + key, goType := m[1], m[2] + rewritten := fmt.Sprintf("unknown key %q", key) + if near := nearestKey(key, keysOfType(goType)); near != "" { + rewritten += fmt.Sprintf(`; did you mean %q?`, near) + } + // The yaml prefix carries the line number, which is the useful half. + if prefix, _, ok := strings.Cut(strings.TrimSpace(line), ": field "); ok { + rewritten = prefix + ": " + rewritten + } + lines = append(lines, rewritten) + } + if len(lines) == 0 { + return err + } + return fmt.Errorf("%s", strings.Join(lines, "\n")) +} + +// keysOfType lists the YAML keys a declaration accepts. +func keysOfType(goType string) []string { + var v any + switch goType { + case "project.EvalConfig": + v = EvalConfig{} + case "project.Eval": + v = Eval{} + case "project.DatasetDecl": + v = DatasetDecl{} + case "project.EvaluatorDecl": + v = EvaluatorDecl{} + case "project.Target": + v = Target{} + case "project.SourceDecl": + v = SourceDecl{} + default: + return nil + } + + t := reflect.TypeOf(v) + keys := make([]string, 0, t.NumField()) + for i := 0; i < t.NumField(); i++ { + tag, _, _ := strings.Cut(t.Field(i).Tag.Get("yaml"), ",") + if tag != "" && tag != "-" { + keys = append(keys, tag) + } + } + return keys +} + +// nearestKey returns the closest known key, or "" when nothing is close enough +// to be worth suggesting. A third of the length is the budget, so `evaulators` +// finds `evaluators` while `banana` suggests nothing. +func nearestKey(key string, known []string) string { + best, bestDist := "", len(key)/3+1 + for _, k := range known { + if d := editDistance(key, k); d < bestDist { + best, bestDist = k, d + } + } + return best +} + +// editDistance is Levenshtein over two short identifiers. +func editDistance(a, b string) int { + prev := make([]int, len(b)+1) + curr := make([]int, len(b)+1) + for j := range prev { + prev[j] = j + } + for i := 1; i <= len(a); i++ { + curr[0] = i + for j := 1; j <= len(b); j++ { + cost := 1 + if a[i-1] == b[j-1] { + cost = 0 + } + curr[j] = min(min(curr[j-1]+1, prev[j]+1), prev[j-1]+cost) + } + prev, curr = curr, prev + } + return prev[len(b)] +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_keys_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_keys_test.go new file mode 100644 index 00000000000..b8e97117a6d --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_keys_test.go @@ -0,0 +1,46 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package project + +import ( + "errors" + "testing" + + "github.com/stretchr/testify/assert" +) + +// Hand-editing the file is the documented way to use one, so a mistyped key has +// to read as a mistyped key rather than as a Go type the reader has never seen. +func TestExplainUnknownKeys(t *testing.T) { + got := explainUnknownKeys(errors.New( + "yaml: unmarshal errors:\n line 7: field evaulators not found in type project.Eval")).Error() + + assert.Contains(t, got, `unknown key "evaulators"`) + assert.Contains(t, got, `did you mean "evaluators"?`) + assert.Contains(t, got, "line 7", "the line number is the useful half of what yaml said") + assert.NotContains(t, got, "project.Eval", "the Go type is an implementation detail") +} + +// Nothing close enough is worth suggesting; a wrong suggestion is worse than +// none. +func TestExplainUnknownKeys_NoNearMiss(t *testing.T) { + got := explainUnknownKeys(errors.New( + "yaml: unmarshal errors:\n line 3: field banana not found in type project.Eval")).Error() + + assert.Contains(t, got, `unknown key "banana"`) + assert.NotContains(t, got, "did you mean") +} + +// Anything that is not an unknown-key failure is passed through untouched. +func TestExplainUnknownKeys_LeavesOtherErrors(t *testing.T) { + original := errors.New("yaml: line 2: did not find expected key") + assert.Equal(t, original, explainUnknownKeys(original)) +} + +func TestKeysOfTypeCoversTheDeclarations(t *testing.T) { + assert.Contains(t, keysOfType("project.Eval"), "evaluators") + assert.Contains(t, keysOfType("project.EvalConfig"), "datasets") + assert.Contains(t, keysOfType("project.DatasetDecl"), "source") + assert.Empty(t, keysOfType("project.Unknown")) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_store.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_store.go index 979587913bc..0a48c00d978 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_store.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_store.go @@ -112,7 +112,7 @@ func LoadEvalConfig(path string) (*EvalConfig, error) { if errors.Is(err, io.EOF) { return &cfg, nil } - return nil, messages.ParsingEvalConfig(path, err) + return nil, messages.ParsingEvalConfig(path, explainUnknownKeys(err)) } return &cfg, nil } From 8ceafa0b98195dda1e8dbb80df8fc6e7ef770d37 Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 11 Aug 2026 05:20:08 +0300 Subject: [PATCH 178/320] Release 1.0.0-beta.5 --- cli/azd/extensions/azure.ai.evaluations/extension.yaml | 2 +- cli/azd/extensions/azure.ai.evaluations/version.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cli/azd/extensions/azure.ai.evaluations/extension.yaml b/cli/azd/extensions/azure.ai.evaluations/extension.yaml index d19e01d1aef..623cf383dc6 100644 --- a/cli/azd/extensions/azure.ai.evaluations/extension.yaml +++ b/cli/azd/extensions/azure.ai.evaluations/extension.yaml @@ -5,7 +5,7 @@ displayName: Foundry evaluations (Beta) description: Define and run Foundry evaluations from your terminal. (Beta) usage: azd ai eval [options] # NOTE: Make sure version.txt is in sync with this version. -version: 1.0.0-beta.4 +version: 1.0.0-beta.5 requiredAzdVersion: ">=1.27.1" language: go capabilities: diff --git a/cli/azd/extensions/azure.ai.evaluations/version.txt b/cli/azd/extensions/azure.ai.evaluations/version.txt index 1bdfe3ec1a2..ae27bfcaecc 100644 --- a/cli/azd/extensions/azure.ai.evaluations/version.txt +++ b/cli/azd/extensions/azure.ai.evaluations/version.txt @@ -1 +1 @@ -1.0.0-beta.4 +1.0.0-beta.5 From 7e02ff078d578a1f035e82545d3c782e204059a0 Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 11 Aug 2026 05:39:17 +0300 Subject: [PATCH 179/320] Pin the live tests to what the service does now Two live assertions had gone stale. `dataset versions list` was pinned to a FORMAT column. The service populates `type` and leaves `format` empty, so that column was blank on every row; the listing now prints TYPE and the test follows it. `run delete` was pinned to the service accepting a DELETE without removing anything, which the test said in as many words: "the removal itself is not asserted, because it does not happen". It does now, promptly enough that the DELETE can answer 404, so the run is gone by the following read. The test's stated purpose is that the command reaches the right resource, so it now accepts either outcome and asserts the answer is about that run. --- .../tests/cli/dataset_test.go | 4 ++- .../tests/cli/run_ops_test.go | 31 ++++++++++++------- 2 files changed, 23 insertions(+), 12 deletions(-) diff --git a/cli/azd/extensions/azure.ai.evaluations/tests/cli/dataset_test.go b/cli/azd/extensions/azure.ai.evaluations/tests/cli/dataset_test.go index efe36c604b6..94d7807a3c7 100644 --- a/cli/azd/extensions/azure.ai.evaluations/tests/cli/dataset_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/tests/cli/dataset_test.go @@ -90,7 +90,9 @@ func TestCLIDatasetList(t *testing.T) { t.Run("table", func(t *testing.T) { r := requireSuccess(t, run(t, "dataset", "versions", "list", ds.Name)) - for _, header := range []string{"NAME", "VERSION", "FORMAT"} { + // TYPE, not FORMAT. The service populates `type` (`uri_file`) and leaves + // `format` empty, so the column this once pinned was blank on every row. + for _, header := range []string{"NAME", "VERSION", "TYPE"} { require.Containsf(t, r.Stdout, header, "the listing lost its %s column", header) } require.Contains(t, r.Stdout, ds.Name) diff --git a/cli/azd/extensions/azure.ai.evaluations/tests/cli/run_ops_test.go b/cli/azd/extensions/azure.ai.evaluations/tests/cli/run_ops_test.go index 58faab218e4..7b2783a0ef0 100644 --- a/cli/azd/extensions/azure.ai.evaluations/tests/cli/run_ops_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/tests/cli/run_ops_test.go @@ -143,12 +143,13 @@ func TestCLIRunCancelAndDelete(t *testing.T) { // Delete is covered as far as the service honours it. // - // The removal itself is not asserted, because it does not happen: the - // service accepts the DELETE and the run is still readable by id and still - // in the listing minutes later. What is asserted instead is that the - // command reaches the right resource — a real run is accepted, an unknown - // one is refused — which is the part that would break if the route or the - // id handling regressed. + // The removal itself is not asserted, because whether it happens is the + // service's to decide and it has changed under this test once already: it + // used to accept the DELETE and leave the run readable minutes later, and it + // now reaps a cancelled run promptly enough that the DELETE can even answer + // 404. What is asserted is that the command reaches the right resource — a + // real run is accepted, an unknown one is refused — which is the part that + // would break if the route or the id handling regressed. t.Run("an in-flight run is cancelled, and the delete is accepted", func(t *testing.T) { runID := startCancellableRun(t, f) @@ -166,11 +167,19 @@ func TestCLIRunCancelAndDelete(t *testing.T) { require.Contains(t, deleted.Stdout, "Deleted run") require.Contains(t, deleted.Stdout, runID) - still := requireSuccess(t, run(t, "run", "show", runID, "--eval", f.EvalID, "-o", "json")) - var survivor runSummary - still.JSON(t, &survivor) - t.Logf("the run is still readable after a successful delete (status %q); "+ - "the service accepts the request without removing anything", survivor.Status) + // Either outcome is the service's prerogative; what matters is that the + // answer is about this run and not a failure of some other kind. + still := run(t, "run", "show", runID, "--eval", f.EvalID, "-o", "json") + if still.ExitCode == 0 { + var survivor runSummary + still.JSON(t, &survivor) + t.Logf("still readable after delete (status %q); the service accepted "+ + "the request without removing anything", survivor.Status) + } else { + require.Contains(t, still.Combined(), runID, + "a read of a deleted run must still name the run it could not find") + t.Log("gone after delete; the service removed it") + } }) // Deleting is not undoable, so the id is required rather than defaulted to From 6e558e7d97d70238bb3c6010d85789153d004b45 Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 11 Aug 2026 06:09:06 +0300 Subject: [PATCH 180/320] Tell a reader holding the agents extension's eval.yaml where it belongs `azd ai agent eval` writes its own eval.yaml, with a completely different shape from this extension's. Pointed at one, this reported a near-miss suggestion per key: line 1: unknown key "name" line 2: unknown key "agent" line 6: unknown key "dataset"; did you mean "datasets"? ... Each suggestion is individually reasonable and collectively wrong: it walks the reader into rewriting another tool's file a line at a time. When nothing at the top level is recognised the file is not a malformed one of ours, and saying so once, with the command that does read it, is the whole of the answer. Only top-level rejections count toward that. The agents extension's `evaluators` key happens to share a name with ours, so the mismatch inside it reports against the nested type; treating that as disqualifying would have kept the old behaviour for exactly the file this is for. Also finishes the path rendering started earlier: nine more messages named a file without normalising the separators. --- .../internal/messages/messages.go | 20 +++++------ .../internal/project/eval_config_keys.go | 34 +++++++++++++++++++ .../internal/project/eval_config_keys_test.go | 31 +++++++++++++++++ 3 files changed, 75 insertions(+), 10 deletions(-) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go b/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go index 74fcc969a7e..60e3454fe5b 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go @@ -84,7 +84,7 @@ func DatasetOverrideNeedsDeclaredEval() error { // DatasetNotInCatalog reports a --dataset the configuration does not declare. func DatasetNotInCatalog(dataset, configPath string) error { - return fmt.Errorf("dataset %q is not in the catalog in %s", dataset, configPath) + return fmt.Errorf("dataset %q is not in the catalog in %s", dataset, filepath.ToSlash(configPath)) } // DatasetHasUnregisteredEdits reports local rows no deployed version holds. @@ -214,7 +214,7 @@ func EvalNotDeployedYet(eval string) error { func NoEvalNamedOrDeclared(configPath string) error { return fmt.Errorf( "no eval was named and none is declared in %s; pass --eval with a name or an id", - configPath) + filepath.ToSlash(configPath)) } // NoEvalGiven reports a command run outside a project with no eval id. @@ -361,7 +361,7 @@ func OutputItemEmpty() error { // NotARegularFile reports an --output-file that names a directory or a device. func NotARegularFile(path string) error { - return fmt.Errorf("%s is not a regular file, so it will not be overwritten", path) + return fmt.Errorf("%s is not a regular file, so it will not be overwritten", filepath.ToSlash(path)) } // CannotWriteInDirectory reports a destination directory that cannot be written @@ -369,7 +369,7 @@ func NotARegularFile(path string) error { // temporary file the writer chose, which the caller never asked for. func CannotWriteInDirectory(dir string, err error) error { if errors.Is(err, fs.ErrNotExist) { - return fmt.Errorf("%s does not exist", dir) + return fmt.Errorf("%s does not exist", filepath.ToSlash(dir)) } return fmt.Errorf("cannot write in %s: %w", dir, err) } @@ -551,7 +551,7 @@ func ReadingInstructions(named string, err error) error { // SeedingFromFile names the local file generation was seeded from. func SeedingFromFile(path string) string { - return fmt.Sprintf(" Seeding generation from %s.\n", path) + return fmt.Sprintf(" Seeding generation from %s.\n", filepath.ToSlash(path)) } // SeedingFromAgent names the agent whose published instructions seeded generation. @@ -677,7 +677,7 @@ func ReattachToJob(selector, jobID string) string { // WroteArtifact reports where a generated artifact landed. func WroteArtifact(path string) string { - return fmt.Sprintf("%s Downloaded %s\n", DoneMark, path) + return fmt.Sprintf("%s Downloaded %s\n", DoneMark, filepath.ToSlash(path)) } // ArtifactExists reports a generation that would overwrite a checked-in file. @@ -806,7 +806,7 @@ func JSONLRowEmpty(path string, line int) error { // JSONLNoRows reports a dataset file with nothing in it to evaluate. func JSONLNoRows(path string) error { - return fmt.Errorf("%s has no rows to evaluate", path) + return fmt.Errorf("%s has no rows to evaluate", filepath.ToSlash(path)) } // ReadingFromFile reports a --from-file that would not stat. @@ -939,7 +939,7 @@ func DatasetFileHasNoRows(name string) error { // NoJSONLInDirectory reports an upload directory holding no dataset. func NoJSONLInDirectory(dir string) error { - return fmt.Errorf("no .jsonl file found in %s", dir) + return fmt.Errorf("no .jsonl file found in %s", filepath.ToSlash(dir)) } // ReadingDatasetFromDir reports the upload failing to gather the local rows. @@ -1498,12 +1498,12 @@ func FurtherNextStep(step string) string { // CreatedCatalogFile reports a configuration created to hold a catalog entry. func CreatedCatalogFile(configPath string) string { - return fmt.Sprintf("%s Created %s with the catalog entry\n", DoneMark, configPath) + return fmt.Sprintf("%s Created %s with the catalog entry\n", DoneMark, filepath.ToSlash(configPath)) } // AddedToCatalog reports a generated artifact recorded in the configuration. func AddedToCatalog(kind, artifact, configPath string) string { - return fmt.Sprintf("%s Added %s %s to %s\n", DoneMark, kind, artifact, configPath) + return fmt.Sprintf("%s Added %s %s to %s\n", DoneMark, kind, artifact, filepath.ToSlash(configPath)) } // ArtifactDescription names a catalogued artifact, with its version when there is one. diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_keys.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_keys.go index 9b8c2156c63..bd796543807 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_keys.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_keys.go @@ -4,6 +4,7 @@ package project import ( + "errors" "fmt" "reflect" "regexp" @@ -24,6 +25,14 @@ func explainUnknownKeys(err error) error { return err } + // Nothing at the top level was recognised, so this is another tool's file + // rather than a typo in one of ours. `azd ai agent eval` writes an eval.yaml + // of its own, and suggesting a near-miss for each of its keys in turn would + // walk the reader into rewriting it a line at a time. + if topLevelKeysAllUnknown(text) { + return errUnrecognisedEvalConfig + } + lines := make([]string, 0, 4) for _, line := range strings.Split(text, "\n") { m := goTypeInField.FindStringSubmatch(line) @@ -47,6 +56,31 @@ func explainUnknownKeys(err error) error { return fmt.Errorf("%s", strings.Join(lines, "\n")) } +var errUnrecognisedEvalConfig = errors.New( + "none of this file's top-level keys are ones an eval configuration declares, " + + "so this is not one. `azd ai agent eval` writes an eval.yaml of its own with " + + "a different shape, and runs it with `azd ai agent eval run`") + +// topLevelKeysAllUnknown reports a file whose top-level shape is not this one. +// +// Only top-level rejections count. A nested one does not disqualify the check: +// another tool's file can still have a key named like one of ours, and the +// mismatch inside it then reports against the nested type rather than the +// config. The threshold is the number of keys a configuration declares, so one +// stray key beside recognised ones stays a typo. +func topLevelKeysAllUnknown(text string) bool { + known := keysOfType("project.EvalConfig") + rejected := 0 + for _, line := range strings.Split(text, "\n") { + m := goTypeInField.FindStringSubmatch(line) + if m == nil || m[2] != "project.EvalConfig" { + continue + } + rejected++ + } + return rejected > 0 && rejected >= len(known) +} + // keysOfType lists the YAML keys a declaration accepts. func keysOfType(goType string) []string { var v any diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_keys_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_keys_test.go index b8e97117a6d..0a26840131d 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_keys_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_keys_test.go @@ -44,3 +44,34 @@ func TestKeysOfTypeCoversTheDeclarations(t *testing.T) { assert.Contains(t, keysOfType("project.DatasetDecl"), "source") assert.Empty(t, keysOfType("project.Unknown")) } + +// `azd ai agent eval` writes an eval.yaml of its own with an entirely different +// shape. Suggesting a near-miss for each of its keys in turn would walk the +// reader into rewriting another tool's file a line at a time. +// +// Its `evaluators` key happens to share a name with ours, so the mismatch +// inside it reports against the nested type. That must not disqualify the check. +func TestExplainUnknownKeys_AnotherToolsFile(t *testing.T) { + got := explainUnknownKeys(errors.New( + "yaml: unmarshal errors:\n" + + " line 1: field name not found in type project.EvalConfig\n" + + " line 2: field agent not found in type project.EvalConfig\n" + + " line 6: field dataset not found in type project.EvalConfig\n" + + " line 13: field local_uri not found in type project.EvaluatorDecl\n" + + " line 14: field options not found in type project.EvalConfig\n" + + " line 16: field max_samples not found in type project.EvalConfig")).Error() + + assert.Contains(t, got, "not one") + assert.Contains(t, got, "azd ai agent eval run", "the reader is told where the file does belong") + assert.NotContains(t, got, "did you mean", + "suggesting a fix per key sends the reader down the wrong path entirely") +} + +// One stray key beside recognised ones is still a typo, so the suggestion stands. +func TestExplainUnknownKeys_OneStrayTopLevelKey(t *testing.T) { + got := explainUnknownKeys(errors.New( + "yaml: unmarshal errors:\n line 1: field datsets not found in type project.EvalConfig")).Error() + + assert.Contains(t, got, `did you mean "datasets"?`) + assert.NotContains(t, got, "does not look like") +} From 1cec1ac435d89020ec79697d16895273c3560951 Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 11 Aug 2026 06:20:32 +0300 Subject: [PATCH 181/320] Release 1.0.0-beta.6 --- cli/azd/extensions/azure.ai.evaluations/extension.yaml | 2 +- cli/azd/extensions/azure.ai.evaluations/version.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cli/azd/extensions/azure.ai.evaluations/extension.yaml b/cli/azd/extensions/azure.ai.evaluations/extension.yaml index 623cf383dc6..5ca89bf95a1 100644 --- a/cli/azd/extensions/azure.ai.evaluations/extension.yaml +++ b/cli/azd/extensions/azure.ai.evaluations/extension.yaml @@ -5,7 +5,7 @@ displayName: Foundry evaluations (Beta) description: Define and run Foundry evaluations from your terminal. (Beta) usage: azd ai eval [options] # NOTE: Make sure version.txt is in sync with this version. -version: 1.0.0-beta.5 +version: 1.0.0-beta.6 requiredAzdVersion: ">=1.27.1" language: go capabilities: diff --git a/cli/azd/extensions/azure.ai.evaluations/version.txt b/cli/azd/extensions/azure.ai.evaluations/version.txt index ae27bfcaecc..6c3924b6842 100644 --- a/cli/azd/extensions/azure.ai.evaluations/version.txt +++ b/cli/azd/extensions/azure.ai.evaluations/version.txt @@ -1 +1 @@ -1.0.0-beta.5 +1.0.0-beta.6 From ba6928750ca83123a8fcde4944c545565b0caf79 Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 11 Aug 2026 06:51:20 +0300 Subject: [PATCH 182/320] Keep the whole rubric on the way to disk, and say what create actually did MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two defects found running the documented hero scenarios end to end. `generate` wrote the rubric by re-marshalling a struct that models only `type` and `dimensions`, so `pass_threshold` was dropped on the way to disk. The downloaded file therefore differed from the version that had just been published, and the next `eval create` republished it — silently without a threshold. Version 1 graded at 0.50 and version 2 graded at nothing, from an edit nobody made. The definition is now written through as it arrived, so a field the service adds later cannot be lost the same way either. `eval create` then reported `Created eval` every time, including when it had created nothing and returned the id it was already given. The scenarios ask the reader to confirm that a second create with no edits publishes nothing, and there was no output that could show it. It now reports each artifact as published or unchanged, and distinguishes a reused eval from a new one by comparing the recorded id. Scenario 4 now reads as the spec describes it: Dataset support-agent-regression is unchanged at version 1.0 Published evaluator support-agent-quality version 2 (-) Skipped: Eval support-agent-regression-eval is unchanged (eval_...) --- .../internal/cmd/eval_group.go | 31 +++++++++- .../internal/cmd/generate.go | 30 +++++++--- .../internal/cmd/generate_rubric_test.go | 57 +++++++++++++++++++ .../internal/messages/messages.go | 10 ++++ 4 files changed, 117 insertions(+), 11 deletions(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_rubric_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/eval_group.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/eval_group.go index ba8dcb55694..e4421ef1f5e 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/eval_group.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/eval_group.go @@ -70,6 +70,20 @@ func newEvalCreateCommand() *cobra.Command { } reconciler := &evalReconciler{ec: ec} + out := cmd.OutOrStdout() + // Reported per artifact, because "publishes nothing when nothing + // changed" is the contract a reader is checking here and a single + // closing line cannot show it. Silent under -o json. + say := func(kind, name, version string, changed bool) { + if isJSON(cmd) { + return + } + if changed { + fmt.Fprintln(out, messages.PublishedVersion(kind, name, version)) + } else { + fmt.Fprintln(out, messages.UnchangedAtVersion(kind, name, version)) + } + } // The eval names its dataset and evaluators, and the service resolves // those names when the eval is created, so they have to be published @@ -78,9 +92,11 @@ func newEvalCreateCommand() *cobra.Command { // the service without a full deploy. baseDir := filepath.Dir(path) if decl, ok := cfg.DatasetDeclaration(eval.Dataset); ok { - if _, _, err := reconciler.EnsureDataset(ctx, *decl, datasetPath); err != nil { + version, changed, err := reconciler.EnsureDataset(ctx, *decl, datasetPath) + if err != nil { return messages.DatasetProblem(decl.Name, err) } + say("dataset", decl.Name, version, changed) } for _, ref := range eval.Evaluators { decl, ok := cfg.EvaluatorDeclaration(ref.Evaluator) @@ -92,11 +108,16 @@ func newEvalCreateCommand() *cobra.Command { if !filepath.IsAbs(local) { local = filepath.Join(baseDir, local) } - if _, _, err := reconciler.EnsureEvaluator(ctx, *decl, local); err != nil { + version, changed, err := reconciler.EnsureEvaluator(ctx, *decl, local) + if err != nil { return messages.EvaluatorProblem(decl.Name, err) } + say("evaluator", decl.Name, version, changed) } + // Read before the call so a reused eval is not announced as a new one. + // An eval is immutable, so the same id back means nothing was created. + existing := ec.recordedEvalID(ctx, eval.Name) id, err := reconciler.EnsureEval(ctx, *eval, datasetPath) if err != nil { return err @@ -107,7 +128,11 @@ func newEvalCreateCommand() *cobra.Command { "id": id, "name": eval.Name, }) } - fmt.Fprint(cmd.OutOrStdout(), messages.EvalCreated(eval.Name, id)) + if id == existing { + fmt.Fprint(out, messages.EvalUnchanged(eval.Name, id)) + } else { + fmt.Fprint(out, messages.EvalCreated(eval.Name, id)) + } return nil }, } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go index fde26ef1a92..6c7aa3c31ce 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go @@ -4,6 +4,7 @@ package cmd import ( + "bytes" "context" "encoding/json" "errors" @@ -421,8 +422,14 @@ func (ec *evalContext) pollGeneration( return poller.Poll(ctx) } -// writeRubric persists only the rubric dimensions so the developer can edit -// weights and descriptions and publish a new version. +// writeRubric persists the rubric so the developer can edit weights and +// descriptions and publish a new version. +// +// The definition is written through as it arrived rather than re-marshalled +// from a struct. Re-marshalling keeps only the fields the struct models, and +// dropped pass_threshold: the file then differed from the version that had just +// been published, so the next deploy republished it, silently without a +// threshold. Anything the service adds later would have been lost the same way. func writeRubric(path string, result json.RawMessage) error { if len(result) == 0 { return messages.RubricJobReturnedNoResult() @@ -431,13 +438,20 @@ func writeRubric(path string, result json.RawMessage) error { return messages.Creating(filepath.Dir(path), err) } - var parsed eval_api.EvaluatorResult - if err := json.Unmarshal(result, &parsed); err == nil && len(parsed.Definition.Dimensions) > 0 { - body, err := json.MarshalIndent(parsed.Definition, "", " ") - if err != nil { - return err + var envelope struct { + Definition json.RawMessage `json:"definition"` + } + if err := json.Unmarshal(result, &envelope); err == nil && len(envelope.Definition) > 0 { + var probe struct { + Dimensions []json.RawMessage `json:"dimensions"` + } + if json.Unmarshal(envelope.Definition, &probe) == nil && len(probe.Dimensions) > 0 { + var pretty bytes.Buffer + if err := json.Indent(&pretty, envelope.Definition, "", " "); err != nil { + return messages.Serializing(path, err) + } + return os.WriteFile(path, pretty.Bytes(), 0o600) } - return os.WriteFile(path, body, 0o600) } // Fall back to the raw payload rather than losing the result. diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_rubric_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_rubric_test.go new file mode 100644 index 00000000000..c38c1d08f21 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_rubric_test.go @@ -0,0 +1,57 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "encoding/json" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// The generated rubric is the file the next deploy compares against, so a field +// dropped on the way to disk republishes the evaluator without it. pass_threshold +// is what decides pass or fail, so losing it changes grading silently. +func TestWriteRubricKeepsTheWholeDefinition(t *testing.T) { + path := filepath.Join(t.TempDir(), "rubric.json") + + result := json.RawMessage(`{ + "name": "support-agent-quality", + "version": "1", + "definition": { + "type": "rubric", + "pass_threshold": 0.5, + "dimensions": [{"id": "accuracy", "description": "Correct.", "weight": 9}], + "something_the_service_added_later": true + } + }`) + + require.NoError(t, writeRubric(path, result)) + + var got map[string]any + body, err := os.ReadFile(path) + require.NoError(t, err) + require.NoError(t, json.Unmarshal(body, &got)) + + assert.Equal(t, 0.5, got["pass_threshold"], + "the threshold decides pass or fail, so losing it changes grading") + assert.Equal(t, true, got["something_the_service_added_later"], + "the definition is written through, so a new field is not lost either") + assert.Equal(t, "rubric", got["type"]) + assert.Len(t, got["dimensions"], 1) + assert.NotContains(t, got, "name", "only the definition is written, not the envelope") +} + +// A payload that is not a rubric is kept verbatim rather than discarded. +func TestWriteRubricFallsBackToTheRawPayload(t *testing.T) { + path := filepath.Join(t.TempDir(), "rubric.json") + require.NoError(t, writeRubric(path, json.RawMessage(`{"unexpected":"shape"}`))) + + body, err := os.ReadFile(path) + require.NoError(t, err) + assert.JSONEq(t, `{"unexpected":"shape"}`, string(body)) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go b/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go index 60e3454fe5b..948766605ef 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go @@ -1268,6 +1268,16 @@ func EvalCreated(eval, id string) string { return fmt.Sprintf("%s Created eval: %s (%s)\n", DoneMark, eval, id) } +// EvalUnchanged reports an eval a create found already in place. +// +// An eval is immutable, so re-running create against an unedited declaration +// creates nothing. Saying "Created" there claims work that did not happen, and +// hides the one thing worth checking: that the id, and so the run history +// hanging off it, survived. +func EvalUnchanged(eval, id string) string { + return fmt.Sprintf("%s Eval %s is unchanged (%s)\n", SkippedMark, eval, id) +} + // ListingEvals reports a failure to list the project's evals. func ListingEvals(err error) error { return fmt.Errorf("listing evals: %w", err) From fb43305e567e39e4784eeaa2859dec69b7ca875f Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 11 Aug 2026 07:11:11 +0300 Subject: [PATCH 183/320] Name the missing configuration, and stop a cancel race failing the suite Running `azd ai eval create` before `init` reported a Windows syscall phrase about a path and named neither command: reading eval config "evals/azure.eval.yaml": open evals\azure.eval.yaml: The system cannot find the path specified. It now says there is no configuration and names `azd ai eval init`. The error still unwraps to fs.ErrNotExist, because the callers that tolerate an absent configuration decide that by asking; a nicer sentence that stopped answering turned every one of those into a failure, which is how the tests caught it. TestLiveRunCancel failed intermittently. The service refuses to cancel a run that has already left the cancellable window, and whether a run is still in it a moment after starting is a race the test begins but does not control. A refused cancel now skips with the run's status, which is what the assertion was really asking about anyway, and the refusal itself is pinned by a separate case. --- .../internal/messages/messages.go | 26 +++++++++++++++++++ .../internal/messages/paths_test.go | 24 ++++++++++++++++- .../tests/live/run_cancel_test.go | 10 ++++++- 3 files changed, 58 insertions(+), 2 deletions(-) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go b/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go index 948766605ef..e648e88ad73 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go @@ -1699,9 +1699,35 @@ func AmbiguousEvalConfig(current, legacy string) error { // ReadingEvalConfig reports a configuration file that would not read. func ReadingEvalConfig(path string, err error) error { + if errors.Is(err, fs.ErrNotExist) { + return NoEvalConfig(path) + } return fmt.Errorf("reading eval config %q: %w", filepath.ToSlash(path), err) } +// NoEvalConfig reports a command run before anything scaffolded a config. +// +// The bare read failure underneath is a Windows syscall phrase about a path, +// which describes the symptom of running `create` before `init` without naming +// either command. +// +// Still unwraps to fs.ErrNotExist, because callers that tolerate an absent +// configuration — OpenEvalConfig, and the reference resolution above it — decide +// that by asking, and a nicer sentence that stopped answering would turn every +// one of those into a failure. +func NoEvalConfig(path string) error { + return &missingFileError{ + msg: fmt.Sprintf( + "no eval configuration at %s; run `azd ai eval init` to scaffold one", + filepath.ToSlash(path)), + } +} + +type missingFileError struct{ msg string } + +func (e *missingFileError) Error() string { return e.msg } +func (e *missingFileError) Unwrap() error { return fs.ErrNotExist } + // ParsingEvalConfig reports a configuration file that would not parse. func ParsingEvalConfig(path string, err error) error { return fmt.Errorf("parsing eval config %q: %w", filepath.ToSlash(path), err) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/messages/paths_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/messages/paths_test.go index 1cc641126a7..8cd54f240eb 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/messages/paths_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/messages/paths_test.go @@ -5,13 +5,35 @@ package messages import ( "errors" + "fmt" + "io/fs" "strings" "testing" "github.com/stretchr/testify/assert" ) -// %q escapes a Windows separator, so a path printed straight through comes back +// Running `create` before `init` is the first thing anyone does wrong, and the +// bare read failure underneath is a Windows syscall phrase naming neither +// command. +// +// It must still unwrap to fs.ErrNotExist: the callers that tolerate an absent +// configuration decide that by asking, so a nicer sentence that stopped +// answering would turn every one of those into a failure. +func TestNoEvalConfigStaysDetectable(t *testing.T) { + err := ReadingEvalConfig(`evals\azure.eval.yaml`, fmt.Errorf("open x: %w", fs.ErrNotExist)) + + assert.Contains(t, err.Error(), "no eval configuration at evals/azure.eval.yaml") + assert.Contains(t, err.Error(), "azd ai eval init") + assert.NotContains(t, err.Error(), "The system cannot find") + assert.True(t, errors.Is(err, fs.ErrNotExist), + "callers tolerate an absent config by asking, so it has to keep answering") + + other := ReadingEvalConfig(`evals\azure.eval.yaml`, errors.New("permission denied")) + assert.Contains(t, other.Error(), "reading eval config") + assert.False(t, errors.Is(other, fs.ErrNotExist)) +} + // as "evals\\azure.eval.yaml". A reader who copies that into a shell gets a path // that does not exist, which is the opposite of what naming the file is for. func TestPathsInMessagesStayCopyable(t *testing.T) { diff --git a/cli/azd/extensions/azure.ai.evaluations/tests/live/run_cancel_test.go b/cli/azd/extensions/azure.ai.evaluations/tests/live/run_cancel_test.go index a6b01729a5e..170f33f673f 100644 --- a/cli/azd/extensions/azure.ai.evaluations/tests/live/run_cancel_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/tests/live/run_cancel_test.go @@ -78,7 +78,15 @@ func TestLiveRunCancel(t *testing.T) { t.Logf("started run %s (status %s)", run.ID, run.Status) canceled, err := env.evalClient.CancelOpenAIEvalRun(ctx, group.ID, run.ID) - require.NoError(t, err, "cancelling the run") + if err != nil { + // The service refuses to cancel a run that already left the cancellable + // window. That is a race this test starts but does not control, and it + // is the behaviour a separate case already pins, so there is nothing + // left here to observe. + current, getErr := env.evalClient.GetOpenAIEvalRun(ctx, group.ID, run.ID) + require.NoError(t, getErr, "reading the run whose cancel was refused") + t.Skipf("cancel was refused with the run already at %q: %v", current.Status, err) + } require.NotNil(t, canceled) t.Logf("cancel returned status %s", canceled.Status) From 2ddaaf26b091144bfd88ad1801c2308d5442cb6a Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 11 Aug 2026 07:11:13 +0300 Subject: [PATCH 184/320] Release 1.0.0-beta.7 --- cli/azd/extensions/azure.ai.evaluations/extension.yaml | 2 +- cli/azd/extensions/azure.ai.evaluations/version.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cli/azd/extensions/azure.ai.evaluations/extension.yaml b/cli/azd/extensions/azure.ai.evaluations/extension.yaml index 5ca89bf95a1..0b16bebc211 100644 --- a/cli/azd/extensions/azure.ai.evaluations/extension.yaml +++ b/cli/azd/extensions/azure.ai.evaluations/extension.yaml @@ -5,7 +5,7 @@ displayName: Foundry evaluations (Beta) description: Define and run Foundry evaluations from your terminal. (Beta) usage: azd ai eval [options] # NOTE: Make sure version.txt is in sync with this version. -version: 1.0.0-beta.6 +version: 1.0.0-beta.7 requiredAzdVersion: ">=1.27.1" language: go capabilities: diff --git a/cli/azd/extensions/azure.ai.evaluations/version.txt b/cli/azd/extensions/azure.ai.evaluations/version.txt index 6c3924b6842..284fe0bbb10 100644 --- a/cli/azd/extensions/azure.ai.evaluations/version.txt +++ b/cli/azd/extensions/azure.ai.evaluations/version.txt @@ -1 +1 @@ -1.0.0-beta.6 +1.0.0-beta.7 From f52ee943b5206bd0232f691ef7693cbb99889389 Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 11 Aug 2026 08:13:23 +0300 Subject: [PATCH 185/320] Answer a 404 in one line, and name the dataset that was never generated Two more messages found by running the documented corner cases in fresh folders. A misspelled `--target` warned with the whole 404 body: the request URL, an eighty-character rule, the status line, an error code and nested JSON, for a fact that fits on one line. Both agent warnings now say the agent is not in the project and keep the detail only for failures nobody has diagnosed yet. A declared dataset whose rows were never generated failed with `GetFileAttributesEx evals\datasets\x.jsonl: The system cannot find the file specified.` This is the same ordering mistake the evaluator path already explains, and it now gets the same treatment: what is missing, and the command that writes it. Deliberately not changed: an unreadable agent still warns rather than stopping. Making it fatal reads better for a typo, but the only way to reach "nothing supplied a generation model" is a target that resolves to no deployment, so stopping there removes the case the flag validation exists for. Two CLI tests name that contract and were right to fail when I tried it. --- .../internal/cmd/generate.go | 7 ++- .../internal/cmd/reconciler.go | 3 + .../internal/messages/agent_warnings_test.go | 61 +++++++++++++++++++ .../internal/messages/messages.go | 39 ++++++++++++ .../tests/cli/generate_test.go | 3 + 5 files changed, 111 insertions(+), 2 deletions(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/messages/agent_warnings_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go index 6c7aa3c31ce..71d218f3820 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go @@ -161,8 +161,11 @@ func (ec *evalContext) resolveGenerationInstruction( agent, err := ec.evalClient.GetAgent(ctx, agentName, ProjectEndpointAPIVersion) if err != nil { - // Generation can still proceed from the agent source alone, so a - // failure to read the agent is reported without stopping. + // Reported without stopping, because the model can still be supplied by + // --generation-model and the caller has its own checks for what is left + // missing. Making an absent agent fatal here reads well for a typo but + // takes away the only path to "nothing supplied a model", which is the + // case the flag validation exists for. if !quiet { fmt.Fprint(out, messages.WarningAgentUnreadable(agentName, err)) } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go index c4877c617ed..d7ce2d4edbe 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go @@ -76,6 +76,9 @@ func (r *evalReconciler) EnsureDataset( } if _, err := os.Stat(localPath); err != nil { + if errors.Is(err, fs.ErrNotExist) { + return "", false, messages.DatasetNotGeneratedYet(decl.Name, localPath) + } return "", false, messages.DatasetSource(localPath, err) } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/messages/agent_warnings_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/messages/agent_warnings_test.go new file mode 100644 index 00000000000..144d8d7a25b --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/messages/agent_warnings_test.go @@ -0,0 +1,61 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package messages + +import ( + "errors" + "net/http" + "testing" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore" + "github.com/stretchr/testify/assert" +) + +func notFoundErr() error { + return &azcore.ResponseError{StatusCode: http.StatusNotFound, ErrorCode: "not_found"} +} + +// A misspelled --target is the common cause, and 404 answers it with ten lines +// of URL, status rule and nested JSON for a fact that fits on one. +func TestAgentWarningsAnswerA404InOneLine(t *testing.T) { + for _, line := range []string{ + WarningAgentUnreadable("suport-agent", notFoundErr()), + CouldNotReadAgentForModel("suport-agent", notFoundErr()), + } { + assert.Contains(t, line, `no agent "suport-agent" in this project`) + assert.NotContains(t, line, "RESPONSE 404") + assert.NotContains(t, line, "https://") + assert.Equal(t, 1, countLines(line), "a 404 is one fact, so it gets one line: %s", line) + } + + // Anything else keeps the detail, because it is not a fact anyone knows yet. + other := WarningAgentUnreadable("support-agent", errors.New("connection reset")) + assert.Contains(t, other, "connection reset") +} + +// The declared dataset has not been generated yet, which is an ordering mistake +// and not a broken configuration. The bare stat failure underneath is a Windows +// syscall name and says nothing about what to run. +func TestDatasetNotGeneratedYet(t *testing.T) { + err := DatasetProblem("support-agent-eval", + DatasetNotGeneratedYet("support-agent-eval", `evals\datasets\support-agent-eval.jsonl`)) + + got := err.Error() + assert.Contains(t, got, "evals/datasets/support-agent-eval.jsonl") + assert.NotContains(t, got, `\\`) + assert.NotContains(t, got, "GetFileAttributesEx") + assert.Contains(t, got, "azd ai eval generate --dataset --dataset-name support-agent-eval") + assert.Equal(t, 1, countOccurrences(got, `"support-agent-eval"`), + "the wrapper names the dataset, so the message must not name it again") +} + +func countLines(s string) int { + n := 0 + for _, r := range s { + if r == '\n' { + n++ + } + } + return n +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go b/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go index e648e88ad73..f4cb863fb77 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go @@ -31,6 +31,7 @@ import ( "azureaieval/internal/exterrors" + "github.com/Azure/azure-sdk-for-go/sdk/azcore" "github.com/azure/azure-dev/cli/azd/pkg/azdext" ) @@ -560,11 +561,27 @@ func SeedingFromAgent(agent string) string { } // WarningAgentUnreadable reports an agent that could not supply context. +// +// A misspelled --target is the common cause and answers 404, whose body is ten +// lines of URL, status rule and nested JSON for a fact that fits on one. func WarningAgentUnreadable(agent string, err error) string { + if notFound(err) { + return fmt.Sprintf(" warning: no agent %q in this project, so generation "+ + "has no agent context to work from\n", agent) + } return fmt.Sprintf(" warning: could not read agent %q for generation context: %v\n", agent, err) } +// notFound reports a service answer of 404. +// +// Written here rather than imported from eval_api, because that package imports +// this one for its own wording and the dependency only goes one way. +func notFound(err error) bool { + var respErr *azcore.ResponseError + return errors.As(err, &respErr) && respErr.StatusCode == http.StatusNotFound +} + // WarningAgentSeedFailedRetrying reports the retry that drops the agent source. func WarningAgentSeedFailedRetrying(agent string) string { return fmt.Sprintf( @@ -894,6 +911,24 @@ func DatasetSource(path string, err error) error { return fmt.Errorf("dataset source %q: %w", filepath.ToSlash(path), err) } +// DatasetNotGeneratedYet reports a declared dataset whose rows are not written +// yet. +// +// `init` declares the dataset it plans and names the command that produces it, +// so reaching a deploy without one is an ordering mistake rather than a broken +// configuration. Said plainly, because the bare stat failure underneath is a +// Windows syscall name and a path with doubled separators. +// +// Both callers wrap this with DatasetProblem, which names the dataset, so this +// does not name it again. +func DatasetNotGeneratedYet(dataset, path string) error { + return fmt.Errorf( + "its rows %s have not been generated yet. "+ + "Run `azd ai eval generate --dataset --dataset-name %s` to write them, "+ + "or point the declaration at a .jsonl you already have", + filepath.ToSlash(path), dataset) +} + // DatasetNotLocalNorFound reports a source-less dataset the project rejected. func DatasetNotLocalNorFound(dataset string, err error) error { return fmt.Errorf( @@ -2119,5 +2154,9 @@ func InvalidSubscriptionID(err error) error { // CouldNotReadAgentForModel reports a target agent whose deployment could not // be read, leaving generation without a default model. func CouldNotReadAgentForModel(agent string, err error) string { + if notFound(err) { + return fmt.Sprintf(" warning: no agent %q in this project, so there is no "+ + "deployment to default to; pass --generation-model\n", agent) + } return fmt.Sprintf(" warning: could not read agent %q for its deployment: %v\n", agent, err) } diff --git a/cli/azd/extensions/azure.ai.evaluations/tests/cli/generate_test.go b/cli/azd/extensions/azure.ai.evaluations/tests/cli/generate_test.go index 299881a8f18..804041bde9a 100644 --- a/cli/azd/extensions/azure.ai.evaluations/tests/cli/generate_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/tests/cli/generate_test.go @@ -87,6 +87,9 @@ func TestCLIGenerateNamesTheArtifact(t *testing.T) { // The target is no longer among them — it is read from the eval's declaration — // but the generation model has no other source, so it is the one input a bare // directory cannot supply. +// +// The target names an agent that does not exist, and that is load-bearing: a +// real one supplies a deployment, and then nothing is missing to report. func TestCLIGenerateNoPromptNamesWhatIsMissing(t *testing.T) { r := requireFailure(t, runIn(t, t.TempDir(), "generate", "--dataset", "--dataset-name", "d", "--target", "a", "--no-prompt")) From bbda6d6768826a650aebd3fec1f39c205c30aea3 Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 11 Aug 2026 08:13:23 +0300 Subject: [PATCH 186/320] Release 1.0.0-beta.8 --- cli/azd/extensions/azure.ai.evaluations/extension.yaml | 2 +- cli/azd/extensions/azure.ai.evaluations/version.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cli/azd/extensions/azure.ai.evaluations/extension.yaml b/cli/azd/extensions/azure.ai.evaluations/extension.yaml index 0b16bebc211..bf3322d0c14 100644 --- a/cli/azd/extensions/azure.ai.evaluations/extension.yaml +++ b/cli/azd/extensions/azure.ai.evaluations/extension.yaml @@ -5,7 +5,7 @@ displayName: Foundry evaluations (Beta) description: Define and run Foundry evaluations from your terminal. (Beta) usage: azd ai eval [options] # NOTE: Make sure version.txt is in sync with this version. -version: 1.0.0-beta.7 +version: 1.0.0-beta.8 requiredAzdVersion: ">=1.27.1" language: go capabilities: diff --git a/cli/azd/extensions/azure.ai.evaluations/version.txt b/cli/azd/extensions/azure.ai.evaluations/version.txt index 284fe0bbb10..4a5b46bbe88 100644 --- a/cli/azd/extensions/azure.ai.evaluations/version.txt +++ b/cli/azd/extensions/azure.ai.evaluations/version.txt @@ -1 +1 @@ -1.0.0-beta.7 +1.0.0-beta.8 From 686d15b4ea7948c3f03247fda54516b99130fc9f Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 11 Aug 2026 09:04:22 +0300 Subject: [PATCH 187/320] Accept an eval's name where its id is accepted, unless the name is ambiguous `eval list` reports names, so a name is what a reader has to hand, but `eval delete` only accepted an id. `eval show` already fell back to resolving a name; delete now does the same. Deleting is not the same as showing, though. An eval is immutable, so editing a declaration creates another one under the same name and leaves the previous one holding its run history, and deleting takes those runs with it. Where showing can pick the newest match and still be useful, deleting would be discarding something the reader did not name. So resolution splits: evalIDsNamed returns every match, evalIDNamed keeps the newest-wins behaviour for readers, and delete refuses a name carried by more than one, listing the ids so the choice can be stated. Verified live: a unique name deletes and reports the id it resolved to; a name shared by two evals exits 1, names both ids, and leaves both. --- .../azure.ai.evaluations/extension.yaml | 2 +- .../internal/cmd/eval_group.go | 17 +++++++++++++- .../internal/cmd/evalref.go | 21 ++++++++++++++--- .../internal/messages/eval_name_test.go | 23 +++++++++++++++++++ .../internal/messages/messages.go | 11 +++++++++ 5 files changed, 69 insertions(+), 5 deletions(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/messages/eval_name_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/extension.yaml b/cli/azd/extensions/azure.ai.evaluations/extension.yaml index bf3322d0c14..07595319dea 100644 --- a/cli/azd/extensions/azure.ai.evaluations/extension.yaml +++ b/cli/azd/extensions/azure.ai.evaluations/extension.yaml @@ -5,7 +5,7 @@ displayName: Foundry evaluations (Beta) description: Define and run Foundry evaluations from your terminal. (Beta) usage: azd ai eval [options] # NOTE: Make sure version.txt is in sync with this version. -version: 1.0.0-beta.8 +version: 1.0.0-beta.9 requiredAzdVersion: ">=1.27.1" language: go capabilities: diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/eval_group.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/eval_group.go index e4421ef1f5e..6459d9209d2 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/eval_group.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/eval_group.go @@ -255,7 +255,22 @@ func newEvalDeleteCommand() *cobra.Command { } defer ec.Close() - if err := ec.evalClient.DeleteOpenAIEval(ctx, evalID); err != nil { + err = ec.evalClient.DeleteOpenAIEval(ctx, evalID) + if err != nil && eval_api.IsNotFound(err) { + // `list` reports names, so a name is what a reader has to hand. + // An eval is immutable, though, so editing a declaration leaves + // another under the same name, and this deletes the runs under + // whichever it picks: with more than one it asks rather than guesses. + switch ids := ec.evalIDsNamed(ctx, evalID); len(ids) { + case 0: + case 1: + evalID = ids[0] + err = ec.evalClient.DeleteOpenAIEval(ctx, evalID) + default: + return messages.AmbiguousEvalName(evalID, ids) + } + } + if err != nil { if eval_api.IsNotFound(err) { return messages.EvalGone(evalID) } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evalref.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evalref.go index 98d04c16870..268a4f9a97f 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evalref.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evalref.go @@ -82,15 +82,30 @@ func (ec *evalContext) recordedEvalID(ctx context.Context, evalName string) stri // Evals are addressed by id, but every listing reports a name, so a name is // what a reader has to hand. Returns empty when nothing matches, leaving the // caller's not-found reporting alone. +// +// The newest match wins when a name is carried by several. An eval is +// immutable, so editing a declaration creates another one under the same name, +// and the newest is the one the configuration currently describes. func (ec *evalContext) evalIDNamed(ctx context.Context, name string) string { + ids := ec.evalIDsNamed(ctx, name) + if len(ids) == 0 { + return "" + } + return ids[0] +} + +// evalIDsNamed finds every eval the service lists under this name, newest +// first, so a caller that must not guess can see the ambiguity. +func (ec *evalContext) evalIDsNamed(ctx context.Context, name string) []string { list, err := ec.evalClient.ListOpenAIEvals(ctx, 0) if err != nil || list == nil { - return "" + return nil } + var ids []string for _, e := range list.Data { if e.Name == name { - return e.ID + ids = append(ids, e.ID) } } - return "" + return ids } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/messages/eval_name_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/messages/eval_name_test.go new file mode 100644 index 00000000000..2a0da8beedc --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/messages/eval_name_test.go @@ -0,0 +1,23 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package messages + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +// An eval is immutable, so editing a declaration creates another under the same +// name and leaves the previous one holding its run history. Deleting takes the +// runs with it, so a name that matches several has to be refused rather than +// resolved to whichever happens to sort first. +func TestAmbiguousEvalName(t *testing.T) { + got := AmbiguousEvalName("support-agent-eval", []string{"eval_aaa", "eval_bbb"}).Error() + + assert.Contains(t, got, `2 evals are named "support-agent-eval"`) + assert.Contains(t, got, "eval_aaa") + assert.Contains(t, got, "eval_bbb") + assert.Contains(t, got, "discards its runs", "the reason it will not guess is the cost of guessing") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go b/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go index f4cb863fb77..b3e6094e538 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go @@ -1330,6 +1330,17 @@ func EvalNotFound(evalID string) error { "`azd ai eval list` shows the ones there are", evalID) } +// AmbiguousEvalName reports a name carried by more than one eval. +// +// An eval is immutable, so editing a declaration creates another under the same +// name and leaves the previous one holding its run history. Deleting takes the +// runs with it, so which one is meant has to be said rather than guessed. +func AmbiguousEvalName(name string, ids []string) error { + return fmt.Errorf( + "%d evals are named %q, and deleting one discards its runs, so name the "+ + "id instead: %s", len(ids), name, strings.Join(ids, ", ")) +} + // EvalGone reports an eval id there is nothing to delete at. func EvalGone(evalID string) error { return fmt.Errorf("no eval %q in this project", evalID) From 4cae58e3547f535305da32184ca24cbb89ddb298 Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 11 Aug 2026 10:09:34 +0300 Subject: [PATCH 188/320] Bump version.txt with extension.yaml The build stamps the binary from version.txt and the registry publishes extension.yaml, so beta.9 shipped a binary still stamped beta.8. TestManifestVersionMatchesVersionFile exists for exactly this and caught it; I bumped one file and not the other. --- cli/azd/extensions/azure.ai.evaluations/version.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/azd/extensions/azure.ai.evaluations/version.txt b/cli/azd/extensions/azure.ai.evaluations/version.txt index 4a5b46bbe88..0f824babd4f 100644 --- a/cli/azd/extensions/azure.ai.evaluations/version.txt +++ b/cli/azd/extensions/azure.ai.evaluations/version.txt @@ -1 +1 @@ -1.0.0-beta.8 +1.0.0-beta.9 From a2ccce87e8f56b37149335b3df32c3e7fd13af22 Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 11 Aug 2026 20:01:59 +0300 Subject: [PATCH 189/320] Carry the dataset fixes into this extension's own copy of those commands This extension serves `azd ai eval dataset ...` from its own copy of the dataset code. Three fixes landed on the sibling azure.ai.dataset and never crossed over, so the same input behaved differently depending on which namespace the user typed. A cross-extension review found all three. The one that mattered: datasetUploadDir collapsed a named .jsonl to its DIRECTORY, and the upload then took whichever file sorted first. Pointing --from-file at one dataset in a folder holding several registered a different one under that name -- and the fingerprint described the file that was named, so the two agreed with each other forever afterwards. The sibling was fixed months ago; this copy was not. Replaced with datasetUploadSource: a named file is itself, a directory resolves to the single .jsonl in it, and several is refused by name. Also ported: - validAssetName, so a mistyped name is refused here with the sentence the sibling already gives rather than a 400 wrapping four levels of nested JSON. - the messages both of those need. Separately, evalIDsNamed claimed "newest first" in its comment while returning whatever order the service listed. `delete` prints those ids when it refuses an ambiguous name, so the order is part of a message someone acts on. It now sorts on created_at via timestampString, which normalises the two shapes the service uses; an eval without a usable timestamp sorts last rather than winning by accident. Every fix has a test, and the upload one was confirmed by restoring the old behaviour and watching it fail. --- .../internal/cmd/dataset.go | 55 +++++++++++-- .../internal/cmd/dataset_source_test.go | 79 +++++++++++++++++++ .../internal/cmd/evalref.go | 28 ++++++- .../internal/cmd/evalref_order_test.go | 51 ++++++++++++ .../azure.ai.evaluations/internal/cmd/init.go | 3 + .../internal/cmd/names.go | 26 ++++++ .../internal/messages/messages.go | 27 +++++++ 7 files changed, 259 insertions(+), 10 deletions(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset_source_test.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/evalref_order_test.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/names.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset.go index 38e63d7ab71..c880ee78895 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset.go @@ -66,11 +66,14 @@ func newDatasetWriteCommand(verb, short string) *cobra.Command { Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { name := args[0] + if !validAssetName(name) { + return messages.InvalidDatasetName(name) + } if fromFile == "" { return requireFlag("from-file") } - localDir, err := datasetUploadDir(fromFile) + localSource, err := datasetUploadSource(fromFile) if err != nil { return err } @@ -102,7 +105,7 @@ func newDatasetWriteCommand(verb, short string) *cobra.Command { } ds, err := ec.datasetClient.UploadNextVersion( - ctx, name, version, localDir, ProjectEndpointAPIVersion, + ctx, name, version, localSource, ProjectEndpointAPIVersion, ) if err != nil { return messages.RegisteringDataset(name, err) @@ -134,19 +137,46 @@ func newDatasetWriteCommand(verb, short string) *cobra.Command { return cmd } -// datasetUploadDir resolves what was named into the directory the upload scans. -func datasetUploadDir(path string) (string, error) { +// datasetUploadSource resolves what was named into the path the upload reads. +// +// A named file is returned as itself. Returning its directory would upload +// whichever .jsonl sorts first, so pointing --from-file at one dataset in a +// folder holding several would register a different one under that name — and +// the fingerprint would describe the file that was named, so the two would +// agree forever afterwards. +// +// A directory is resolved to the single .jsonl inside it, which is what the +// flag offers. Several is not that, and picking one would be a guess. +func datasetUploadSource(path string) (string, error) { info, err := os.Stat(path) if err != nil { return "", messages.ReadingFromFile(path, err) } - if info.IsDir() { + if !info.IsDir() { + if !strings.EqualFold(filepath.Ext(path), ".jsonl") { + return "", messages.FromFileMustBeJSONL(path) + } return path, nil } - if !strings.EqualFold(filepath.Ext(path), ".jsonl") { - return "", messages.FromFileMustBeJSONL(path) + + entries, err := os.ReadDir(path) + if err != nil { + return "", messages.ReadingFromFile(path, err) + } + var found []string + for _, e := range entries { + if !e.IsDir() && strings.EqualFold(filepath.Ext(e.Name()), ".jsonl") { + found = append(found, e.Name()) + } + } + switch len(found) { + case 0: + return "", messages.FromFileDirectoryHasNoJSONL(path) + case 1: + return filepath.Join(path, found[0]), nil + default: + return "", messages.FromFileDirectoryIsAmbiguous(path, found) } - return filepath.Dir(path), nil } func newDatasetListCommand() *cobra.Command { @@ -196,6 +226,9 @@ func newDatasetVersionsListCommand() *cobra.Command { Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { name := args[0] + if !validAssetName(name) { + return messages.InvalidDatasetName(name) + } ctx := cmd.Context() ec, err := newEvalContext(ctx, endpointFlg) @@ -253,6 +286,9 @@ func newDatasetShowCommand() *cobra.Command { Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { name := args[0] + if !validAssetName(name) { + return messages.InvalidDatasetName(name) + } ctx := cmd.Context() ec, err := newEvalContext(ctx, endpointFlg) @@ -318,6 +354,9 @@ func newDatasetDeleteCommand() *cobra.Command { Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { name := args[0] + if !validAssetName(name) { + return messages.InvalidDatasetName(name) + } if version == "" { return requireFlag("version") } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset_source_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset_source_test.go new file mode 100644 index 00000000000..bdf9ec25eec --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset_source_test.go @@ -0,0 +1,79 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// A named file has to be uploaded as itself. +// +// This resolver used to return the file's DIRECTORY, and the upload then took +// whichever .jsonl sorted first. Pointing --from-file at one dataset in a +// folder holding several therefore registered a different one under that name, +// and because the fingerprint described the file that was named, the two agreed +// with each other forever afterwards. The sibling dataset extension was fixed; +// this copy was not, so the same command uploaded different bytes depending on +// which namespace the user typed. +func TestDatasetUploadSourceReadsTheFileThatWasNamed(t *testing.T) { + dir := t.TempDir() + first := filepath.Join(dir, "a-sorts-first.jsonl") + wanted := filepath.Join(dir, "b-is-the-one-named.jsonl") + require.NoError(t, os.WriteFile(first, []byte(`{"query":"a"}`), 0o600)) + require.NoError(t, os.WriteFile(wanted, []byte(`{"query":"b"}`), 0o600)) + + got, err := datasetUploadSource(wanted) + + require.NoError(t, err) + assert.Equal(t, wanted, got, "the file that was named, not the one that sorts first") +} + +// A directory is offered by the flag, so one .jsonl inside it resolves. +func TestDatasetUploadSourceResolvesADirectoryHoldingOne(t *testing.T) { + dir := t.TempDir() + only := filepath.Join(dir, "only.jsonl") + require.NoError(t, os.WriteFile(only, []byte(`{"query":"a"}`), 0o600)) + + got, err := datasetUploadSource(dir) + + require.NoError(t, err) + assert.Equal(t, only, got) +} + +// Several is not "a directory containing one", and picking would be a guess. +func TestDatasetUploadSourceRefusesAnAmbiguousDirectory(t *testing.T) { + dir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(dir, "a.jsonl"), []byte(`{}`), 0o600)) + require.NoError(t, os.WriteFile(filepath.Join(dir, "b.jsonl"), []byte(`{}`), 0o600)) + + _, err := datasetUploadSource(dir) + + require.Error(t, err) + assert.Contains(t, err.Error(), "a.jsonl") + assert.Contains(t, err.Error(), "b.jsonl", "naming them is what makes the refusal actionable") +} + +func TestDatasetUploadSourceRefusesADirectoryWithNoJSONL(t *testing.T) { + _, err := datasetUploadSource(t.TempDir()) + + require.Error(t, err) + assert.Contains(t, err.Error(), "no .jsonl file") +} + +// The service refuses a bad name with a 400 wrapping four levels of JSON, so +// the guard exists to say it plainly. The sibling extension had it; this copy, +// which serves the same commands under `azd ai eval dataset`, did not. +func TestValidAssetNameMatchesTheSibling(t *testing.T) { + for _, ok := range []string{"golden", "a_b-c", "A1"} { + assert.Truef(t, validAssetName(ok), "%q is a name the service accepts", ok) + } + for _, bad := range []string{"", "has space", "slash/name", "dots.here", "uni\u00e9"} { + assert.Falsef(t, validAssetName(bad), "%q must be refused before the round trip", bad) + } +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evalref.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evalref.go index 268a4f9a97f..a53dd878f8c 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evalref.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evalref.go @@ -5,8 +5,10 @@ package cmd import ( "context" + "sort" "azureaieval/internal/messages" + "azureaieval/internal/pkg/eval_api" "azureaieval/internal/project" ) @@ -96,16 +98,38 @@ func (ec *evalContext) evalIDNamed(ctx context.Context, name string) string { // evalIDsNamed finds every eval the service lists under this name, newest // first, so a caller that must not guess can see the ambiguity. +// +// The order is established here rather than taken from the service, which does +// not promise one. timestampString normalises both shapes the service uses for +// created_at to RFC3339 UTC, and those sort chronologically as text. An eval +// whose timestamp is missing or unparseable sorts last rather than winning by +// accident. func (ec *evalContext) evalIDsNamed(ctx context.Context, name string) []string { list, err := ec.evalClient.ListOpenAIEvals(ctx, 0) if err != nil || list == nil { return nil } - var ids []string + return idsNamedIn(list, name) +} + +// idsNamedIn picks the evals carrying this name, newest first. +// +// timestampString normalises both shapes the service uses for created_at to +// RFC3339 UTC, and those sort chronologically as text. An eval whose timestamp +// is missing or unparseable sorts last rather than winning by accident. +func idsNamedIn(list *eval_api.OpenAIEvalList, name string) []string { + var matches []eval_api.OpenAIEval for _, e := range list.Data { if e.Name == name { - ids = append(ids, e.ID) + matches = append(matches, e) } } + sort.SliceStable(matches, func(i, j int) bool { + return timestampString(matches[i].CreatedAt) > timestampString(matches[j].CreatedAt) + }) + ids := make([]string, 0, len(matches)) + for _, m := range matches { + ids = append(ids, m.ID) + } return ids } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evalref_order_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evalref_order_test.go new file mode 100644 index 00000000000..d0d0b4a5853 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evalref_order_test.go @@ -0,0 +1,51 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "testing" + + "azureaieval/internal/pkg/eval_api" + + "github.com/stretchr/testify/assert" +) + +// `delete` refuses a name carried by several evals and lists the ids, so the +// order those ids come back in is part of a message a reader acts on. The +// service does not promise an order, and this used to hand back whatever the +// listing happened to contain while the comment above it claimed newest-first. +func TestEvalIDsNamedSortsNewestFirst(t *testing.T) { + list := &eval_api.OpenAIEvalList{Data: []eval_api.OpenAIEval{ + {ID: "eval_oldest", Name: "shared", CreatedAt: "2026-01-01T00:00:00Z"}, + {ID: "eval_newest", Name: "shared", CreatedAt: "2026-08-01T00:00:00Z"}, + {ID: "eval_middle", Name: "shared", CreatedAt: "2026-04-01T00:00:00Z"}, + {ID: "eval_other", Name: "different", CreatedAt: "2026-09-01T00:00:00Z"}, + }} + + got := idsNamedIn(list, "shared") + + assert.Equal(t, []string{"eval_newest", "eval_middle", "eval_oldest"}, got) +} + +// The service spells created_at as epoch seconds on some routes and RFC3339 on +// others, so both have to order the same way. +func TestEvalIDsNamedSortsAcrossTimestampShapes(t *testing.T) { + list := &eval_api.OpenAIEvalList{Data: []eval_api.OpenAIEval{ + {ID: "eval_old", Name: "shared", CreatedAt: float64(1767225600)}, // 2026-01-01 + {ID: "eval_new", Name: "shared", CreatedAt: "2026-08-01T00:00:00Z"}, + }} + + assert.Equal(t, []string{"eval_new", "eval_old"}, idsNamedIn(list, "shared")) +} + +// An eval the service described without a usable timestamp must not win by +// accident; it sorts last and the ones that can be ordered still are. +func TestEvalIDsNamedPutsUndatedLast(t *testing.T) { + list := &eval_api.OpenAIEvalList{Data: []eval_api.OpenAIEval{ + {ID: "eval_undated", Name: "shared"}, + {ID: "eval_dated", Name: "shared", CreatedAt: "2026-01-01T00:00:00Z"}, + }} + + assert.Equal(t, []string{"eval_dated", "eval_undated"}, idsNamedIn(list, "shared")) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go index 7d80347ac04..354fef0fdaf 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go @@ -171,6 +171,9 @@ func newInitCommand() *cobra.Command { if source == initSourceTraces { fmt.Fprint(out, messages.UsingTraceSource()) } + if names := plan.evaluatorNames(); len(names) > 0 { + fmt.Fprint(out, messages.GradingWith(names)) + } if judgeModel != "" { fmt.Fprint(out, messages.JudgeModelDeployment(judgeModel)) } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/names.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/names.go new file mode 100644 index 00000000000..4ae09cd3446 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/names.go @@ -0,0 +1,26 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import "regexp" + +// assetNamePattern is what the service accepts for a dataset name. Its own +// refusal is a 400 carrying four levels of nested JSON, and the sentence that +// matters is at the bottom of it. +// +// This extension carries its own copy of the dataset commands, so it needs its +// own copy of the guard: without it the same mistyped name is refused clearly +// by `azd ai dataset` and obscurely by `azd ai eval dataset`. +var assetNamePattern = regexp.MustCompile(`^[A-Za-z0-9_-]+$`) + +const assetNameMaxLength = 255 + +// validAssetName reports whether the service will accept this name, so a +// mistyped one is refused before a round trip rather than after. +func validAssetName(name string) bool { + if name == "" || len(name) > assetNameMaxLength { + return false + } + return assetNamePattern.MatchString(name) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go b/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go index b3e6094e538..9305a359b50 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go @@ -838,6 +838,25 @@ func FromFileMustBeJSONL(path string) error { filepath.ToSlash(path)) } +// FromFileDirectoryHasNoJSONL reports a directory with nothing to upload. +func FromFileDirectoryHasNoJSONL(dir string) error { + return fmt.Errorf("no .jsonl file in %q; --from-file needs one to upload", filepath.ToSlash(dir)) +} + +// FromFileDirectoryIsAmbiguous refuses to guess which dataset was meant. +func FromFileDirectoryIsAmbiguous(dir string, names []string) error { + return fmt.Errorf( + "%q holds %d .jsonl files (%s); name the one to upload with --from-file", + filepath.ToSlash(dir), len(names), strings.Join(names, ", ")) +} + +// InvalidDatasetName reports a name the service will not accept. +func InvalidDatasetName(name string) error { + return fmt.Errorf( + "dataset name %q is invalid: use letters, digits, dashes and underscores, "+ + "up to 255 characters", name) +} + // RegisteringDataset reports the service refusing to publish the dataset. func RegisteringDataset(dataset string, err error) error { return fmt.Errorf("registering dataset %q: %w", dataset, err) @@ -1522,6 +1541,14 @@ func JudgeModelDeployment(model string) string { return fmt.Sprintf("%s Judge model deployment: %s\n", DoneMark, model) } +// GradingWith reports the evaluators the scaffold settled on. +// +// Omitting --evaluator picks them, so without this the one thing `init` decided +// on the reader's behalf is the one thing it does not mention. +func GradingWith(evaluators []string) string { + return fmt.Sprintf("%s Grading with: %s\n", DoneMark, strings.Join(evaluators, ", ")) +} + // CreatedHeading opens the list of what a scaffold wrote. func CreatedHeading() string { return "\nCreated\n" From 11b000a0f2a83cbe7eb393a3643d65475e8489f2 Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 11 Aug 2026 21:53:03 +0300 Subject: [PATCH 190/320] Stop a concurrent reader from seeing a config that lost its evals Found by a concurrency review, then reproduced: a reader observed a configuration with every eval missing 26 times in 0.6 seconds. Three things combined. os.WriteFile truncates before it writes, so the file is zero bytes on disk mid-write. ReadFileNoBOM returns no error for zero bytes. And LoadEvalConfig deliberately maps io.EOF to "a configuration with nothing in it" -- which is right for the empty file `generate` writes before it has anything, and catastrophic for a file another process is halfway through replacing. A reader landing in that window parsed an empty config, appended its own entry, and wrote back a file with every eval, dataset and threshold gone, reporting success. SaveEvalConfigTo now renames into place, so a reader sees the whole old file or the whole new one. writeFileAtomic's own comment already described this bug exactly; it was applied everywhere except the file that matters most. Also in this round, from the same review and a bug bash: - `--fail-on` with `--no-wait` was silently dropped. The gate needs a result and --no-wait returns before there is one, so a pipeline written that way believed it was gated and exited 0 however the run turned out. Refused up front now, naming the reattach that does work. - Generation submitted requests with no sources at all. A kind can be selected without being asked for and without anything to build it from, which added to neither the sources nor the unbuildable list, so an empty request went out and came back as a 400 wrapping thirty lines of JSON around one sentence. Refused locally instead. - The dataset existence probe read version "1" while this CLI's first publish is NextVersion(""), which is "1.0". So the probe was inert for the exact case it was added for -- create, then immediately update. Both extensions. - A message told the reader to pass `--eval-id`, removed long ago. TestSuggestedFlagsExist only inspects flags inside a quoted command, so a bare one escaped it; TestBareFlagsInMessagesExist now checks that every flag any message names exists on some command. Every fix has a test, and each was confirmed by reintroducing the bug and watching the test fail. --- .../azure.ai.evaluations/extension.yaml | 2 +- .../internal/cmd/dataset.go | 26 ++++-- .../cmd/dataset_version_probe_test.go | 24 +++++ .../internal/cmd/gating_nowait_test.go | 55 +++++++++++ .../internal/cmd/generate.go | 21 ++++- .../azure.ai.evaluations/internal/cmd/run.go | 7 ++ .../internal/cmd/surface_test.go | 34 +++++++ .../internal/messages/messages.go | 27 +++++- .../project/eval_config_atomic_test.go | 91 +++++++++++++++++++ .../internal/project/eval_config_store.go | 33 ++++++- .../azure.ai.evaluations/version.txt | 2 +- 11 files changed, 307 insertions(+), 15 deletions(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset_version_probe_test.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/gating_nowait_test.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_atomic_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/extension.yaml b/cli/azd/extensions/azure.ai.evaluations/extension.yaml index 07595319dea..4a62ea9b0ea 100644 --- a/cli/azd/extensions/azure.ai.evaluations/extension.yaml +++ b/cli/azd/extensions/azure.ai.evaluations/extension.yaml @@ -5,7 +5,7 @@ displayName: Foundry evaluations (Beta) description: Define and run Foundry evaluations from your terminal. (Beta) usage: azd ai eval [options] # NOTE: Make sure version.txt is in sync with this version. -version: 1.0.0-beta.9 +version: 1.0.0-beta.10 requiredAzdVersion: ">=1.27.1" language: go capabilities: diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset.go index c880ee78895..39544914bcb 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset.go @@ -17,9 +17,14 @@ import ( "github.com/spf13/cobra" ) -// firstDatasetVersion is the version the service assigns to a dataset's first -// publish, and so the one that exists for every dataset that exists at all. -const firstDatasetVersion = "1" +// firstDatasetVersions are the versions a dataset's first publish can carry, +// probed when the version listing has not caught up yet. +// +// The service assigns nothing; the client picks. This CLI's first publish is +// NextVersion(""), so probing a hardcoded "1" never found a dataset this CLI +// had just created -- which is the one case the probe exists for. "1" is still +// probed because a generation job, the SDK or the portal can register one. +var firstDatasetVersions = []string{dataset_api.NextVersion(""), "1"} func newDatasetCommand() *cobra.Command { cmd := &cobra.Command{ @@ -92,12 +97,15 @@ func newDatasetWriteCommand(verb, short string) *cobra.Command { if !exists { // The version listing lags a publish, so a `create` followed by // an `update` was told the dataset it had just made does not - // exist. A direct read of the first version settles it: point - // reads go consistent immediately. - if _, err := ec.datasetClient.GetDataset( - ctx, name, firstDatasetVersion, ProjectEndpointAPIVersion, - ); err == nil { - exists = true + // exist. A direct read settles it: point reads go consistent + // immediately. + for _, v := range firstDatasetVersions { + if _, err := ec.datasetClient.GetDataset( + ctx, name, v, ProjectEndpointAPIVersion, + ); err == nil { + exists = true + break + } } } if err := checkAssetExistence(verb, "dataset", name, exists); err != nil { diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset_version_probe_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset_version_probe_test.go new file mode 100644 index 00000000000..79908be2f60 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset_version_probe_test.go @@ -0,0 +1,24 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "testing" + + "azureaieval/internal/pkg/dataset_api" + + "github.com/stretchr/testify/assert" +) + +// The existence probe exists for one case: `create` then immediately `update`, +// where the version listing has not caught up. It probed a hardcoded "1" while +// this CLI's first publish is NextVersion(""), which is "1.0" -- so for the +// case it was written for it read a version that never existed and the fallback +// was inert. Deriving it keeps the two in step if the base ever moves. +func TestFirstDatasetVersionsCoverWhatThisCLIPublishes(t *testing.T) { + assert.Contains(t, firstDatasetVersions, dataset_api.NextVersion(""), + "the probe has to look for the version a create actually writes") + assert.Contains(t, firstDatasetVersions, "1", + "a generation job, the SDK or the portal can register a plain 1") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/gating_nowait_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/gating_nowait_test.go new file mode 100644 index 00000000000..6a0d5f208f3 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/gating_nowait_test.go @@ -0,0 +1,55 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "bytes" + "context" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// `--no-wait --fail-on ...` reads as "start it and tell me if it regressed". +// It cannot be: --no-wait returns before there is a result, so the gate was +// dropped and the command exited 0 however the run turned out. A pipeline +// written that way believes it is gated and is not, which is worse than not +// gating at all. +// +// Refused up front, before any network work. +func TestFailOnWithNoWaitIsRefused(t *testing.T) { + for _, gate := range []string{"any-failure", "pass-rate=0.8"} { + root := NewRootCommand() + var out bytes.Buffer + root.SetOut(&out) + root.SetErr(&out) + root.SetArgs([]string{"run", "start", "--no-wait", "--fail-on", gate}) + + err := root.ExecuteContext(context.Background()) + + require.Errorf(t, err, "--no-wait with --fail-on %s must not be accepted", gate) + assert.Contains(t, err.Error(), "--fail-on") + assert.Contains(t, err.Error(), "--no-wait") + assert.Containsf(t, err.Error(), "run show", + "the refusal has to name the way to gate a run started with --no-wait") + } +} + +// The gate on its own still parses and still reaches the run, so the refusal +// above is about the combination and not about --fail-on. +func TestFailOnAloneIsStillAccepted(t *testing.T) { + root := NewRootCommand() + var out bytes.Buffer + root.SetOut(&out) + root.SetErr(&out) + root.SetArgs([]string{"run", "start", "--fail-on", "pass-rate=0.8"}) + + err := root.ExecuteContext(context.Background()) + + if err != nil { + assert.NotContains(t, err.Error(), "--no-wait", + "a gate without --no-wait must not be refused for needing the wait") + } +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go index 71d218f3820..ec25868683c 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go @@ -212,7 +212,7 @@ func (ec *evalContext) generateRubric( sources, unbuildable := eval_api.BuildGenerationSources( plan.From, plan.Agent, "", plan.Instruction, plan.traceOptions(), ) - if err := refuseUnbuildableSources(unbuildable); err != nil { + if err := refuseUnusableSources(sources, unbuildable); err != nil { return nil, err } req := eval_api.NewEvaluatorGenerationJobRequest(plan.Name, plan.Model, sources) @@ -254,6 +254,23 @@ func (ec *evalContext) generateRubric( // Submitting anyway would run a billed job seeded from less than was asked for // and return a plausible-looking artifact, which is the worst outcome: the // caller has no way to tell it apart from one built the way they intended. +// refuseUnusableSources rejects a generation the service could only refuse. +// +// Unbuildable kinds each get their own reason. Beyond those, a request with no +// sources at all is refused here rather than sent: a kind can be selected +// without being asked for and without anything to build it from, which added to +// neither list, so an empty request went out and came back as a 400 wrapping +// thirty lines of JSON around one sentence. +func refuseUnusableSources(sources []eval_api.GenerationSource, kinds []string) error { + if err := refuseUnbuildableSources(kinds); err != nil { + return err + } + if len(sources) == 0 { + return messages.NothingToGenerateFrom() + } + return nil +} + func refuseUnbuildableSources(kinds []string) error { if len(kinds) == 0 { return nil @@ -301,7 +318,7 @@ func (ec *evalContext) generateDataset( sources, unbuildable := eval_api.BuildGenerationSources( plan.From, plan.Agent, "", plan.Instruction, plan.traceOptions(), ) - if err := refuseUnbuildableSources(unbuildable); err != nil { + if err := refuseUnusableSources(sources, unbuildable); err != nil { return nil, err } req := eval_api.NewDataGenerationJobRequest(plan.Name, plan.Model, plan.SampleSize, sources) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go index 683a4226a77..25598df7972 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go @@ -95,6 +95,13 @@ func buildRunCommand(use, short string) *cobra.Command { if err != nil { return err } + // A gate is a verdict on a result. Returning before there is one + // used to drop the gate silently, so `--no-wait --fail-on ...` + // exited 0 however the run turned out -- a pipeline that believes + // it is gated and is not. + if !wait && threshold.set { + return messages.GateNeedsTheWait() + } ec, err := newEvalContext(ctx, endpointFlg) if err != nil { diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/surface_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/surface_test.go index d2c20f10d1a..7e733390b59 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/surface_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/surface_test.go @@ -522,6 +522,40 @@ func TestSuggestedCommandsExist(t *testing.T) { require.NoError(t, err) } +// Every flag a message names has to exist on some command. +// +// TestSuggestedFlagsExist only looks inside a quoted `azd ai eval ...` command, +// so a message that names a flag on its own escapes it. One did: +// DatasetHasUnregisteredEdits told the reader to pass `--eval-id `, a flag +// removed long before, and the check above saw no command to attach it to. +func TestBareFlagsInMessagesExist(t *testing.T) { + flagRef := regexp.MustCompile(`--([a-z][a-z0-9-]{1,})`) + + known := map[string]bool{} + root := NewRootCommand() + root.PersistentFlags().VisitAll(func(f *pflag.Flag) { known[f.Name] = true }) + walk(t, root, nil, func(_ string, c *cobra.Command) { + c.Flags().VisitAll(func(f *pflag.Flag) { known[f.Name] = true }) + }) + // azd's own, which messages legitimately name. + for _, global := range []string{"cwd", "debug", "environment", "no-prompt", "output", "help"} { + known[global] = true + } + + body, err := os.ReadFile(filepath.Join("..", "messages", "messages.go")) + require.NoError(t, err) + + for i, line := range strings.Split(string(body), "\n") { + if strings.HasPrefix(strings.TrimSpace(line), "//") { + continue + } + for _, m := range flagRef.FindAllStringSubmatch(line, -1) { + assert.Truef(t, known[m[1]], + "messages.go:%d names --%s, which no command accepts", i+1, m[1]) + } + } +} + // A command suggested with an argument has to be suggested with the argument // filled in. // diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go b/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go index 9305a359b50..effed28503d 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go @@ -88,11 +88,36 @@ func DatasetNotInCatalog(dataset, configPath string) error { return fmt.Errorf("dataset %q is not in the catalog in %s", dataset, filepath.ToSlash(configPath)) } +// NothingToGenerateFrom refuses a generation request carrying no sources. +// +// The service answers one with "At least one source is required", wrapped in a +// 400 and thirty lines of JSON. Nothing about that names the two things a +// reader can actually supply. +func NothingToGenerateFrom() error { + return errors.New( + "nothing to generate from: pass --target to seed from an " + + "agent's instructions, or declare one under target: in the eval " + + "configuration. A trace-backed eval names its agent under source:, " + + "which selects traces to read and does not seed generation") +} + +// GateNeedsTheWait refuses a gate on a run the command will not wait for. +// +// The two flags together read as "start it and tell me if it regressed", but +// the verdict does not exist yet when --no-wait returns, so the gate was +// silently dropped and the command exited 0 however the run turned out. +func GateNeedsTheWait() error { + return errors.New( + "--fail-on needs a result to judge, and --no-wait returns before there " + + "is one. Drop --no-wait, or reattach with `azd ai eval run show " + + " --wait --fail-on `") +} + // DatasetHasUnregisteredEdits reports local rows no deployed version holds. func DatasetHasUnregisteredEdits(dataset string) error { return fmt.Errorf( "dataset %q has local edits that are not registered.\n"+ - " Run `azd up` to register them, or `--eval-id ` to run against "+ + " Run `azd up` to register them, or `--eval ` to run against "+ "an existing eval", dataset) } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_atomic_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_atomic_test.go new file mode 100644 index 00000000000..71af32fa100 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_atomic_test.go @@ -0,0 +1,91 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package project + +import ( + "os" + "path/filepath" + "sync" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// The config is read by other processes while this one writes it, and +// os.WriteFile truncates before it writes. A reader landing in that window got +// zero bytes -- which parses as a VALID EMPTY CONFIG, not an error -- and would +// then write back a file with every eval missing, reporting success. +// +// Replacing by rename means a reader sees the whole old file or the whole new +// one. This drives a writer and a reader concurrently and asserts the reader +// never observes a config that lost its evals. +func TestSaveEvalConfigNeverExposesAHalfWrittenFile(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "azure.eval.yaml") + + full := &EvalConfig{ + Datasets: []DatasetDecl{{Name: "golden", Source: "./datasets/golden.jsonl"}}, + Evals: []Eval{ + {Name: "first", EvaluationLevel: "turn"}, + {Name: "second", EvaluationLevel: "turn"}, + }, + } + require.NoError(t, SaveEvalConfigTo(path, full)) + + var wg sync.WaitGroup + stop := make(chan struct{}) + var truncated int + + wg.Add(1) + go func() { + defer wg.Done() + for i := 0; i < 300; i++ { + cfg, err := LoadEvalConfig(path) + if err != nil { + continue // a read that fails is honest; a silent empty one is not + } + if len(cfg.Evals) != 2 { + truncated++ + } + } + close(stop) + }() + + wg.Add(1) + go func() { + defer wg.Done() + for { + select { + case <-stop: + return + default: + _ = SaveEvalConfigTo(path, full) + } + } + }() + wg.Wait() + + assert.Zerof(t, truncated, + "a concurrent reader saw a config with its evals missing %d times", truncated) +} + +// The replacement must leave the file complete and parseable. +func TestSaveEvalConfigRoundTripsThroughTheRename(t *testing.T) { + path := filepath.Join(t.TempDir(), "azure.eval.yaml") + want := &EvalConfig{Evals: []Eval{{Name: "only", EvaluationLevel: "turn"}}} + + require.NoError(t, SaveEvalConfigTo(path, want)) + require.NoError(t, SaveEvalConfigTo(path, want)) // over an existing file + + got, err := LoadEvalConfig(path) + require.NoError(t, err) + require.Len(t, got.Evals, 1) + assert.Equal(t, "only", got.Evals[0].Name) + + // The temporary file is this function's business and must not be left over. + entries, err := os.ReadDir(filepath.Dir(path)) + require.NoError(t, err) + assert.Len(t, entries, 1, "the rename must not leave a temporary file behind") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_store.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_store.go index 0a48c00d978..2992e922977 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_store.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_store.go @@ -135,12 +135,43 @@ func SaveEvalConfig(evalDir string, cfg *EvalConfig) error { // SaveEvalConfigTo writes cfg over an explicit path, for callers that already // resolved one. +// +// The replacement is atomic because os.WriteFile truncates first, and this file +// is read by other processes. A reader landing inside that window sees zero +// bytes, and a zero-byte config parses as a valid empty one rather than as an +// error, so it would go on to write back a configuration with every eval +// missing. Renaming into place means a reader sees either the whole old file or +// the whole new one. func SaveEvalConfigTo(path string, cfg *EvalConfig) error { body, err := yaml.Marshal(cfg) if err != nil { return messages.SerializingEvalConfig(err) } - if err := os.WriteFile(path, body, 0o600); err != nil { + + dir := filepath.Dir(path) + tmp, err := os.CreateTemp(dir, ".azd-eval-config-*") + if err != nil { + return messages.WritingEvalConfig(path, err) + } + tmpName := tmp.Name() + defer os.Remove(tmpName) + + if _, err := tmp.Write(body); err != nil { + tmp.Close() + return messages.WritingEvalConfig(path, err) + } + if err := tmp.Chmod(0o600); err != nil { + tmp.Close() + return messages.WritingEvalConfig(path, err) + } + if err := tmp.Close(); err != nil { + return messages.WritingEvalConfig(path, err) + } + // Windows will not rename onto an existing file. + if err := os.Remove(path); err != nil && !errors.Is(err, os.ErrNotExist) { + return messages.WritingEvalConfig(path, err) + } + if err := os.Rename(tmpName, path); err != nil { return messages.WritingEvalConfig(path, err) } return nil diff --git a/cli/azd/extensions/azure.ai.evaluations/version.txt b/cli/azd/extensions/azure.ai.evaluations/version.txt index 0f824babd4f..15f5cb52e6a 100644 --- a/cli/azd/extensions/azure.ai.evaluations/version.txt +++ b/cli/azd/extensions/azure.ai.evaluations/version.txt @@ -1 +1 @@ -1.0.0-beta.9 +1.0.0-beta.10 From 74ddca5c4c23ffbb496064237487130d83da8838 Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 11 Aug 2026 23:30:36 +0300 Subject: [PATCH 191/320] Ask which evaluators to grade with An eval grades on a SET, so unlike the target and the judge model there is no "the only one" to detect. What it defaults to was therefore a default rather than a detection, and which criteria define quality for an agent is the substantive decision in the configuration -- so init now asks, through MultiSelect with the defaults ticked. The choices are what is knowable without a service call, since init makes none: the pair init proposes, plus whatever the catalog already declares. Enumerating the service's built-ins would need a call, and a hardcoded copy would drift. Anything else is reachable with --evaluator. --no-prompt takes the preselection. Erroring there would break every CI path and the init -> generate flow, and the preselection is defensible precisely because it is what the prompt proposes. Two things this had to keep working: - A rubric that is CHOSEN rather than defaulted still has to be generated. It arrives through the explicit branch now, which did not set generateRubric, so the config would have declared a file nothing produces and `create` would fail looking for it -- finding 0b, reintroduced by the prompt. Pinned by a test that fails without it. - The "Grading with:" line prints only when nothing was asked. Reporting a set decided FOR the reader is useful; echoing one they just picked is noise. Raised in review of the spec PR: I had defended the default as "detect when unambiguous", which conflated it with --judge-model. The spec transcript already showed a reader selecting two evaluators. --- .../azure.ai.evaluations/internal/cmd/init.go | 25 +++- .../internal/cmd/init_evaluators.go | 126 ++++++++++++++++++ .../internal/cmd/init_evaluators_test.go | 49 +++++++ .../internal/cmd/init_rubric_choice_test.go | 66 +++++++++ .../internal/messages/messages.go | 17 +++ .../internal/project/eval_config_store.go | 12 +- .../project/service_config_strict_test.go | 82 ++++++++++++ .../internal/project/service_target_eval.go | 20 ++- 8 files changed, 389 insertions(+), 8 deletions(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_evaluators.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_evaluators_test.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_rubric_choice_test.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/project/service_config_strict_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go index 354fef0fdaf..d79237cf47b 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go @@ -117,6 +117,20 @@ func newInitCommand() *cobra.Command { cfg.RemoveEval(evalName) } + // Asked, not detected: an eval grades on a set, so there is no + // "the only one" to settle on, and which criteria define quality + // is the substantive decision in the configuration. + evaluatorsWereChosen := len(evaluators) > 0 + if len(evaluators) == 0 { + var asked bool + evaluators, asked, err = resolveEvaluators( + cmd, cfg, target+"-quality", source == initSourceTraces) + if err != nil { + return err + } + evaluatorsWereChosen = asked + } + if err := os.MkdirAll(filepath.Join(path, project.DefaultDatasetsDir), 0o750); err != nil { return messages.CreatingDatasetsDir(err) } @@ -171,7 +185,9 @@ func newInitCommand() *cobra.Command { if source == initSourceTraces { fmt.Fprint(out, messages.UsingTraceSource()) } - if names := plan.evaluatorNames(); len(names) > 0 { + // Only what was settled without asking: a reader who just picked + // from a list does not need it read back to them. + if names := plan.evaluatorNames(); len(names) > 0 && !evaluatorsWereChosen { fmt.Fprint(out, messages.GradingWith(names)) } if judgeModel != "" { @@ -352,6 +368,13 @@ func planScaffold(in scaffoldInput) scaffold { Name: e, Source: fmt.Sprintf("./%s/%s.json", project.DefaultEvaluatorsDir, e), }) + // Chosen, not defaulted, but it is still the rubric init offers to + // write, so it still has to be generated. Without this the config + // declares a file that nothing produces and `create` fails looking + // for it. + if e == in.rubricName { + out.generateRubric = true + } } } eval.Evaluators = refs diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_evaluators.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_evaluators.go new file mode 100644 index 00000000000..1aee19273a2 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_evaluators.go @@ -0,0 +1,126 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "github.com/spf13/cobra" + + "azureaieval/internal/messages" + "azureaieval/internal/pkg/evalcore" + "azureaieval/internal/project" + + "github.com/azure/azure-dev/cli/azd/pkg/azdext" +) + +// defaultEvaluators is what `init` proposes: one built-in that judges whether +// the agent did what was asked, plus a rubric generated from the agent's own +// instructions, which is what makes the criteria specific to this agent. +// +// The rubric is not offered for a trace-backed eval, which has no target to +// read instructions from. +func defaultEvaluators(rubricName string, traceBacked bool) []string { + refs := []string{evalcore.BuiltinPrefix + "task_adherence"} + if !traceBacked { + refs = append(refs, rubricName) + } + return refs +} + +// evaluatorChoices are the references `init` can offer. +// +// `init` makes no service calls, so the service's full built-in catalogue is +// not knowable here; offering a hardcoded copy of it would drift. What is +// knowable is the pair init proposes and whatever this configuration already +// declares. Anything else is reachable with --evaluator. +func evaluatorChoices(cfg *project.EvalConfig, rubricName string, traceBacked bool) []string { + seen := map[string]bool{} + var out []string + add := func(ref string) { + if ref == "" || seen[ref] { + return + } + seen[ref] = true + out = append(out, ref) + } + + for _, ref := range defaultEvaluators(rubricName, traceBacked) { + add(ref) + } + if cfg != nil { + for _, decl := range cfg.Evaluators { + add(decl.Name) + } + } + return out +} + +// resolveEvaluators settles what the eval grades on. +// +// Unlike the target and the judge model, there is no "the only one" here: an +// eval grades on a SET, and which criteria define quality for this agent is the +// substantive decision in the whole configuration. So this asks rather than +// detects, with the defaults preselected. Under --no-prompt the preselection +// stands, which is what keeps CI and the init -> generate flow working. +// +// The second return says whether the reader chose. Only a set decided FOR them +// is worth reporting back; echoing a selection they just made is noise. +func resolveEvaluators( + cmd *cobra.Command, + cfg *project.EvalConfig, + rubricName string, + traceBacked bool, +) ([]string, bool, error) { + defaults := defaultEvaluators(rubricName, traceBacked) + if noPrompt(cmd) { + return defaults, false, nil + } + chosen, err := promptEvaluators(cmd, evaluatorChoices(cfg, rubricName, traceBacked), defaults) + if err != nil { + return nil, false, err + } + return chosen, true, nil +} + +// promptEvaluators asks which references to grade with, defaults ticked. +func promptEvaluators(cmd *cobra.Command, choices, preselected []string) ([]string, error) { + azdClient, err := azdext.NewAzdClient() + if err != nil { + return nil, messages.ConnectingToAzd(err) + } + defer azdClient.Close() + + ticked := map[string]bool{} + for _, p := range preselected { + ticked[p] = true + } + opts := make([]*azdext.MultiSelectChoice, 0, len(choices)) + for i := range choices { + opts = append(opts, &azdext.MultiSelectChoice{ + Label: choices[i], + Value: choices[i], + Selected: ticked[choices[i]], + }) + } + + resp, err := azdClient.Prompt().MultiSelect(cmd.Context(), &azdext.MultiSelectRequest{ + Options: &azdext.MultiSelectOptions{ + Message: messages.SelectEvaluatorsPrompt(), + Choices: opts, + }, + }) + if err != nil { + return nil, messages.SelectingEvaluators(err) + } + + chosen := make([]string, 0, len(resp.GetValues())) + for _, v := range resp.GetValues() { + chosen = append(chosen, v.GetValue()) + } + if len(chosen) == 0 { + // An eval that grades on nothing is rejected by the service, and the + // refusal names none of this. + return nil, messages.NoEvaluatorsChosen() + } + return chosen, nil +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_evaluators_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_evaluators_test.go new file mode 100644 index 00000000000..e0e5fb929f1 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_evaluators_test.go @@ -0,0 +1,49 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "testing" + + "azureaieval/internal/pkg/evalcore" + "azureaieval/internal/project" + + "github.com/stretchr/testify/assert" +) + +// What an eval grades on is a SET, so there is no "the only one" to detect the +// way there is for the target and the judge model. Which criteria define +// quality is the substantive decision in the configuration, so init asks. +// +// The defaults are what it proposes, and they are what --no-prompt takes. +func TestDefaultEvaluatorsProposeABuiltinAndTheRubric(t *testing.T) { + assert.Equal(t, + []string{evalcore.BuiltinPrefix + "task_adherence", "support-agent-quality"}, + defaultEvaluators("support-agent-quality", false)) +} + +// A trace-backed eval has no target whose instructions a rubric is written +// from, so proposing one would plan a file nothing can generate. +func TestDefaultEvaluatorsSkipTheRubricForTraces(t *testing.T) { + assert.Equal(t, + []string{evalcore.BuiltinPrefix + "task_adherence"}, + defaultEvaluators("support-agent-quality", true)) +} + +// The prompt offers what is knowable without a service call -- init makes none +// -- which is the pair it proposes plus whatever the catalog already declares. +func TestEvaluatorChoicesOfferTheCatalogToo(t *testing.T) { + cfg := &project.EvalConfig{Evaluators: []project.EvaluatorDecl{ + {Name: "support-agent-quality"}, // already proposed, must not double up + {Name: "tone-check"}, + }} + + got := evaluatorChoices(cfg, "support-agent-quality", false) + + assert.Equal(t, []string{ + evalcore.BuiltinPrefix + "task_adherence", + "support-agent-quality", + "tone-check", + }, got) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_rubric_choice_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_rubric_choice_test.go new file mode 100644 index 00000000000..0c9ae913601 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_rubric_choice_test.go @@ -0,0 +1,66 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "testing" + + "azureaieval/internal/pkg/evalcore" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// Now that init asks rather than defaults, the rubric arrives as an explicit +// choice instead of through the empty-evaluators branch. It still has to be +// generated: the configuration declares a file, and if nothing produces it, +// `create` fails looking for it -- which is finding 0b, reintroduced by the +// prompt if the chosen path did not set this. +func TestScaffoldGeneratesARubricThatWasChosenRatherThanDefaulted(t *testing.T) { + plan, cfg := scaffoldFor(t, scaffoldInput{ + evalName: "support-agent-eval", + target: "support-agent", + dataset: "golden", + evaluators: []string{ + evalcore.BuiltinPrefix + "task_adherence", + "support-agent-quality", + }, + }) + + assert.True(t, plan.generateRubric, + "a chosen rubric still has to be generated, or its file never exists") + + require.Len(t, cfg.Evaluators, 1, "only the rubric is a catalog entry; the builtin is not") + assert.Equal(t, "support-agent-quality", cfg.Evaluators[0].Name) +} + +// A built-in is resolved by the service and has no local file, so choosing only +// built-ins must not plan a generation. +func TestScaffoldGeneratesNoRubricForBuiltinsAlone(t *testing.T) { + plan, cfg := scaffoldFor(t, scaffoldInput{ + evalName: "support-agent-eval", + target: "support-agent", + dataset: "golden", + evaluators: []string{evalcore.BuiltinPrefix + "task_adherence"}, + }) + + assert.False(t, plan.generateRubric) + assert.Empty(t, cfg.Evaluators) +} + +// An evaluator the author already has on disk is not the rubric init offers, so +// it is declared without planning a generation over it. +func TestScaffoldDoesNotGenerateAnUnrelatedEvaluator(t *testing.T) { + plan, cfg := scaffoldFor(t, scaffoldInput{ + evalName: "support-agent-eval", + target: "support-agent", + dataset: "golden", + evaluators: []string{"tone-check"}, + }) + + assert.False(t, plan.generateRubric, + "only the rubric init offers to write is one it knows how to generate") + require.Len(t, cfg.Evaluators, 1) + assert.Equal(t, "tone-check", cfg.Evaluators[0].Name) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go b/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go index effed28503d..e83022e3078 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go @@ -101,6 +101,23 @@ func NothingToGenerateFrom() error { "which selects traces to read and does not seed generation") } +// SelectEvaluatorsPrompt asks which references the eval grades on. +func SelectEvaluatorsPrompt() string { + return "Select evaluators to grade with:" +} + +// SelectingEvaluators reports a failed evaluator prompt. +func SelectingEvaluators(err error) error { + return fmt.Errorf("selecting evaluators: %w", err) +} + +// NoEvaluatorsChosen reports an eval that would grade on nothing. +func NoEvaluatorsChosen() error { + return errors.New( + "an eval has to grade on at least one evaluator: select one, or pass " + + "--evaluator") +} + // GateNeedsTheWait refuses a gate on a run the command will not wait for. // // The two flags together read as "start it and tell me if it regressed", but diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_store.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_store.go index 2992e922977..06cedfb5276 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_store.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_store.go @@ -101,7 +101,17 @@ func LoadEvalConfig(path string) (*EvalConfig, error) { if err != nil { return nil, messages.ReadingEvalConfig(path, err) } + return DecodeEvalConfig(data, path) +} +// DecodeEvalConfig is the one strict decoder, so every route into a +// configuration reports a mistyped key the same way. +// +// `azd up` reads the configuration through the service entry rather than off +// disk, and that route used json.Unmarshal, which drops unknown keys in +// silence. The same typo was therefore named by `azd ai eval run` and ignored +// by `azd up`. The name is what the diagnostic points at. +func DecodeEvalConfig(data []byte, name string) (*EvalConfig, error) { decoder := yaml.NewDecoder(bytes.NewReader(data)) decoder.KnownFields(true) @@ -112,7 +122,7 @@ func LoadEvalConfig(path string) (*EvalConfig, error) { if errors.Is(err, io.EOF) { return &cfg, nil } - return nil, messages.ParsingEvalConfig(path, explainUnknownKeys(err)) + return nil, messages.ParsingEvalConfig(name, explainUnknownKeys(err)) } return &cfg, nil } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/service_config_strict_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/service_config_strict_test.go new file mode 100644 index 00000000000..73f59c69ff7 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/service_config_strict_test.go @@ -0,0 +1,82 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package project + +import ( + "testing" + + "github.com/azure/azure-dev/cli/azd/pkg/azdext" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/types/known/structpb" +) + +func serviceWith(t *testing.T, props map[string]any) *azdext.ServiceConfig { + t.Helper() + s, err := structpb.NewStruct(props) + require.NoError(t, err) + return &azdext.ServiceConfig{Name: "support-agent-evals", AdditionalProperties: s} +} + +// `azd up` reads the configuration through the service entry, not off disk, and +// that route used json.Unmarshal -- which drops unknown keys silently. So a +// misspelled key was named by `azd ai eval run` and ignored by `azd up`, and +// the setting the author thought they had wrote simply did not exist. +// +// Both routes now go through the same strict decoder. +func TestEvalConfigFromServiceRejectsAMistypedKey(t *testing.T) { + svc := serviceWith(t, map[string]any{ + "evals": []any{map[string]any{ + "name": "support-agent-eval", + "evaulators": []any{}, // the typo `azd ai eval run` already catches + "evaluation_level": "turn", + }}, + }) + + _, err := EvalConfigFromService(svc, "") + + require.Error(t, err, "a key this extension does not know is a typo, on either route") + assert.Contains(t, err.Error(), "evaulators") + assert.Contains(t, err.Error(), "evaluators", "the near miss is what makes it actionable") +} + +// The keys the schema does know still decode, so the strictness did not close +// the door on the authoring style it is meant to serve. +func TestEvalConfigFromServiceAcceptsADeclaredConfig(t *testing.T) { + svc := serviceWith(t, map[string]any{ + "datasets": []any{map[string]any{"name": "golden", "source": "./datasets/golden.jsonl"}}, + "evals": []any{map[string]any{ + "name": "support-agent-eval", + "dataset": "golden", + "evaluation_level": "turn", + }}, + }) + + cfg, err := EvalConfigFromService(svc, "") + + require.NoError(t, err) + require.Len(t, cfg.Evals, 1) + assert.Equal(t, "support-agent-eval", cfg.Evals[0].Name) + require.Len(t, cfg.Datasets, 1) + assert.Equal(t, "golden", cfg.Datasets[0].Name) +} + +// `$ref` is a directive rather than configuration. ResolveFileRefs replaces it +// with the file's content, but it survives when resolution was skipped, and a +// strict decoder would then refuse a config for carrying the very thing that +// pointed at it. +func TestEvalConfigFromServiceIgnoresTheRefDirective(t *testing.T) { + svc := serviceWith(t, map[string]any{ + "$ref": "./evals/azure.eval.yaml", + "evals": []any{map[string]any{ + "name": "support-agent-eval", + "evaluation_level": "turn", + }}, + }) + + cfg, err := EvalConfigFromService(svc, "") + + require.NoError(t, err) + require.Len(t, cfg.Evals, 1) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go index 0552542d880..f502adf6dc3 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go @@ -16,6 +16,7 @@ import ( "github.com/azure/azure-dev/cli/azd/pkg/azdext" "github.com/azure/azure-dev/cli/azd/pkg/foundry" + "go.yaml.in/yaml/v3" "google.golang.org/protobuf/types/known/structpb" ) @@ -247,16 +248,23 @@ func EvalConfigFromService(svc *azdext.ServiceConfig, projectRoot string) (*Eval values = resolved } - raw, err := json.Marshal(values) + // `$ref` is a directive, not configuration: ResolveFileRefs has already + // replaced it with the file's content. It only survives when resolution was + // skipped, and that config cannot deploy anyway. + delete(values, "$ref") + + // Decoded by the same strict reader the on-disk path uses, so `azd up` and + // `azd ai eval run` name a mistyped key identically instead of one + // explaining it and the other ignoring it. + raw, err := yaml.Marshal(values) if err != nil { return nil, messages.ReadingServiceConfig(err) } - - var cfg EvalConfig - if err := json.Unmarshal(raw, &cfg); err != nil { - return nil, messages.ParsingServiceConfig(err) + cfg, err := DecodeEvalConfig(raw, svc.GetName()) + if err != nil { + return nil, err } - return &cfg, nil + return cfg, nil } // serviceProps prefers the inline properties, falling back to the nested From 15a829d6f283abb35220a5b6020814d6ee06049d Mon Sep 17 00:00:00 2001 From: mohessie Date: Wed, 12 Aug 2026 04:22:28 +0300 Subject: [PATCH 192/320] Refuse the flags and values that were accepted and ignored A negative row cap read as "no cap", so `--max-samples -1` and a config carrying `max_samples: -1` both sent every row to a run billed per row -- the opposite of what a cap asks for, and silently. Three more were accepted and dropped: --from and --max-samples are documented "Dataset only" and did nothing under --evaluator; --trace-days -5 was read as 0, producing a rubric with none of the trace seeding asked for; and --output-dir with --no-wait named a directory nothing would ever be written to. Each is refused before any network work, the way --fail-on with --no-wait already was. Resolving a declared source against its base directory had grown three implementations, two of which disagreed: an absolute dataset source came out as evals/C:/data/rows.jsonl under `eval create` while `azd up` handled it, because the resolver was unexported and cmd wrote its own. Exported as project.ResolveSource and the copies deleted. State writes after a successful publish discarded their errors, though the sentinel beside them says only "no azd environment" is ordinary. Those keys are how the next deploy recognises what it already published, so losing one silently means `azd up` creates a second immutable version of something it had already created. They now report anything that is not the ordinary case, and stay quiet for standalone --project-endpoint runs. Also: four doc comments had drifted onto the wrong declaration, one of them describing an identifier that no longer exists, and exterrors carried 37 unreferenced codes naming toolbox and skill resources this extension has no concept of. --- .../azure.ai.evaluations/extension.yaml | 2 +- .../extensions/azure.ai.evaluations/go.mod | 6 +- .../internal/cmd/build.go | 14 +-- .../internal/cmd/catalog.go | 9 ++ .../internal/cmd/context.go | 73 +++++++++++++- .../internal/cmd/eval_group.go | 11 +-- .../internal/cmd/evalref.go | 36 ++++++- .../internal/cmd/evalref_drift_test.go | 75 +++++++++++++- .../internal/cmd/generate_composite.go | 18 ++++ .../internal/cmd/generate_flag_guards_test.go | 96 ++++++++++++++++++ .../azure.ai.evaluations/internal/cmd/init.go | 42 ++++++-- .../internal/cmd/no_environment_test.go | 50 ++++++++++ .../internal/cmd/reconciler.go | 48 ++++----- .../azure.ai.evaluations/internal/cmd/run.go | 9 +- .../internal/cmd/run_output.go | 7 ++ .../internal/exterrors/codes.go | 55 ++--------- .../internal/exterrors/errors.go | 4 +- .../internal/messages/messages.go | 82 +++++++++++++-- .../internal/pkg/dataset_api/operations.go | 7 ++ .../internal/project/eval_config.go | 5 + .../project/eval_config_atomic_test.go | 71 ++++++++++++- .../internal/project/eval_config_lock.go | 89 +++++++++++++++++ .../internal/project/eval_config_lock_test.go | 94 ++++++++++++++++++ .../project/eval_config_readonly_test.go | 49 +++++++++ .../internal/project/eval_config_store.go | 99 +++++++++++++++++-- .../internal/project/max_samples_test.go | 53 ++++++++++ .../internal/project/readfile.go | 3 +- .../internal/project/resolve_source_test.go | 30 ++++++ .../internal/project/service_target_eval.go | 15 ++- .../internal/project/version_spelling_test.go | 74 ++++++++++++++ .../azure.ai.evaluations/version.txt | 2 +- 31 files changed, 1098 insertions(+), 130 deletions(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_flag_guards_test.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/no_environment_test.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_lock.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_lock_test.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_readonly_test.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/project/max_samples_test.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/project/resolve_source_test.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/project/version_spelling_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/extension.yaml b/cli/azd/extensions/azure.ai.evaluations/extension.yaml index 4a62ea9b0ea..0a2338700dc 100644 --- a/cli/azd/extensions/azure.ai.evaluations/extension.yaml +++ b/cli/azd/extensions/azure.ai.evaluations/extension.yaml @@ -5,7 +5,7 @@ displayName: Foundry evaluations (Beta) description: Define and run Foundry evaluations from your terminal. (Beta) usage: azd ai eval [options] # NOTE: Make sure version.txt is in sync with this version. -version: 1.0.0-beta.10 +version: 1.0.0-beta.11 requiredAzdVersion: ">=1.27.1" language: go capabilities: diff --git a/cli/azd/extensions/azure.ai.evaluations/go.mod b/cli/azd/extensions/azure.ai.evaluations/go.mod index ab57a735e7f..ac9e1bfbc3d 100644 --- a/cli/azd/extensions/azure.ai.evaluations/go.mod +++ b/cli/azd/extensions/azure.ai.evaluations/go.mod @@ -7,10 +7,13 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.14.0-beta.3 github.com/azure/azure-dev/cli/azd v1.28.0 github.com/fatih/color v1.18.0 + github.com/gofrs/flock v0.12.1 github.com/google/uuid v1.6.0 github.com/spf13/cobra v1.10.1 + github.com/spf13/pflag v1.0.10 github.com/stretchr/testify v1.11.1 go.yaml.in/yaml/v3 v3.0.4 + google.golang.org/grpc v1.80.0 google.golang.org/protobuf v1.36.11 ) @@ -49,7 +52,6 @@ require ( github.com/drone/envsubst v1.0.3 // indirect github.com/go-logr/logr v1.4.3 // indirect github.com/go-logr/stdr v1.2.2 // indirect - github.com/gofrs/flock v0.12.1 // indirect github.com/golang-jwt/jwt/v5 v5.3.0 // indirect github.com/golobby/container/v3 v3.3.2 // indirect github.com/gorilla/css v1.0.1 // indirect @@ -78,7 +80,6 @@ require ( github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 // indirect github.com/sethvargo/go-retry v0.3.0 // indirect github.com/spf13/cast v1.10.0 // indirect - github.com/spf13/pflag v1.0.10 // indirect github.com/theckman/yacspin v0.13.12 // indirect github.com/wk8/go-ordered-map/v2 v2.1.8 // indirect github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect @@ -101,6 +102,5 @@ require ( golang.org/x/text v0.38.0 // indirect golang.org/x/time v0.9.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20260401024825-9d38bb4040a9 // indirect - google.golang.org/grpc v1.80.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build.go index 0989c4b50f5..1d30335a7b0 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build.go @@ -15,10 +15,6 @@ import ( "azureaieval/internal/project" ) -// evaluatorSchemas indexes the published evaluator contracts by name. -// -// A failure is deliberately not fatal: without schemas the builder falls back -// to the agent-target shape, which is what it always used to send. // evaluatorSchemas indexes the published contract of every evaluator a group // can reference. // @@ -27,6 +23,9 @@ import ( // a schema and falling back to legacyInputs — which happens to match // query/response and so looks right for the common evaluators while quietly // dropping the fields anything else needs. +// +// A failure is deliberately not fatal: without schemas the builder falls back +// to the agent-target shape, which is what it always used to send. func (ec *evalContext) evaluatorSchemas(ctx context.Context) map[string]*eval_api.EvaluatorSummary { if ec.schemas != nil { return ec.schemas @@ -63,11 +62,12 @@ var sampleBindings = map[string]string{ "tool_definitions": "{{sample.tool_definitions}}", } -// modelSampleBindings are what a model target produces. A model answers as -// plain text and calls no tools, so binding an agent's richer output would -// leave the evaluator waiting on fields the run never produces. // sampleBindingsFor returns the run-time bindings a target of this kind can // satisfy. An empty target kind means nothing is invoked, so nothing is bound. +// +// A model target gets none: a model answers as plain text and calls no tools, +// so binding an agent's richer output would leave the evaluator waiting on +// fields the run never produces. func sampleBindingsFor(targetType string) map[string]string { if targetType == project.TargetTypeAgent { return sampleBindings diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/catalog.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/catalog.go index 224dc3d114a..d9898ffedad 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/catalog.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/catalog.go @@ -79,6 +79,15 @@ func updateCatalog( ref *project.ArtifactRef, apply func(*project.EvalConfig) bool, ) error { + // Held across the read and the write: two generates adding different + // entries would otherwise both read the same state, and the second write + // would drop the first one's entry while reporting success. + unlock, _, err := project.LockEvalConfig(cmd.Context(), evalDir) + if err != nil { + return err + } + defer unlock() + cfg, err := project.OpenEvalConfig(evalDir) if err != nil { return err diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/context.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/context.go index 5dbb7b59c28..d5198f9e9f1 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/context.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/context.go @@ -5,6 +5,7 @@ package cmd import ( "context" + "errors" "log" "strings" @@ -110,6 +111,22 @@ func lookupEndpointFromAzd(ctx context.Context, azdClient *azdext.AzdClient) (en // A write that fails for any other reason still is. var errNoAzdEnvironment = messages.ErrNoAzdEnvironment +// remember persists a value that the extension can recover without, so a +// failure to store it must not fail the work that produced it. +// +// Running outside a project is ordinary -- the atomic commands are meant to +// work standalone against the data plane -- so having nowhere to write is not +// worth a word. Anything else is: these keys are how a later deploy recognises +// what it already published, and losing one silently means the next `azd up` +// creates a second immutable version of something it had already created. +func (ec *evalContext) remember(ctx context.Context, key, value string) { + err := ec.setEnvValue(ctx, key, value) + if err == nil || errors.Is(err, errNoAzdEnvironment) { + return + } + log.Printf("[env] could not record %s: %v", key, err) +} + // setEnvValue persists a value into the active azd environment. azd itself // writes none of these keys — the extension owns them. func (ec *evalContext) setEnvValue(ctx context.Context, key, value string) error { @@ -131,10 +148,64 @@ func (ec *evalContext) setEnvValue(ctx context.Context, key, value string) error return nil } +// azdNoDefaultEnvironment is what azd's environment service answers with when +// the project has no environment selected. It arrives over gRPC as a status +// whose message carries the text, so the text is what there is to match on. +// +// azd returns this as an ERROR rather than an empty answer, which is the whole +// difficulty: "there is no environment" and "azd could not be reached" are both +// non-nil errors, and only the first is something to tell the user about. +const azdNoDefaultEnvironment = "default environment not found" + +// confirmedNoAzdEnvironment reports that azd answered, and the answer was that +// there is no current environment. +// +// ec.envName being empty is not that answer. It is left empty by any failure to +// reach azd as well as by there being no environment, so reading it as "there +// is none" turns a transient gRPC hiccup into advice to create an environment +// the user already has. +// +// The environment name is recovered here when there turns out to be one, so a +// caller whose earlier lookup came up empty because of a hiccup can retry it. +func (ec *evalContext) confirmedNoAzdEnvironment(ctx context.Context) bool { + if ec.envName != "" { + return false + } + if ec.azdClient == nil { + // No azd to ask: running standalone against the data plane, where + // there is genuinely nowhere to have recorded an id. + return true + } + envResp, err := ec.azdClient.Environment().GetCurrent(ctx, &azdext.EmptyRequest{}) + if err != nil { + return isNoDefaultEnvironmentError(err) + } + if envResp == nil || envResp.Environment == nil || envResp.Environment.Name == "" { + return true + } + ec.envName = envResp.Environment.Name + return false +} + +// isNoDefaultEnvironmentError picks azd's "there is no environment" out of +// every other reason the call could have failed. +// +// The distinction is the whole point: a transport failure must not be reported +// as a missing environment, or a gRPC hiccup tells the user to create one they +// already have. Matching on text because that is what survives the trip -- the +// sentinel is wrapped in a gRPC status on the way out of azd, so errors.Is has +// nothing to compare against on this side. +func isNoDefaultEnvironmentError(err error) bool { + if err == nil { + return false + } + return strings.Contains(strings.ToLower(err.Error()), azdNoDefaultEnvironment) +} + // getEnvValue reads a value from the active azd environment, returning empty // when it is unset. func (ec *evalContext) getEnvValue(ctx context.Context, key string) string { - if ec.envName == "" { + if ec.envName == "" || ec.azdClient == nil { return "" } val, err := ec.azdClient.Environment().GetValue(ctx, &azdext.GetEnvRequest{ diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/eval_group.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/eval_group.go index 6459d9209d2..49fb8acd64d 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/eval_group.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/eval_group.go @@ -64,9 +64,10 @@ func newEvalCreateCommand() *cobra.Command { // Local sources resolve against the file, not the working directory, // so the columns are read from where the declaration points. + baseDir := filepath.Dir(path) datasetPath := "" - if decl, ok := cfg.DatasetDeclaration(eval.Dataset); ok && decl.Source != "" { - datasetPath = filepath.Join(filepath.Dir(path), decl.Source) + if decl, ok := cfg.DatasetDeclaration(eval.Dataset); ok { + datasetPath = project.ResolveSource(baseDir, decl.Source) } reconciler := &evalReconciler{ec: ec} @@ -90,7 +91,6 @@ func newEvalCreateCommand() *cobra.Command { // first. `azd up` reconciles the whole file; this reconciles only what // this eval refers to, which is also what makes a rubric edit reach // the service without a full deploy. - baseDir := filepath.Dir(path) if decl, ok := cfg.DatasetDeclaration(eval.Dataset); ok { version, changed, err := reconciler.EnsureDataset(ctx, *decl, datasetPath) if err != nil { @@ -104,10 +104,7 @@ func newEvalCreateCommand() *cobra.Command { if !ok || decl.Source == "" { continue } - local := decl.Source - if !filepath.IsAbs(local) { - local = filepath.Join(baseDir, local) - } + local := project.ResolveSource(baseDir, decl.Source) version, changed, err := reconciler.EnsureEvaluator(ctx, *decl, local) if err != nil { return messages.EvaluatorProblem(decl.Name, err) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evalref.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evalref.go index a53dd878f8c..e1b6120bc72 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evalref.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evalref.go @@ -57,7 +57,41 @@ func (ec *evalContext) resolveEvalRef( case err == nil: id := ec.recordedEvalID(ctx, eval.Name) if id == "" { - return evalRef{}, messages.EvalNotDeployedYet(eval.Name) + // Nothing recorded is not the same as nothing published. The + // id is kept in the azd environment, so a run against + // --project-endpoint with no project, or a `create` that ran + // before the environment existed, leaves a published eval + // with no note of it. The service lists evals by name, so + // ask it before deciding this was never deployed. + // + // Refused rather than guessed when a name carries several, as + // `eval delete` already does. Newest-wins is fine for showing + // something, but this id also reaches `run start`, and grading + // against the wrong definition produces results that look + // right and answer a different question. + ids := ec.evalIDsNamed(ctx, eval.Name) + if len(ids) > 1 { + return evalRef{}, messages.AmbiguousEvalName(eval.Name, ids) + } + if len(ids) == 1 { + id = ids[0] + } + } + if id == "" { + // With no environment there was nowhere an id could have been + // recorded, so telling the reader to deploy again would not + // help. Only said when azd confirmed there is none. + if ec.confirmedNoAzdEnvironment(ctx) { + return evalRef{}, messages.NoEnvironmentToRememberEval(eval.Name) + } + // That call recovers the environment name when the first + // lookup missed it -- a transient failure in newEvalContext + // leaves it empty, and recordedEvalID answers "" without + // asking when it is. Now that there is a name, ask properly + // before reporting a deployed eval as missing. + if id = ec.recordedEvalID(ctx, eval.Name); id == "" { + return evalRef{}, messages.EvalNotDeployedYet(eval.Name) + } } return evalRef{ID: id, Eval: eval, Config: cfg, ConfigPath: configPath}, nil case nameOrID == "": diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evalref_drift_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evalref_drift_test.go index f52e5b1654d..ed1e4c68223 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evalref_drift_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evalref_drift_test.go @@ -13,6 +13,7 @@ import ( "testing" "azureaieval/internal/pkg/dataset_api" + "azureaieval/internal/pkg/eval_api" "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" "github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime" @@ -108,6 +109,25 @@ func writeEvalYAML(t *testing.T, body string) string { return evals } +// evalContextListingEvals builds a context whose service lists exactly these +// evals. resolveEvalRef asks the service by name when no id was recorded, so a +// context without a client cannot exercise it. +func evalContextListingEvals(t *testing.T, envName, body string) *evalContext { + t.Helper() + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(body)) + })) + t.Cleanup(srv.Close) + + pipeline := runtime.NewPipeline("test", "v1", runtime.PipelineOptions{}, + &policy.ClientOptions{Retry: policy.RetryOptions{MaxRetries: -1}}) + return &evalContext{ + envName: envName, + evalClient: eval_api.NewEvalClientFromPipeline(srv.URL, pipeline), + } +} + // A declared eval that was never deployed has no id to address, and the // service would answer 404 for a name it never saw. Naming `azd up` is the // difference between a dead end and a next step. @@ -121,7 +141,11 @@ evals: evaluators: - evaluator: builtin.relevance `) - ec := &evalContext{} + // An environment exists; the id was simply never recorded in it. Without + // this the case under test is "nowhere to record", which is a different + // answer. The service lists nothing, which is what never deployed looks + // like from the outside. + ec := evalContextListingEvals(t, "dev", `{"data":[]}`) _, err := ec.resolveEvalRef(context.Background(), dir, "support-quality") @@ -130,6 +154,55 @@ evals: assert.Contains(t, err.Error(), "azd up", "the error has to say what would fix it") } +// The id lives in the azd environment, so `--project-endpoint` against a +// directory that never had one has a published eval and no note of it. Failing +// there would make the declaration unusable outside a project, though the +// service can be asked for the same name. +func TestResolveEvalRefFindsAPublishedEvalByName(t *testing.T) { + dir := writeEvalYAML(t, ` +datasets: + - name: golden +evals: + - name: support-quality + dataset: golden + evaluators: + - evaluator: builtin.relevance +`) + ec := evalContextListingEvals(t, "", + `{"data":[{"id":"eval_published","name":"support-quality"}]}`) + + ref, err := ec.resolveEvalRef(context.Background(), dir, "support-quality") + + require.NoError(t, err) + assert.Equal(t, "eval_published", ref.ID, + "the service knows the id this environment never recorded") + assert.True(t, ref.Declared(), "and it is still the declaration that was matched") +} + +// With no azd environment at all there is nowhere the id could have been +// recorded, so `create` may well have published this eval and had nowhere to +// note it. Sending the reader to `azd up` lands them in the same place. +func TestResolveEvalRefNamesTheMissingEnvironment(t *testing.T) { + dir := writeEvalYAML(t, ` +datasets: + - name: golden +evals: + - name: support-quality + dataset: golden + evaluators: + - evaluator: builtin.relevance +`) + ec := evalContextListingEvals(t, "", `{"data":[]}`) + + _, err := ec.resolveEvalRef(context.Background(), dir, "support-quality") + + require.Error(t, err) + assert.Contains(t, err.Error(), "azd env new", + "the fix is an environment, not another deploy") + assert.NotContains(t, err.Error(), "azd up", + "deploying again would record the id in the same nowhere") +} + // An eval made by `azd ai eval create` has no evals: entry, so anything that // is not a declared name is sent on as an id rather than refused. func TestResolveEvalRefTreatsAnUnknownNameAsAnID(t *testing.T) { diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_composite.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_composite.go index f2d1bf290cc..d2e0cc93cd9 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_composite.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_composite.go @@ -54,6 +54,24 @@ func newGenerateCommand() *cobra.Command { RunE: func(cmd *cobra.Command, args []string) error { dataset, evaluator := selectedArtifacts(wantDataset, wantEvaluator) + // Checked before any network work, so a flag that cannot apply + // costs nothing to find out about. Changed() rather than the value, + // so a zero the caller actually typed is still caught and an + // untouched default is not. + if !dataset { + for _, flag := range []string{"from", "max-samples"} { + if cmd.Flags().Changed(flag) { + return messages.DatasetOnlyFlag(flag) + } + } + } + if traceDays < 0 { + return messages.NegativeTraceDays(traceDays) + } + if flags.noWait && cmd.Flags().Changed("output-dir") { + return messages.OutputDirNeedsTheWait() + } + if dataset { for _, src := range from { if err := project.ValidateGenerateSource(src); err != nil { diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_flag_guards_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_flag_guards_test.go new file mode 100644 index 00000000000..8f7b2d2e328 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_flag_guards_test.go @@ -0,0 +1,96 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "bytes" + "context" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func runGenerate(t *testing.T, args ...string) error { + t.Helper() + root := NewRootCommand() + var out bytes.Buffer + root.SetOut(&out) + root.SetErr(&out) + root.SetArgs(append([]string{"generate"}, args...)) + return root.ExecuteContext(context.Background()) +} + +// --from and --max-samples are documented "Dataset only", and --evaluator +// generates no dataset. They were accepted and ignored, so the run produced a +// rubric and said nothing about the sample count that was asked for. +func TestDatasetOnlyFlagsAreRefusedForEvaluatorOnlyGeneration(t *testing.T) { + for flag, args := range map[string][]string{ + "--from": {"--evaluator", "--evaluator-name", "ev", "--from", "prompt"}, + "--max-samples": {"--evaluator", "--evaluator-name", "ev", "--max-samples", "50"}, + } { + err := runGenerate(t, args...) + + require.Errorf(t, err, "%s cannot apply to an evaluator-only generation", flag) + assert.Contains(t, err.Error(), flag) + assert.Contains(t, err.Error(), "--evaluator", + "the refusal has to name the flag that made it inapplicable") + } +} + +// The same flags with a dataset selected are the ordinary case, so the refusal +// above has to be about the combination rather than about the flags. +func TestDatasetOnlyFlagsAreStillAcceptedForADataset(t *testing.T) { + err := runGenerate(t, + "--dataset", "--dataset-name", "ds", "--from", "prompt", "--max-samples", "50") + + if err != nil { + assert.NotContains(t, err.Error(), "only affects the dataset", + "a dataset generation must not be refused its own flags") + } +} + +// Zero already means "seed the rubric from no traces", so a negative window has +// nothing left to mean. It was accepted and read as zero, quietly producing a +// rubric with none of the trace seeding that was asked for. +func TestNegativeTraceDaysIsRefused(t *testing.T) { + err := runGenerate(t, "--evaluator", "--evaluator-name", "ev", "--trace-days", "-5") + + require.Error(t, err) + assert.Contains(t, err.Error(), "--trace-days") + assert.Contains(t, err.Error(), "-5", "the value that was rejected") +} + +// Zero is a real answer and has to keep working. +func TestZeroTraceDaysIsAccepted(t *testing.T) { + err := runGenerate(t, "--evaluator", "--evaluator-name", "ev", "--trace-days", "0") + + if err != nil { + assert.NotContains(t, err.Error(), "--trace-days", + "0 is how a caller says to read no traces") + } +} + +// --no-wait returns as soon as the job is submitted, so there is no artifact to +// place. Accepting both left the caller waiting for a file never coming. +func TestOutputDirWithNoWaitIsRefused(t *testing.T) { + err := runGenerate(t, + "--dataset", "--dataset-name", "ds", "--no-wait", "--output-dir", t.TempDir()) + + require.Error(t, err) + assert.Contains(t, err.Error(), "--output-dir") + assert.Contains(t, err.Error(), "--no-wait") + assert.Contains(t, err.Error(), "job show", + "the refusal has to name how to collect the artifact later") +} + +// An output directory without --no-wait is what the flag is for. +func TestOutputDirAloneIsStillAccepted(t *testing.T) { + err := runGenerate(t, "--dataset", "--dataset-name", "ds", "--output-dir", t.TempDir()) + + if err != nil { + assert.NotContains(t, err.Error(), "nothing to write to", + "an output directory without --no-wait must not be refused") + } +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go index d79237cf47b..8ccf4d02e7b 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go @@ -109,17 +109,22 @@ func newInitCommand() *cobra.Command { if cfg == nil { cfg = &project.EvalConfig{} } - if cfg.HasEval(evalName) { - if !force { - return messages.EvalAlreadyDeclared( - evalName, filepath.ToSlash(configPath)) - } - cfg.RemoveEval(evalName) + // Checked before the prompt as well as after it, so a name that is + // already taken is reported without asking a question first. + if cfg.HasEval(evalName) && !force { + return messages.EvalAlreadyDeclared( + evalName, filepath.ToSlash(configPath)) } // Asked, not detected: an eval grades on a set, so there is no // "the only one" to settle on, and which criteria define quality // is the substantive decision in the configuration. + // + // Deliberately outside the lock below. This is an unbounded human + // pause, and a lock held across it would either block a concurrent + // `generate` for as long as someone leaves the terminal, or -- once + // that side gave up waiting -- protect nothing at all. The listing + // it offers is only a menu; the authoritative read is taken after. evaluatorsWereChosen := len(evaluators) > 0 if len(evaluators) == 0 { var asked bool @@ -131,6 +136,31 @@ func newInitCommand() *cobra.Command { evaluatorsWereChosen = asked } + // The read-modify-write starts here, and nothing inside it waits on + // a person. The configuration is read again because the copy above + // was taken before the prompt, and a `generate` may well have + // finished writing to it since. + unlockConfig, _, err := project.LockEvalConfig(cmd.Context(), path) + if err != nil { + return err + } + defer unlockConfig() + + cfg, err = project.OpenEvalConfig(path) + if err != nil { + return err + } + if cfg == nil { + cfg = &project.EvalConfig{} + } + if cfg.HasEval(evalName) { + if !force { + return messages.EvalAlreadyDeclared( + evalName, filepath.ToSlash(configPath)) + } + cfg.RemoveEval(evalName) + } + if err := os.MkdirAll(filepath.Join(path, project.DefaultDatasetsDir), 0o750); err != nil { return messages.CreatingDatasetsDir(err) } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/no_environment_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/no_environment_test.go new file mode 100644 index 00000000000..7944b3e3081 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/no_environment_test.go @@ -0,0 +1,50 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "errors" + "fmt" + "testing" + + "github.com/stretchr/testify/assert" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +// azd reports "there is no environment" as an ERROR, not as an empty answer: +// +// if defaultEnvironment == "" { +// return nil, environment.ErrDefaultEnvironmentNotFound +// } +// +// So a check that treats every error as "could not ask" can never conclude +// there is no environment, and the diagnostic that names `azd env new` is +// unreachable -- which is what the first version of this did. Equally, treating +// every error as "no environment" would tell someone whose azd hiccupped to +// create an environment they already have. +// +// The sentinel is wrapped in a gRPC status on the way out of azd, so the text +// is what there is to match on. These are the real shapes. +func TestNoDefaultEnvironmentIsToldApartFromAFailureToAsk(t *testing.T) { + // The text of azd's environment.ErrDefaultEnvironmentNotFound. + const azdText = "default environment not found" + + assert.True(t, isNoDefaultEnvironmentError(errors.New(azdText)), + "the sentinel's own text") + assert.True(t, + isNoDefaultEnvironmentError(status.Error(codes.Unknown, azdText)), + "and the same thing after azd wraps it in a gRPC status") + assert.True(t, + isNoDefaultEnvironmentError(fmt.Errorf("getting environment: %w", errors.New(azdText))), + "and wrapped again by a caller") + + assert.False(t, + isNoDefaultEnvironmentError(status.Error(codes.Unavailable, "connection refused")), + "azd being unreachable is not an answer about environments") + assert.False(t, + isNoDefaultEnvironmentError(status.Error(codes.DeadlineExceeded, "context deadline exceeded")), + "nor is a timeout") + assert.False(t, isNoDefaultEnvironmentError(nil), "nor is success") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go index d7ce2d4edbe..9d4b4aec91b 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go @@ -71,7 +71,7 @@ func (r *evalReconciler) EnsureDataset( // Recorded so a run reads the version reconciliation settled on. Without // this a pin is honoured at deploy and then ignored at run time, which // scores different rows than the ones the author asked for. - _ = r.ec.setEnvValue(ctx, versionKey("dataset", decl.Name), version) + r.ec.remember(ctx, versionKey("dataset", decl.Name), version) return version, false, nil } @@ -129,8 +129,8 @@ func (r *evalReconciler) EnsureDataset( } return "", false, err } - _ = r.ec.setEnvValue(ctx, key, digest) - _ = r.ec.setEnvValue(ctx, versionKey("dataset", decl.Name), ds.Version) + r.ec.remember(ctx, key, digest) + r.ec.remember(ctx, versionKey("dataset", decl.Name), ds.Version) return ds.Version, true, nil } @@ -143,21 +143,13 @@ func (r *evalReconciler) EnsureDataset( return "", false, err } - _ = r.ec.setEnvValue(ctx, key, digest) - _ = r.ec.setEnvValue(ctx, versionKey("dataset", decl.Name), ds.Version) - _ = r.ec.setEnvValue(ctx, envKeyDatasetVersion, ds.Version) + r.ec.remember(ctx, key, digest) + r.ec.remember(ctx, versionKey("dataset", decl.Name), ds.Version) + r.ec.remember(ctx, envKeyDatasetVersion, ds.Version) return ds.Version, true, nil } -// checkDatasetDrift fails when the service holds a newer version than the one -// recorded at the last deploy. -// -// Local content being unchanged is not enough to reuse the recorded version: -// someone may have published a newer one outside the repo, and silently -// pinning the eval to the older version would quietly evaluate against -// stale data. Publishing is not destructive — versions are immutable — so the -// remedy is to sync, not to overwrite. // validateJSONL checks that every row is a JSON object before the file is // published. // @@ -288,9 +280,9 @@ func (r *evalReconciler) EnsureEvaluator( // it is what a later deploy compares against to notice that // someone moved the evaluator on from here. if remote != "" { - _ = r.ec.setEnvValue(ctx, versionKey("evaluator", decl.Name), remote) + r.ec.remember(ctx, versionKey("evaluator", decl.Name), remote) } - _ = r.ec.setEnvValue(ctx, digestKey, digest) + r.ec.remember(ctx, digestKey, digest) return versionFromRaw(existing, decl.Version), false, nil } @@ -317,8 +309,8 @@ func (r *evalReconciler) EnsureEvaluator( return "", false, err } r.awaitEvaluatorReadable(ctx, decl.Name, created.Version) - _ = r.ec.setEnvValue(ctx, versionKey("evaluator", decl.Name), created.Version) - _ = r.ec.setEnvValue(ctx, digestKey, digest) + r.ec.remember(ctx, versionKey("evaluator", decl.Name), created.Version) + r.ec.remember(ctx, digestKey, digest) return created.Version, true, nil } @@ -488,10 +480,10 @@ func (r *evalReconciler) EnsureEval( // Record the digest on reuse as well, otherwise an eval deployed // before fingerprinting existed never establishes a baseline and // later edits go undetected. - _ = r.ec.setEnvValue(ctx, key, digest) - _ = r.ec.setEnvValue(ctx, idKey("eval", group.Name), cached) - _ = r.ec.setEnvValue(ctx, digestIDKey(digest), cached) - _ = r.ec.setEnvValue(ctx, envKeyEvalID, cached) + r.ec.remember(ctx, key, digest) + r.ec.remember(ctx, idKey("eval", group.Name), cached) + r.ec.remember(ctx, digestIDKey(digest), cached) + r.ec.remember(ctx, envKeyEvalID, cached) return cached, nil } } @@ -500,12 +492,12 @@ func (r *evalReconciler) EnsureEval( if err != nil { return "", err } - _ = r.ec.setEnvValue(ctx, key, digest) - _ = r.ec.setEnvValue(ctx, idKey("eval", group.Name), created.ID) - _ = r.ec.setEnvValue(ctx, digestIDKey(digest), created.ID) + r.ec.remember(ctx, key, digest) + r.ec.remember(ctx, idKey("eval", group.Name), created.ID) + r.ec.remember(ctx, digestIDKey(digest), created.ID) // EVAL_ID stays the last-deployed eval, which is what the commands // fall back to when a config names only one. - _ = r.ec.setEnvValue(ctx, envKeyEvalID, created.ID) + r.ec.remember(ctx, envKeyEvalID, created.ID) return created.ID, nil } @@ -661,9 +653,9 @@ func (ec *evalContext) recordDeployedDataset( if err != nil { return } - _ = ec.setEnvValue(ctx, project.FingerprintKey("dataset", name), digest) + ec.remember(ctx, project.FingerprintKey("dataset", name), digest) if version != "" { - _ = ec.setEnvValue(ctx, versionKey("dataset", name), version) + ec.remember(ctx, versionKey("dataset", name), version) } } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go index 25598df7972..aeec6058326 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go @@ -102,6 +102,11 @@ func buildRunCommand(use, short string) *cobra.Command { if !wait && threshold.set { return messages.GateNeedsTheWait() } + // resolveMaxSamples reads anything not above zero as "no cap", so a + // negative one sent the whole dataset to a billed run. + if maxSamples < 0 { + return messages.NegativeMaxSamplesFlag(maxSamples) + } ec, err := newEvalContext(ctx, endpointFlg) if err != nil { @@ -193,7 +198,7 @@ func buildRunCommand(use, short string) *cobra.Command { // Remembered per group as well as globally: a single shared key // belongs to whichever group ran last, so another group asking for // "the last run" would be handed one that is not its own. - _ = ec.setEnvValue(ctx, idKey("evalrun", evalID), run.ID) + ec.remember(ctx, idKey("evalrun", evalID), run.ID) if err := ec.setEnvValue(ctx, envKeyEvalRunID, run.ID); err != nil { // Persisting the run id is a convenience for later commands. // Reported on stdout because azd does not surface an @@ -324,7 +329,7 @@ func (ec *evalContext) resolveEvalIDFromConfig( if err := ec.setEnvValue(ctx, idKey("eval", group.Name), created.ID); err != nil { fmt.Fprint(out, messages.Warning(err)) } - _ = ec.setEnvValue(ctx, envKeyEvalID, created.ID) + ec.remember(ctx, envKeyEvalID, created.ID) return created.ID, nil } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output.go index 7e349295f76..73dca1a8653 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output.go @@ -291,6 +291,13 @@ func addEvalPathFlag(cmd *cobra.Command, target *string) { // explicit says whether the caller named the run rather than leaving it to // default. A remembered run that no longer resolves is worth falling through // on; one that was asked for by name is not. +// +// The remembered id is preferred over the service's listing, and deliberately. +// Listing looks like the fix for two concurrent starts leaving this key holding +// whichever wrote last, but ListOpenAIEvalRuns sends no order parameter, so +// "the first row" is not promised to be the newest; and `run cancel` defaults +// through here, so guessing would cancel a run this environment never started. +// The remembered id is at least scoped to the environment that made it. func (ec *evalContext) latestOrNamedRun( cmd *cobra.Command, evalID, runID string, diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/exterrors/codes.go b/cli/azd/extensions/azure.ai.evaluations/internal/exterrors/codes.go index 58fc300148c..3e2a177d546 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/exterrors/codes.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/exterrors/codes.go @@ -3,6 +3,13 @@ package exterrors +// The codes azd renders alongside an error's category and suggestion. +// +// Only the ones this extension actually raises are listed. This file used to +// carry the toolbox and skill vocabulary it was copied from -- 37 codes for +// resources this extension has no concept of -- which offered anyone looking +// for the right code a menu belonging to a different product. + // Error codes for user cancellation. const ( CodeCancelled = "cancelled" @@ -10,62 +17,16 @@ const ( // Error codes for validation failures (user input, manifests, flags). const ( - CodeInvalidParameter = "invalid_parameter" - CodeInvalidPositionalArg = "invalid_positional_arg" + CodeInvalidParameter = "invalid_parameter" ) // Error codes for dependency failures (missing resources, services, env values). const ( - CodeAzdClientFailed = "azd_client_failed" CodeMissingProjectEndpoint = "missing_project_endpoint" ) // Error codes for auth failures. const ( - CodeNotLoggedIn = "not_logged_in" CodeLoginExpired = "login_expired" CodeAuthFailed = "auth_failed" ) - -// Error codes for toolbox operations. -const ( - CodeToolboxNotFound = "toolbox_not_found" - CodeToolboxVersionNotFound = "toolbox_version_not_found" - CodeInvalidToolboxName = "invalid_toolbox_name" - CodeMissingUpdateField = "missing_update_field" - CodeDefaultVersionDelete = "default_version_delete" - CodeOnlyVersionDelete = "only_version_delete" - CodeMissingForceFlag = "missing_force_flag" - CodeUnsupportedConnectionCategory = "unsupported_connection_category" - CodeMissingIndex = "missing_index" - CodeUnsupportedIndexFlag = "unsupported_index_flag" - CodeMissingInstanceName = "missing_instance_name" - CodeUnsupportedInstanceNameFlag = "unsupported_instance_name_flag" - CodeInvalidSkillName = "invalid_skill_name" - CodeInvalidSkillSpec = "invalid_skill_spec" - CodeDuplicateSkill = "duplicate_skill" - CodeSkillNotInToolbox = "skill_not_in_toolbox" - CodeSkillAlreadyAttached = "skill_already_attached" - CodeDuplicateConnection = "duplicate_connection" - CodeDuplicateToolName = "duplicate_tool_name" - CodeMissingToolType = "missing_tool_type" - CodeConnectionNotFound = "connection_not_found" - CodeConnectionNotInToolbox = "connection_not_in_toolbox" - CodeConnectionMissingTarget = "connection_missing_target" - CodeLastToolRemoval = "last_tool_removal" - CodePendingToolboxStoreFailed = "pending_toolbox_store_failed" -) - -// Operation names for [ServiceFromAzure] errors. -// These are prefixed to the Azure error code (e.g., "get_toolbox.NotFound"). -const ( - OpCreateToolboxVersion = "create_toolbox_version" - OpGetToolbox = "get_toolbox" - OpDeleteToolbox = "delete_toolbox" - OpDeleteToolboxVersion = "delete_toolbox_version" - OpSetDefaultVersion = "set_default_version" - OpListToolboxes = "list_toolboxes" - OpGetToolboxVersion = "get_toolbox_version" - OpListToolboxVersions = "list_toolbox_versions" - OpResolveProjectConnection = "resolve_project_connection" -) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/exterrors/errors.go b/cli/azd/extensions/azure.ai.evaluations/internal/exterrors/errors.go index 4e3c02b6731..d6428a6dfa1 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/exterrors/errors.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/exterrors/errors.go @@ -1,8 +1,8 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. -// Package exterrors provides structured error helpers for the azure.ai.toolboxes -// extension. +// Package exterrors provides structured error helpers for the +// azure.ai.evaluations extension. // // Use plain Go errors until the current code can confidently choose a final // category, code, and suggestion. At that point, create a structured error with diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go b/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go index e83022e3078..c3fcc1ea585 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go @@ -246,6 +246,20 @@ func EvalNotDeployed(evalID string) error { "`azd up` creates the ones your config declares", evalID) } +// NoEnvironmentToRememberEval reports an eval whose id had nowhere to be kept. +// +// `create` publishes the eval and records its id in the azd environment. With +// no environment there is nowhere to record it, so create reports success and +// the next command cannot find what it made. Saying "not deployed" there sends +// the reader to deploy it again, which lands in the same place. +func NoEnvironmentToRememberEval(eval string) error { + return fmt.Errorf( + "eval %q may exist in the project, but this directory has no azd "+ + "environment to have recorded its id in. Create one with "+ + "`azd env new ` and run `azd ai eval create` again, or name "+ + "the eval's id with --eval", eval) +} + // EvalNotDeployedYet reports a declared eval that no deploy has created. func EvalNotDeployedYet(eval string) error { return fmt.Errorf( @@ -518,12 +532,6 @@ func GenerationFailed(kind string, err error) error { return fmt.Errorf("generating the %s: %w", kind, err) } -// SomeGenerationsFailed reports a composite generate where at least one job -// did not finish. The others may well have. -// -// Two structured failures of the same category stay structured, so an expired -// login still arrives as an auth error carrying its suggestion rather than as -// a flat string. // multiError presents several failures as one line while keeping every cause // reachable through errors.Is and errors.As. // @@ -537,6 +545,12 @@ type multiError struct { func (m *multiError) Error() string { return m.msg } func (m *multiError) Unwrap() []error { return m.causes } +// SomeGenerationsFailed reports a composite generate where at least one job +// did not finish. The others may well have. +// +// Two structured failures of the same category stay structured, so an expired +// login still arrives as an auth error carrying its suggestion rather than as +// a flat string. func SomeGenerationsFailed(failures []error) error { if len(failures) == 1 { return failures[0] @@ -1884,10 +1898,66 @@ func FromNotASource(from string, sources []string) error { } // SampleSizeOutOfRange reports a row count the generation service would reject. +// SampleSizeOutOfRange reports a generation sample count the service will not take. func SampleSizeOutOfRange(min, max, got int) error { return fmt.Errorf("sample size must be between %d and %d, got %d", min, max, got) } +// NegativeMaxSamples reports a declared row cap below zero. +// +// Anything not above zero reads as "no cap", so this used to send the whole +// dataset to a run that is billed per row -- the opposite of what a cap asks +// for, and silent. +func NegativeMaxSamples(index int, name string, got int) error { + return fmt.Errorf( + "evals[%d] %q: max_samples cannot be negative, got %d. "+ + "Remove it to send every row, or set the number of rows to send", + index, name, got) +} + +// NegativeMaxSamplesFlag reports the same thing given on the command line. +func NegativeMaxSamplesFlag(got int) error { + return fmt.Errorf( + "--max-samples cannot be negative, got %d. "+ + "Omit it to send every row, or give the number of rows to send", got) +} + +// DatasetOnlyFlag reports a dataset flag given to a run that generates no +// dataset. +// +// The flag is documented "Dataset only" and was accepted and ignored, so +// `--evaluator --max-samples 50` produced a rubric and said nothing about the +// number the caller asked for. +func DatasetOnlyFlag(flag string) error { + return fmt.Errorf( + "--%s only affects the dataset, and --evaluator generates no dataset. "+ + "Drop --%s, or drop --evaluator to generate both", flag, flag) +} + +// NegativeTraceDays reports a trace window below zero. +// +// Zero already means "do not read traces", so a negative value has nothing +// left to mean; it used to be accepted and treated as zero, which silently +// produced a rubric with none of the trace seeding that was asked for. +func NegativeTraceDays(got int) error { + return fmt.Errorf( + "--trace-days cannot be negative, got %d. "+ + "Use 0 to seed the rubric from no traces, or the number of days to read", got) +} + +// OutputDirNeedsTheWait reports an output directory that nothing will be +// written to. +// +// --no-wait returns as soon as the job is submitted, so there is no artifact +// to place. Accepting both left the caller waiting for a file that was never +// coming. +func OutputDirNeedsTheWait() error { + return errors.New( + "--output-dir has nothing to write to with --no-wait, which returns " + + "before the artifact exists. Drop --no-wait, or collect the " + + "artifact later with `azd ai eval job show`") +} + // EndpointEmpty reports a project endpoint given as blank. func EndpointEmpty() error { return exterrors.Validation( diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/operations.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/operations.go index ee7654ee445..f87869a499f 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/operations.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/operations.go @@ -216,6 +216,13 @@ func (c *DatasetClient) UploadVersion( } // Step 2: Upload the JSONL file to blob storage. + // One blob per dataset, which is what the container-listing fallback in + // DownloadDatasetContent expects to find. Naming it for the content instead + // would stop two racing publishes of one version overwriting each other -- + // but it leaves several .jsonl beside each other, and that fallback picks + // the first by name, so a download could return rows no version points at. + // The overwrite is the narrower harm and stays until the version can be + // allocated by the service rather than guessed from a lagging listing. blobName := name + ".jsonl" if err := c.UploadBlob(ctx, uploadURI, blobName, []byte(content)); err != nil { return nil, messages.UploadingBlob(err) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go index f3dc8e313c2..e00cf4e12dc 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go @@ -285,6 +285,11 @@ func (c *EvalConfig) validateEval(i int, eval Eval) error { if eval.Dataset != "" && eval.Source != nil { return messages.DatasetAndSourceBothDeclared(i, eval.Name) } + // A negative cap read as "no cap", so the whole dataset went to a billed + // run when the config asked for fewer rows than that. + if eval.MaxSamples < 0 { + return messages.NegativeMaxSamples(i, eval.Name, eval.MaxSamples) + } if eval.Dataset != "" { if _, ok := c.DatasetDeclaration(eval.Dataset); !ok { return messages.DatasetNotInDatasetsCatalog(i, eval.Name, eval.Dataset) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_atomic_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_atomic_test.go index 71af32fa100..0934262b365 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_atomic_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_atomic_test.go @@ -7,7 +7,9 @@ import ( "os" "path/filepath" "sync" + "sync/atomic" "testing" + "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -37,14 +39,21 @@ func TestSaveEvalConfigNeverExposesAHalfWrittenFile(t *testing.T) { var wg sync.WaitGroup stop := make(chan struct{}) var truncated int + var replacements int64 wg.Add(1) go func() { defer wg.Done() - for i := 0; i < 300; i++ { + deadline := time.Now().Add(500 * time.Millisecond) + for time.Now().Before(deadline) { cfg, err := LoadEvalConfig(path) if err != nil { - continue // a read that fails is honest; a silent empty one is not + // NOT skipped: "the file does not exist" is exactly what a + // remove-then-rename exposes, and OpenEvalConfig turns it into + // "there is no configuration yet" -- the same loss this guards + // against, by another route. + truncated++ + continue } if len(cfg.Evals) != 2 { truncated++ @@ -61,14 +70,68 @@ func TestSaveEvalConfigNeverExposesAHalfWrittenFile(t *testing.T) { case <-stop: return default: - _ = SaveEvalConfigTo(path, full) + if SaveEvalConfigTo(path, full) == nil { + atomic.AddInt64(&replacements, 1) + } } } }() wg.Wait() + require.NotZero(t, atomic.LoadInt64(&replacements), + "the writer has to have replaced the file, or nothing was under test") assert.Zerof(t, truncated, - "a concurrent reader saw a config with its evals missing %d times", truncated) + "a concurrent reader failed to see the whole config %d times", truncated) +} + +// OpenEvalConfig maps a missing file to "no configuration yet", which callers +// answer by writing a fresh one. So a replacement that momentarily unlinks the +// destination is as destructive as one that truncates it, and this pins the +// window closed from that side too. +func TestOpenEvalConfigNeverSeesTheFileVanish(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "azure.eval.yaml") + full := &EvalConfig{Evals: []Eval{{Name: "first", EvaluationLevel: "turn"}}} + require.NoError(t, SaveEvalConfigTo(path, full)) + + var wg sync.WaitGroup + stop := make(chan struct{}) + var vanished, replacements int64 + + wg.Add(1) + go func() { + defer wg.Done() + // Wall clock, not an iteration count. Three hundred os.Stat calls take + // microseconds, which is not long enough for the writer to be scheduled + // even once -- the test passed against the unlinking version it was + // written to catch. + deadline := time.Now().Add(500 * time.Millisecond) + for time.Now().Before(deadline) { + if _, err := os.Stat(path); os.IsNotExist(err) { + atomic.AddInt64(&vanished, 1) + } + } + close(stop) + }() + wg.Add(1) + go func() { + defer wg.Done() + for { + select { + case <-stop: + return + default: + if SaveEvalConfigTo(path, full) == nil { + atomic.AddInt64(&replacements, 1) + } + } + } + }() + wg.Wait() + + require.NotZero(t, atomic.LoadInt64(&replacements), + "the writer has to have replaced the file, or nothing was under test") + assert.Zerof(t, vanished, "the config was absent %d times during replacement", vanished) } // The replacement must leave the file complete and parseable. diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_lock.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_lock.go new file mode 100644 index 00000000000..5820fc28f1d --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_lock.go @@ -0,0 +1,89 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package project + +import ( + "context" + "errors" + "log" + "os" + "path/filepath" + "time" + + "azureaieval/internal/messages" + + "github.com/gofrs/flock" +) + +// configLockTimeout bounds the wait for another process's read-modify-write. +// Nothing that holds this lock waits on a person -- the evaluator prompt is +// deliberately outside it -- so a wait longer than this is a stale lock rather +// than contention. +const configLockTimeout = 30 * time.Second + +// evalConfigLockName is the lock file, beside the configuration it guards. +// +// Not in the OS temp directory, which looked tidier and was wrong twice over: a +// lock file there is created 0600 by whoever runs first, so a second user on +// the same machine can never open it and silently never locks; and two +// containers bind-mounting one project have separate temp directories, so they +// never see each other's lock at all. Beside the config it shares the project's +// lifetime, permissions and mount, and the `git status` noise that argued for +// temp is answered by ignoreLockFile. +const evalConfigLockName = ".azure.eval.lock" + +// LockEvalConfig serialises read-modify-write on the configuration across +// processes, returning the release function and whether the lock was taken. +// +// Updating the configuration means reading the file, adding an entry and +// writing it back. Two processes doing that at once can both read the same +// state, and the second write then drops the first one's entry -- a lost update +// that reports success on both sides. The atomic write stops a reader seeing a +// half-written file; it cannot stop this. +// +// Advisory and best-effort: a lock that could not be taken is logged and the +// work goes ahead, because failing a scaffold over a lock file would be worse +// than the lost update it guards against. The boolean is what lets a caller +// tell the difference. +func LockEvalConfig(ctx context.Context, evalDir string) (func(), bool, error) { + if ctx == nil { + // cobra hands a nil context to a command that was not run through + // Execute, and waiting on nil panics. + ctx = context.Background() + } + if err := os.MkdirAll(evalDir, 0o750); err != nil { + return nil, false, messages.Creating(evalDir, err) + } + + lock := flock.New(filepath.Join(evalDir, evalConfigLockName)) + ignoreLockFile(evalDir) + waitCtx, cancel := context.WithTimeout(ctx, configLockTimeout) + defer cancel() + + locked, err := lock.TryLockContext(waitCtx, 50*time.Millisecond) + if err != nil || !locked { + // Said out loud rather than swallowed. A lost update that happened + // because the lock was never held is otherwise unexplainable after the + // fact, and the previous version returned a nil error on every path, + // which made the callers' error handling dead code. + log.Printf("[lock] proceeding without the config lock for %s (locked=%t): %v", + evalDir, locked, err) + return func() {}, false, nil + } + return func() { _ = lock.Unlock() }, true, nil +} + +// ignoreLockFile keeps the lock out of `git status`, which is the one thing the +// OS temp directory had going for it. +// +// Only when there is no .gitignore of its own to respect: editing a file the +// user maintains is not this function's business, and a visible lock file is a +// far smaller problem than a surprising edit. +func ignoreLockFile(evalDir string) { + path := filepath.Join(evalDir, ".gitignore") + if _, err := os.Stat(path); err == nil || !errors.Is(err, os.ErrNotExist) { + return + } + _ = os.WriteFile(path, []byte(evalConfigLockName+"\n"), 0o600) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_lock_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_lock_test.go new file mode 100644 index 00000000000..5d5f76f4608 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_lock_test.go @@ -0,0 +1,94 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package project + +import ( + "context" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// The lock has to say whether it was taken. The first version returned a nil +// error on every path, including the ones where it gave up, so the callers' +// error handling was dead code and a lost update caused by an unheld lock was +// impossible to explain afterwards. +func TestLockEvalConfigReportsWhetherItWasTaken(t *testing.T) { + dir := filepath.Join(t.TempDir(), "evals") + + unlock, locked, err := LockEvalConfig(context.Background(), dir) + require.NoError(t, err) + require.True(t, locked, "an uncontended lock has to be taken") + require.NotNil(t, unlock) + unlock() + + // And it is reusable once released. + unlock2, locked2, err := LockEvalConfig(context.Background(), dir) + require.NoError(t, err) + assert.True(t, locked2, "releasing has to make it available again") + unlock2() +} + +// The lock lives beside the configuration it guards, not in the OS temp +// directory. Temp looked tidier and was wrong twice over: the file is created +// 0600 by whoever runs first, so a second user on the same machine can never +// open it and silently never locks; and two containers bind-mounting one +// project have separate temp directories, so they never see each other's lock. +func TestLockEvalConfigLivesBesideTheConfig(t *testing.T) { + dir := filepath.Join(t.TempDir(), "evals") + + unlock, _, err := LockEvalConfig(context.Background(), dir) + require.NoError(t, err) + defer unlock() + + _, err = os.Stat(filepath.Join(dir, evalConfigLockName)) + assert.NoError(t, err, "the lock belongs in the directory it guards") +} + +// Beside the config means inside a directory the user commits, so the lock has +// to keep itself out of `git status` -- which is the one thing the temp +// directory had going for it. +func TestLockEvalConfigIgnoresItself(t *testing.T) { + dir := filepath.Join(t.TempDir(), "evals") + + unlock, _, err := LockEvalConfig(context.Background(), dir) + require.NoError(t, err) + defer unlock() + + body, err := os.ReadFile(filepath.Join(dir, ".gitignore")) + require.NoError(t, err) + assert.Contains(t, string(body), evalConfigLockName) +} + +// A .gitignore the user maintains is theirs. Appending to it is a surprising +// edit, and a visible lock file is the far smaller problem. +func TestLockEvalConfigLeavesAnExistingGitignoreAlone(t *testing.T) { + dir := filepath.Join(t.TempDir(), "evals") + require.NoError(t, os.MkdirAll(dir, 0o750)) + theirs := filepath.Join(dir, ".gitignore") + require.NoError(t, os.WriteFile(theirs, []byte("*.local\n"), 0o600)) + + unlock, _, err := LockEvalConfig(context.Background(), dir) + require.NoError(t, err) + defer unlock() + + body, err := os.ReadFile(theirs) + require.NoError(t, err) + assert.Equal(t, "*.local\n", string(body), "the user's file is not ours to edit") +} + +// cobra hands a nil context to a command that was not run through Execute, and +// waiting on a nil context panics. +func TestLockEvalConfigToleratesANilContext(t *testing.T) { + dir := filepath.Join(t.TempDir(), "evals") + + //nolint:staticcheck // the nil context is the case under test + unlock, _, err := LockEvalConfig(nil, dir) + require.NoError(t, err) + require.NotNil(t, unlock) + unlock() +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_readonly_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_readonly_test.go new file mode 100644 index 00000000000..2dc9974a1db --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_readonly_test.go @@ -0,0 +1,49 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package project + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// A config can be read-only: a Perforce or TFVC checkout marks files that way +// by default, as does `attrib +R` and some archive extractions. +// +// This used to work by accident. The replacement removed the destination first, +// and os.Remove clears FILE_ATTRIBUTE_READONLY and retries the delete, so the +// attribute never reached the rename. Dropping the unlink -- which was right, +// because it opened a window where the config did not exist -- took that repair +// with it, and Windows reports a rename onto a read-only destination with the +// same errno as one a reader holds open, so it cannot be told apart earlier. +func TestSaveEvalConfigReplacesAReadOnlyFile(t *testing.T) { + path := filepath.Join(t.TempDir(), "azure.eval.yaml") + first := &EvalConfig{Evals: []Eval{{Name: "first", EvaluationLevel: "turn"}}} + require.NoError(t, SaveEvalConfigTo(path, first)) + require.NoError(t, os.Chmod(path, 0o444)) + + second := &EvalConfig{Evals: []Eval{{Name: "second", EvaluationLevel: "turn"}}} + require.NoError(t, SaveEvalConfigTo(path, second), + "a read-only config has to be replaceable, as it was before the rename") + + got, err := LoadEvalConfig(path) + require.NoError(t, err) + require.Len(t, got.Evals, 1) + assert.Equal(t, "second", got.Evals[0].Name, "and the new content has to be there") +} + +// The retry exists for a window measured in microseconds. A file that is +// genuinely unreadable shares an errno with that window on Windows, so it pays +// the budget before it is reported -- which is only acceptable while the budget +// stays small. +func TestContentionBudgetsStaySmall(t *testing.T) { + assert.LessOrEqual(t, readRetryBudget.Milliseconds(), int64(250), + "every unreadable file pays this, and ReadFileNoBOM reads one per evaluator") + assert.LessOrEqual(t, renameRetryBudget.Milliseconds(), int64(500), + "a read-only destination waits this out before the attribute is cleared") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_store.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_store.go index 06cedfb5276..d0b49d41c16 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_store.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_store.go @@ -10,6 +10,9 @@ import ( "io/fs" "os" "path/filepath" + "runtime" + "syscall" + "time" "azureaieval/internal/messages" @@ -177,12 +180,96 @@ func SaveEvalConfigTo(path string, cfg *EvalConfig) error { if err := tmp.Close(); err != nil { return messages.WritingEvalConfig(path, err) } - // Windows will not rename onto an existing file. - if err := os.Remove(path); err != nil && !errors.Is(err, os.ErrNotExist) { - return messages.WritingEvalConfig(path, err) - } - if err := os.Rename(tmpName, path); err != nil { - return messages.WritingEvalConfig(path, err) + // Straight over the destination, and never by unlinking it first. Windows + // refuses a rename while a reader holds the destination open, so the + // obvious fallback -- remove, then rename -- turns a collision into a + // window where the config does not exist, and OpenEvalConfig reports a + // missing file as "no configuration yet", which callers answer by writing a + // fresh one. That is the same data loss this function exists to prevent. + // Contention is measured in microseconds, so it is waited out instead. + if err := renameOverContention(tmpName, path); err != nil { + // The unlink this replaced was doing something else worth keeping: + // os.Remove clears a read-only attribute and retries, so a config marked + // read-only (a Perforce or TFVC checkout, `attrib +R`, some archive + // extractions) could still be replaced. Windows fails a rename onto a + // read-only destination with the same errno as one a reader holds open, + // so the two cannot be told apart before the wait. + if !clearReadOnly(path) { + return messages.WritingEvalConfig(path, err) + } + if err := os.Rename(tmpName, path); err != nil { + return messages.WritingEvalConfig(path, err) + } } return nil } + +// clearReadOnly drops a read-only attribute, reporting whether it had one to +// drop. os.Chmod is what carries FILE_ATTRIBUTE_READONLY on Windows. +func clearReadOnly(path string) bool { + info, err := os.Stat(path) + if err != nil || info.Mode().Perm()&0o200 != 0 { + return false + } + return os.Chmod(path, info.Mode().Perm()|0o200) == nil +} + +// The budgets are deliberately different. A replacement window is measured in +// microseconds, so neither needs to be generous -- and every millisecond here +// is also charged to a file that is genuinely unreadable, because Windows +// reports "someone has this open" and "you may not have this" as one errno. +const ( + renameRetryBudget = 500 * time.Millisecond + readRetryBudget = 250 * time.Millisecond +) + +func renameOverContention(from, to string) error { + deadline := time.Now().Add(renameRetryBudget) + delay := time.Millisecond + for { + err := os.Rename(from, to) + if err == nil || !isSharingContention(err) || time.Now().After(deadline) { + return err + } + time.Sleep(delay) + if delay < 16*time.Millisecond { + delay *= 2 + } + } +} + +// isSharingContention reports the errors Windows raises while another handle is +// open. It cannot be precise: renaming onto a destination a reader holds open +// and renaming onto one the caller may not touch both report ERROR_ACCESS_DENIED, +// so a genuine permission failure is waited on before it is reported. The +// budget is what keeps that wait short enough to be worth the trade. +func isSharingContention(err error) bool { + if err == nil || errors.Is(err, os.ErrNotExist) { + return false + } + if runtime.GOOS != "windows" { + return false + } + var errno syscall.Errno + if !errors.As(err, &errno) { + return false + } + // ERROR_ACCESS_DENIED and ERROR_SHARING_VIOLATION. + return errno == 5 || errno == 32 +} + +// readFileOverContention reads a file that another process may be replacing. +func readFileOverContention(path string) ([]byte, error) { + deadline := time.Now().Add(readRetryBudget) + delay := time.Millisecond + for { + body, err := os.ReadFile(path) + if err == nil || !isSharingContention(err) || time.Now().After(deadline) { + return body, err + } + time.Sleep(delay) + if delay < 16*time.Millisecond { + delay *= 2 + } + } +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/max_samples_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/max_samples_test.go new file mode 100644 index 00000000000..218f5c159bb --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/max_samples_test.go @@ -0,0 +1,53 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package project + +import ( + "testing" + + "azureaieval/internal/pkg/evalcore" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// resolveMaxSamples reads anything not above zero as "no cap", so a negative +// max_samples used to send the WHOLE dataset to a run that is billed per row -- +// the opposite of what a cap asks for, and with nothing said about it. +func TestNegativeMaxSamplesIsRefused(t *testing.T) { + cfg := &EvalConfig{ + Datasets: []DatasetDecl{{Name: "golden", Source: "./datasets/golden.jsonl"}}, + Evals: []Eval{{ + Name: "support-quality", + Dataset: "golden", + EvaluationLevel: "turn", + MaxSamples: -1, + Evaluators: evalcore.EvaluatorList{{Evaluator: "builtin.relevance"}}, + }}, + } + + err := cfg.Validate() + + require.Error(t, err) + assert.Contains(t, err.Error(), "max_samples") + assert.Contains(t, err.Error(), "support-quality", "the eval that carries it") + assert.Contains(t, err.Error(), "-1", "and the value that was rejected") +} + +// Zero is how a config says "send every row", and has to keep working. +func TestUnsetMaxSamplesIsStillAllowed(t *testing.T) { + cfg := &EvalConfig{ + Datasets: []DatasetDecl{{Name: "golden", Source: "./datasets/golden.jsonl"}}, + Evals: []Eval{{ + Name: "support-quality", + Dataset: "golden", + EvaluationLevel: "turn", + Evaluators: evalcore.EvaluatorList{{Evaluator: "builtin.relevance"}}, + }}, + } + require.NoError(t, cfg.Validate()) + + cfg.Evals[0].MaxSamples = 25 + assert.NoError(t, cfg.Validate(), "a positive cap is the ordinary case") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/readfile.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/readfile.go index 446d3531e11..dbb6a249284 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/readfile.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/readfile.go @@ -5,7 +5,6 @@ package project import ( "bytes" - "os" ) // utf8BOM is what Windows editors and PowerShell's Set-Content write ahead of @@ -19,7 +18,7 @@ var utf8BOM = []byte{0xEF, 0xBB, 0xBF} // character rather than the cause: "invalid character 'ï' looking for beginning // of value" is not something a developer can act on. func ReadFileNoBOM(path string) ([]byte, error) { - data, err := os.ReadFile(path) //nolint:gosec // a path the caller named + data, err := readFileOverContention(path) if err != nil { return nil, err } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/resolve_source_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/resolve_source_test.go new file mode 100644 index 00000000000..7f1966391aa --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/resolve_source_test.go @@ -0,0 +1,30 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package project + +import ( + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" +) + +// A declared source is relative to the configuration, but a user may write an +// absolute one. `eval create` used to join it unconditionally, producing +// evals/C:/data/rows.jsonl, while `azd up` resolved it correctly -- so the same +// file worked or failed depending on which command published it. +func TestResolveSourceLeavesAnAbsolutePathAlone(t *testing.T) { + abs := filepath.Join(string(filepath.Separator), "data", "rows.jsonl") + if filepath.VolumeName(`C:\`) != "" { + abs = `C:\data\rows.jsonl` + } + + assert.Equal(t, abs, ResolveSource("evals", abs), + "an absolute source is already where it says it is") + assert.Equal(t, filepath.Join("evals", "datasets", "rows.jsonl"), + ResolveSource("evals", "./datasets/rows.jsonl"), + "a relative source hangs off the configuration's directory") + assert.Empty(t, ResolveSource("evals", ""), + "nothing declared stays nothing, rather than becoming the directory") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go index f502adf6dc3..ebfe560e61f 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go @@ -157,7 +157,7 @@ func (p *EvalServiceTargetProvider) Deploy( datasetPaths := map[string]string{} for _, decl := range cfg.Datasets { report(progress, messages.ReconcilingDataset(decl.Name)) - localPath := resolveSource(baseDir, decl.Source) + localPath := ResolveSource(baseDir, decl.Source) datasetPaths[decl.Name] = localPath version, changed, err := reconciler.EnsureDataset(ctx, decl, localPath) if err != nil { @@ -170,7 +170,7 @@ func (p *EvalServiceTargetProvider) Deploy( // ones need no publish. for _, decl := range cfg.CustomEvaluators() { report(progress, messages.ReconcilingEvaluator(decl.Name)) - localPath := resolveSource(baseDir, decl.Source) + localPath := ResolveSource(baseDir, decl.Source) version, changed, err := reconciler.EnsureEvaluator(ctx, decl, localPath) if err != nil { return nil, messages.EvaluatorProblem(decl.Name, err) @@ -300,9 +300,14 @@ func serviceRelativeDir(svc *azdext.ServiceConfig) string { return "." } -// resolveSource joins a declared source against the service directory, leaving -// absolute paths and empty values alone. -func resolveSource(baseDir, source string) string { +// ResolveSource joins a declared source against the directory holding the +// configuration, leaving absolute paths and empty values alone. +// +// Exported because `eval create` resolves the same declarations as `azd up` +// and had grown its own copy that joined unconditionally, so an absolute +// source came out as evals/C:/data/rows.jsonl there while `azd up` handled it. +// One resolver is what stops the two drifting again. +func ResolveSource(baseDir, source string) string { if source == "" { return "" } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/version_spelling_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/version_spelling_test.go new file mode 100644 index 00000000000..da6e50e25eb --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/version_spelling_test.go @@ -0,0 +1,74 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package project + +import ( + "testing" + + "github.com/azure/azure-dev/cli/azd/pkg/azdext" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/types/known/structpb" +) + +// An unquoted version is a number, and `azd up` reads the configuration through +// the service entry rather than off disk. That route arrives as protobuf, whose +// only numeric kind is a double, so `1` and `1.0` are the same value by the +// time this extension is handed it -- and it renders as "1", while reading the +// same file off disk gives "1.0". +// +// The spelling is destroyed before this code runs, so there is nothing here to +// recover it from: emitting "1.0" would break a config that meant 1, and +// rejecting the number would fail a file that `azd ai eval run` accepts. This +// pins the divergence so it is visible and cannot widen unnoticed. +// +// The fix available to a user is to quote it, which both routes preserve. +func TestNumericVersionLosesItsSpellingOnTheServiceRoute(t *testing.T) { + fromService := func(t *testing.T, version any) string { + t.Helper() + props, err := structpb.NewStruct(map[string]any{ + "evals": []any{map[string]any{ + "name": "support-quality", + "dataset": "golden", + "evaluation_level": "turn", + "evaluators": []any{map[string]any{ + "evaluator": "builtin.relevance", + "version": version, + }}, + }}, + }) + require.NoError(t, err) + + cfg, err := EvalConfigFromService( + &azdext.ServiceConfig{Name: "evals", AdditionalProperties: props}, "") + require.NoError(t, err) + require.Len(t, cfg.Evals, 1) + require.Len(t, cfg.Evals[0].Evaluators, 1) + return cfg.Evals[0].Evaluators[0].Version + } + + assert.Equal(t, "1", fromService(t, 1.0), + "1.0 and 1 are one number in protobuf, so the decimal cannot survive") + assert.Equal(t, "1.5", fromService(t, 1.5), + "a fractional part is not lost, only a trailing zero") + assert.Equal(t, "1.0", fromService(t, "1.0"), + "quoting is what carries the spelling through, and is the advice to give") +} + +// The same file read off disk keeps what the user wrote, which is the half of +// the divergence that behaves. +func TestQuotedAndUnquotedVersionsOffDisk(t *testing.T) { + cfg, err := DecodeEvalConfig([]byte(` +evals: + - name: support-quality + dataset: golden + evaluation_level: turn + evaluators: + - evaluator: builtin.relevance + version: 1.0 +`), "eval.yaml") + require.NoError(t, err) + assert.Equal(t, "1.0", cfg.Evals[0].Evaluators[0].Version, + "YAML hands a string field the scalar as written") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/version.txt b/cli/azd/extensions/azure.ai.evaluations/version.txt index 15f5cb52e6a..1bc145e2040 100644 --- a/cli/azd/extensions/azure.ai.evaluations/version.txt +++ b/cli/azd/extensions/azure.ai.evaluations/version.txt @@ -1 +1 @@ -1.0.0-beta.10 +1.0.0-beta.11 \ No newline at end of file From 971de0de55d23b90aa8c98051378aed23440efe2 Mon Sep 17 00:00:00 2001 From: mohessie Date: Wed, 12 Aug 2026 04:57:25 +0300 Subject: [PATCH 193/320] Report the two failures that were only ever written to a discarded log remember() and LockEvalConfig both reported through the standard logger, which root.go points at io.Discard unless --debug. Both carried a comment saying the failure was said out loud. Neither said anything, so the previous commit claimed ground it had not taken: a state write that fails for a reason other than "no azd environment" was exactly as silent as the `_ =` it replaced. Both now write to stderr, which keeps `-o json` on stdout parseable. LockEvalConfig also returned a locked bool documented as "what lets a caller tell the difference", which both callers discarded. Removed rather than left as an API nobody reads. The lock's .gitignore was written before the lock was attempted, so a process that never took the lock still left a file behind in a directory the user commits. It is written only once the file is ours. The artifact-scoped flag guards covered two of five cases in the same function. --dataset --trace-days 7, --dataset --evaluator-name foo and --evaluator --dataset-name bar were all still accepted and dropped. Replaced the ad-hoc pair with one table over every flag that buildGeneratePlans reads under only one artifact. Also removes a duplicate doc comment the previous commit added to SampleSizeOutOfRange, and a pre-existing one on ReattachToJob. --- .../azure.ai.evaluations/extension.yaml | 2 +- .../internal/cmd/catalog.go | 2 +- .../internal/cmd/context.go | 9 +++ .../internal/cmd/generate_composite.go | 37 ++++++++-- .../internal/cmd/generate_flag_guards_test.go | 72 +++++++++++++------ .../azure.ai.evaluations/internal/cmd/init.go | 2 +- .../internal/messages/messages.go | 36 +++++++--- .../internal/project/eval_config_lock.go | 31 ++++---- .../internal/project/eval_config_lock_test.go | 25 ++++--- .../azure.ai.evaluations/version.txt | 2 +- 10 files changed, 148 insertions(+), 70 deletions(-) diff --git a/cli/azd/extensions/azure.ai.evaluations/extension.yaml b/cli/azd/extensions/azure.ai.evaluations/extension.yaml index 0a2338700dc..2dca34eea0b 100644 --- a/cli/azd/extensions/azure.ai.evaluations/extension.yaml +++ b/cli/azd/extensions/azure.ai.evaluations/extension.yaml @@ -5,7 +5,7 @@ displayName: Foundry evaluations (Beta) description: Define and run Foundry evaluations from your terminal. (Beta) usage: azd ai eval [options] # NOTE: Make sure version.txt is in sync with this version. -version: 1.0.0-beta.11 +version: 1.0.0-beta.12 requiredAzdVersion: ">=1.27.1" language: go capabilities: diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/catalog.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/catalog.go index d9898ffedad..5e553e953a0 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/catalog.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/catalog.go @@ -82,7 +82,7 @@ func updateCatalog( // Held across the read and the write: two generates adding different // entries would otherwise both read the same state, and the second write // would drop the first one's entry while reporting success. - unlock, _, err := project.LockEvalConfig(cmd.Context(), evalDir) + unlock, err := project.LockEvalConfig(cmd.Context(), evalDir) if err != nil { return err } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/context.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/context.go index d5198f9e9f1..12245b3f873 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/context.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/context.go @@ -6,7 +6,9 @@ package cmd import ( "context" "errors" + "fmt" "log" + "os" "strings" "azureaieval/internal/foundry/projectctx" @@ -119,11 +121,18 @@ var errNoAzdEnvironment = messages.ErrNoAzdEnvironment // worth a word. Anything else is: these keys are how a later deploy recognises // what it already published, and losing one silently means the next `azd up` // creates a second immutable version of something it had already created. +// +// Written to stderr, not through log: the standard logger is pointed at +// io.Discard unless --debug, so logging this would be the same silence with a +// more reassuring name. stderr keeps `-o json` on stdout parseable. azd does +// not surface an extension's stderr, so under `azd up` this reaches the debug +// log and no further -- direct invocations are where it shows. func (ec *evalContext) remember(ctx context.Context, key, value string) { err := ec.setEnvValue(ctx, key, value) if err == nil || errors.Is(err, errNoAzdEnvironment) { return } + fmt.Fprint(os.Stderr, messages.Warning(err)) log.Printf("[env] could not record %s: %v", key, err) } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_composite.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_composite.go index d2e0cc93cd9..3fb8b4f45c3 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_composite.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_composite.go @@ -58,12 +58,8 @@ func newGenerateCommand() *cobra.Command { // costs nothing to find out about. Changed() rather than the value, // so a zero the caller actually typed is still caught and an // untouched default is not. - if !dataset { - for _, flag := range []string{"from", "max-samples"} { - if cmd.Flags().Changed(flag) { - return messages.DatasetOnlyFlag(flag) - } - } + if err := refuseInapplicableFlags(cmd, dataset, evaluator); err != nil { + return err } if traceDays < 0 { return messages.NegativeTraceDays(traceDays) @@ -165,6 +161,35 @@ type generateRequest struct { traceDays int } +// artifactScopedFlags are the flags buildGeneratePlans reads only while +// building one kind of artifact. Given for the other kind they were accepted +// and dropped, so `--dataset --trace-days 7` produced a dataset and said +// nothing about the seven days. +var artifactScopedFlags = []struct { + name string + forEval bool // read under req.evaluator rather than req.dataset + otherFor string +}{ + {name: "from", otherFor: "--evaluator"}, + {name: "max-samples", otherFor: "--evaluator"}, + {name: "dataset-name", otherFor: "--evaluator"}, + {name: "trace-days", forEval: true, otherFor: "--dataset"}, + {name: "evaluator-name", forEval: true, otherFor: "--dataset"}, +} + +func refuseInapplicableFlags(cmd *cobra.Command, dataset, evaluator bool) error { + for _, f := range artifactScopedFlags { + applies := dataset + if f.forEval { + applies = evaluator + } + if !applies && cmd.Flags().Changed(f.name) { + return messages.FlagDoesNotApply(f.name, f.otherFor) + } + } + return nil +} + // buildGeneratePlans settles everything that does not need the network, for // each artifact asked for. Ordered dataset first, which is the order their // progress is replayed in. diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_flag_guards_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_flag_guards_test.go index 8f7b2d2e328..6445937d1b9 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_flag_guards_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_flag_guards_test.go @@ -22,32 +22,64 @@ func runGenerate(t *testing.T, args ...string) error { return root.ExecuteContext(context.Background()) } -// --from and --max-samples are documented "Dataset only", and --evaluator -// generates no dataset. They were accepted and ignored, so the run produced a -// rubric and said nothing about the sample count that was asked for. -func TestDatasetOnlyFlagsAreRefusedForEvaluatorOnlyGeneration(t *testing.T) { - for flag, args := range map[string][]string{ - "--from": {"--evaluator", "--evaluator-name", "ev", "--from", "prompt"}, - "--max-samples": {"--evaluator", "--evaluator-name", "ev", "--max-samples", "50"}, - } { - err := runGenerate(t, args...) - - require.Errorf(t, err, "%s cannot apply to an evaluator-only generation", flag) - assert.Contains(t, err.Error(), flag) - assert.Contains(t, err.Error(), "--evaluator", - "the refusal has to name the flag that made it inapplicable") +// Each of these is read while building one kind of artifact and ignored while +// building the other, so given for the wrong one they were accepted and +// dropped -- `--evaluator --max-samples 50` produced a rubric and said nothing +// about the 50, and `--dataset --trace-days 7` a dataset and nothing about the +// seven days. +func TestFlagsThatCannotApplyAreRefused(t *testing.T) { + cases := []struct { + flag string + narrowed string + args []string + }{ + {"--from", "--evaluator", + []string{"--evaluator", "--evaluator-name", "ev", "--from", "prompt"}}, + {"--max-samples", "--evaluator", + []string{"--evaluator", "--evaluator-name", "ev", "--max-samples", "50"}}, + {"--dataset-name", "--evaluator", + []string{"--evaluator", "--evaluator-name", "ev", "--dataset-name", "ds"}}, + {"--trace-days", "--dataset", + []string{"--dataset", "--dataset-name", "ds", "--trace-days", "7"}}, + {"--evaluator-name", "--dataset", + []string{"--dataset", "--dataset-name", "ds", "--evaluator-name", "ev"}}, + } + + for _, c := range cases { + err := runGenerate(t, c.args...) + + require.Errorf(t, err, "%s cannot apply under %s", c.flag, c.narrowed) + assert.Contains(t, err.Error(), c.flag) + assert.Containsf(t, err.Error(), c.narrowed, + "the refusal has to name the flag that made %s inapplicable", c.flag) } } -// The same flags with a dataset selected are the ordinary case, so the refusal -// above has to be about the combination rather than about the flags. -func TestDatasetOnlyFlagsAreStillAcceptedForADataset(t *testing.T) { +// Generating both is the default, and every one of those flags applies then. +// Without this the guard above could be satisfied by refusing them always. +func TestNoFlagIsRefusedWhenBothArtifactsAreGenerated(t *testing.T) { err := runGenerate(t, - "--dataset", "--dataset-name", "ds", "--from", "prompt", "--max-samples", "50") + "--dataset-name", "ds", "--evaluator-name", "ev", + "--from", "prompt", "--max-samples", "50", "--trace-days", "7") if err != nil { - assert.NotContains(t, err.Error(), "only affects the dataset", - "a dataset generation must not be refused its own flags") + assert.NotContains(t, err.Error(), "has no effect on what", + "generating both artifacts makes every one of these flags applicable") + } +} + +// And each flag is still accepted for the artifact it does affect. +func TestEachFlagIsAcceptedForItsOwnArtifact(t *testing.T) { + forDataset := runGenerate(t, + "--dataset", "--dataset-name", "ds", "--from", "prompt", "--max-samples", "50") + if forDataset != nil { + assert.NotContains(t, forDataset.Error(), "has no effect on what") + } + + forEvaluator := runGenerate(t, + "--evaluator", "--evaluator-name", "ev", "--trace-days", "7") + if forEvaluator != nil { + assert.NotContains(t, forEvaluator.Error(), "has no effect on what") } } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go index 8ccf4d02e7b..ee21050686c 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go @@ -140,7 +140,7 @@ func newInitCommand() *cobra.Command { // a person. The configuration is read again because the copy above // was taken before the prompt, and a `generate` may well have // finished writing to it since. - unlockConfig, _, err := project.LockEvalConfig(cmd.Context(), path) + unlockConfig, err := project.LockEvalConfig(cmd.Context(), path) if err != nil { return err } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go b/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go index c3fcc1ea585..72ddbb0cce0 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go @@ -739,7 +739,6 @@ func JobSubmitted(jobID string) string { return fmt.Sprintf(" submitted job %s\n", jobID) } -// ReattachToJob says how to come back to a job started with --no-wait. // ReattachToJob says how to come back to a job started with --no-wait. // // The selector is part of the line because `job` requires it: the two @@ -1897,8 +1896,22 @@ func FromNotASource(from string, sources []string) error { from, strings.Join(sources, ", ")) } +// ConfigLockUnavailable reports a config lock that could not be taken. +// +// Not fatal, and said out loud for that reason: the work goes ahead unlocked, +// so a lost update afterwards has no other explanation on record. +func ConfigLockUnavailable(evalDir string, err error) error { + if err == nil { + return fmt.Errorf( + "another process is still updating %s, so this update is not "+ + "serialised against it", filepath.ToSlash(evalDir)) + } + return fmt.Errorf( + "could not lock %s, so this update is not serialised against other "+ + "processes: %w", filepath.ToSlash(evalDir), err) +} + // SampleSizeOutOfRange reports a row count the generation service would reject. -// SampleSizeOutOfRange reports a generation sample count the service will not take. func SampleSizeOutOfRange(min, max, got int) error { return fmt.Errorf("sample size must be between %d and %d, got %d", min, max, got) } @@ -1922,16 +1935,17 @@ func NegativeMaxSamplesFlag(got int) error { "Omit it to send every row, or give the number of rows to send", got) } -// DatasetOnlyFlag reports a dataset flag given to a run that generates no -// dataset. +// FlagDoesNotApply reports a flag given to a generate that produces nothing it +// could affect. // -// The flag is documented "Dataset only" and was accepted and ignored, so -// `--evaluator --max-samples 50` produced a rubric and said nothing about the -// number the caller asked for. -func DatasetOnlyFlag(flag string) error { - return fmt.Errorf( - "--%s only affects the dataset, and --evaluator generates no dataset. "+ - "Drop --%s, or drop --evaluator to generate both", flag, flag) +// Each of these is read while building one kind of artifact and ignored while +// building the other, so given for the wrong one they were accepted and +// dropped: `--evaluator --max-samples 50` produced a rubric and said nothing +// about the 50. +func FlagDoesNotApply(flag, narrowedBy string) error { + return fmt.Errorf( + "--%s has no effect on what %s generates. "+ + "Drop --%s, or drop %s to generate both", flag, narrowedBy, flag, narrowedBy) } // NegativeTraceDays reports a trace window below zero. diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_lock.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_lock.go index 5820fc28f1d..e88a9de5e2f 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_lock.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_lock.go @@ -6,7 +6,7 @@ package project import ( "context" "errors" - "log" + "fmt" "os" "path/filepath" "time" @@ -34,7 +34,7 @@ const configLockTimeout = 30 * time.Second const evalConfigLockName = ".azure.eval.lock" // LockEvalConfig serialises read-modify-write on the configuration across -// processes, returning the release function and whether the lock was taken. +// processes, returning the release function. // // Updating the configuration means reading the file, adding an entry and // writing it back. Two processes doing that at once can both read the same @@ -42,36 +42,35 @@ const evalConfigLockName = ".azure.eval.lock" // that reports success on both sides. The atomic write stops a reader seeing a // half-written file; it cannot stop this. // -// Advisory and best-effort: a lock that could not be taken is logged and the +// Advisory and best-effort: a lock that could not be taken is reported and the // work goes ahead, because failing a scaffold over a lock file would be worse -// than the lost update it guards against. The boolean is what lets a caller -// tell the difference. -func LockEvalConfig(ctx context.Context, evalDir string) (func(), bool, error) { +// than the lost update it guards against. Reported on stderr rather than +// through log, which is pointed at io.Discard unless --debug -- an earlier +// version logged it and was therefore exactly as silent as saying nothing. +func LockEvalConfig(ctx context.Context, evalDir string) (func(), error) { if ctx == nil { // cobra hands a nil context to a command that was not run through // Execute, and waiting on nil panics. ctx = context.Background() } if err := os.MkdirAll(evalDir, 0o750); err != nil { - return nil, false, messages.Creating(evalDir, err) + return nil, messages.Creating(evalDir, err) } lock := flock.New(filepath.Join(evalDir, evalConfigLockName)) - ignoreLockFile(evalDir) waitCtx, cancel := context.WithTimeout(ctx, configLockTimeout) defer cancel() locked, err := lock.TryLockContext(waitCtx, 50*time.Millisecond) if err != nil || !locked { - // Said out loud rather than swallowed. A lost update that happened - // because the lock was never held is otherwise unexplainable after the - // fact, and the previous version returned a nil error on every path, - // which made the callers' error handling dead code. - log.Printf("[lock] proceeding without the config lock for %s (locked=%t): %v", - evalDir, locked, err) - return func() {}, false, nil + fmt.Fprint(os.Stderr, messages.Warning(messages.ConfigLockUnavailable(evalDir, err))) + return func() {}, nil } - return func() { _ = lock.Unlock() }, true, nil + // Only once the file is ours: a lock that was never taken has no artifact + // to hide, and writing into a directory the user commits is not something + // to do on the way past. + ignoreLockFile(evalDir) + return func() { _ = lock.Unlock() }, nil } // ignoreLockFile keeps the lock out of `git status`, which is the one thing the diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_lock_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_lock_test.go index 5d5f76f4608..426b45f24c5 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_lock_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_lock_test.go @@ -13,23 +13,22 @@ import ( "github.com/stretchr/testify/require" ) -// The lock has to say whether it was taken. The first version returned a nil -// error on every path, including the ones where it gave up, so the callers' -// error handling was dead code and a lost update caused by an unheld lock was -// impossible to explain afterwards. -func TestLockEvalConfigReportsWhetherItWasTaken(t *testing.T) { +// The lock is advisory: a scaffold must not fail because a lock file could +// not be taken, and an earlier version reported that case only through log, +// which is pointed at io.Discard unless --debug -- exactly as silent as saying +// nothing. It now reports on stderr. +func TestLockEvalConfigIsTakenAndReleased(t *testing.T) { dir := filepath.Join(t.TempDir(), "evals") - unlock, locked, err := LockEvalConfig(context.Background(), dir) + unlock, err := LockEvalConfig(context.Background(), dir) require.NoError(t, err) - require.True(t, locked, "an uncontended lock has to be taken") require.NotNil(t, unlock) unlock() // And it is reusable once released. - unlock2, locked2, err := LockEvalConfig(context.Background(), dir) + unlock2, err := LockEvalConfig(context.Background(), dir) require.NoError(t, err) - assert.True(t, locked2, "releasing has to make it available again") + require.NotNil(t, unlock2) unlock2() } @@ -41,7 +40,7 @@ func TestLockEvalConfigReportsWhetherItWasTaken(t *testing.T) { func TestLockEvalConfigLivesBesideTheConfig(t *testing.T) { dir := filepath.Join(t.TempDir(), "evals") - unlock, _, err := LockEvalConfig(context.Background(), dir) + unlock, err := LockEvalConfig(context.Background(), dir) require.NoError(t, err) defer unlock() @@ -55,7 +54,7 @@ func TestLockEvalConfigLivesBesideTheConfig(t *testing.T) { func TestLockEvalConfigIgnoresItself(t *testing.T) { dir := filepath.Join(t.TempDir(), "evals") - unlock, _, err := LockEvalConfig(context.Background(), dir) + unlock, err := LockEvalConfig(context.Background(), dir) require.NoError(t, err) defer unlock() @@ -72,7 +71,7 @@ func TestLockEvalConfigLeavesAnExistingGitignoreAlone(t *testing.T) { theirs := filepath.Join(dir, ".gitignore") require.NoError(t, os.WriteFile(theirs, []byte("*.local\n"), 0o600)) - unlock, _, err := LockEvalConfig(context.Background(), dir) + unlock, err := LockEvalConfig(context.Background(), dir) require.NoError(t, err) defer unlock() @@ -87,7 +86,7 @@ func TestLockEvalConfigToleratesANilContext(t *testing.T) { dir := filepath.Join(t.TempDir(), "evals") //nolint:staticcheck // the nil context is the case under test - unlock, _, err := LockEvalConfig(nil, dir) + unlock, err := LockEvalConfig(nil, dir) require.NoError(t, err) require.NotNil(t, unlock) unlock() diff --git a/cli/azd/extensions/azure.ai.evaluations/version.txt b/cli/azd/extensions/azure.ai.evaluations/version.txt index 1bc145e2040..e521dc4f817 100644 --- a/cli/azd/extensions/azure.ai.evaluations/version.txt +++ b/cli/azd/extensions/azure.ai.evaluations/version.txt @@ -1 +1 @@ -1.0.0-beta.11 \ No newline at end of file +1.0.0-beta.12 \ No newline at end of file From 467e2b04fc79fdbbe5f0e0ac6dfa699f64bc23c9 Mon Sep 17 00:00:00 2001 From: mohessie Date: Wed, 12 Aug 2026 05:19:01 +0300 Subject: [PATCH 194/320] Share the one replacement that survived being wrong twice writeFileAtomic in output.go removed the destination before renaming over it, under a comment asserting that Windows needs the destination gone first. It does not: os.Rename passes MOVEFILE_REPLACE_EXISTING and replaces. It fails on contention and on a read-only destination, with the same errno for both, which is what the config writer already learned the hard way -- the unlink there produced a measured 137 reads that saw no file at all, and a config that momentarily does not exist reads as "no configuration yet". So `evaluator show --output-file existing.json` could delete the file it was replacing and then report an error, which is the failure that fix exists to prevent, in a second copy nobody had looked at. Exported the proven one as project.ReplaceFile and pointed both writers at it: retry the rename through contention, and only if that is spent clear a read-only attribute and try once more. Never unlink. --- .../internal/cmd/output.go | 11 ++--- .../internal/project/eval_config_store.go | 40 +++++++++++++------ 2 files changed, 31 insertions(+), 20 deletions(-) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/output.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/output.go index dfa295440c4..ad2f2df5cdc 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/output.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/output.go @@ -14,6 +14,7 @@ import ( "text/tabwriter" "azureaieval/internal/messages" + "azureaieval/internal/project" "github.com/fatih/color" "github.com/spf13/cobra" @@ -161,9 +162,8 @@ func requireFlag(name string) error { // Every error names the path the caller passed. The temporary file is this // function's business and appears nowhere the caller asked for. func writeFileAtomic(path string, body []byte) error { - // The rename below needs the destination gone on Windows, so refuse - // anything that is not a regular file rather than removing it: pointed at a - // directory, this would otherwise delete it. + // Refuse anything that is not a regular file: pointed at a directory, the + // replacement below would report a confusing rename failure instead. switch info, err := os.Stat(path); { case err == nil && !info.Mode().IsRegular(): return messages.NotARegularFile(path) @@ -190,10 +190,7 @@ func writeFileAtomic(path string, body []byte) error { if err := tmp.Close(); err != nil { return messages.Creating(path, err) } - if err := os.Remove(path); err != nil && !errors.Is(err, os.ErrNotExist) { - return messages.Creating(path, err) - } - if err := os.Rename(tmpName, path); err != nil { + if err := project.ReplaceFile(tmpName, path); err != nil { return messages.Creating(path, err) } return nil diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_store.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_store.go index d0b49d41c16..c44241efdec 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_store.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_store.go @@ -187,23 +187,37 @@ func SaveEvalConfigTo(path string, cfg *EvalConfig) error { // missing file as "no configuration yet", which callers answer by writing a // fresh one. That is the same data loss this function exists to prevent. // Contention is measured in microseconds, so it is waited out instead. - if err := renameOverContention(tmpName, path); err != nil { - // The unlink this replaced was doing something else worth keeping: - // os.Remove clears a read-only attribute and retries, so a config marked - // read-only (a Perforce or TFVC checkout, `attrib +R`, some archive - // extractions) could still be replaced. Windows fails a rename onto a - // read-only destination with the same errno as one a reader holds open, - // so the two cannot be told apart before the wait. - if !clearReadOnly(path) { - return messages.WritingEvalConfig(path, err) - } - if err := os.Rename(tmpName, path); err != nil { - return messages.WritingEvalConfig(path, err) - } + if err := ReplaceFile(tmpName, path); err != nil { + return messages.WritingEvalConfig(path, err) } return nil } +// ReplaceFile moves a freshly written temporary file over a destination. +// +// Never by unlinking the destination first, which is the obvious shape and is +// wrong twice over. Windows refuses a rename while a reader holds the +// destination open, so remove-then-rename turns a collision into a window where +// the file does not exist -- and a config that momentarily does not exist reads +// as "no configuration yet", which callers answer by writing a fresh empty one. +// Contention is measured in microseconds, so it is waited out instead. +// +// The unlink was doing one thing worth keeping: os.Remove clears a read-only +// attribute and retries, so a file marked read-only (a Perforce or TFVC +// checkout, `attrib +R`, some archive extractions) could still be replaced. +// Windows reports a rename onto a read-only destination with the same errno as +// one a reader holds open, so the two cannot be told apart before the wait. +func ReplaceFile(from, to string) error { + err := renameOverContention(from, to) + if err == nil { + return nil + } + if !clearReadOnly(to) { + return err + } + return os.Rename(from, to) +} + // clearReadOnly drops a read-only attribute, reporting whether it had one to // drop. os.Chmod is what carries FILE_ATTRIBUTE_READONLY on Windows. func clearReadOnly(path string) bool { From 4a1b61a443655331e30a338dbfeb66422dde93f8 Mon Sep 17 00:00:00 2001 From: mohessie Date: Wed, 12 Aug 2026 05:36:38 +0300 Subject: [PATCH 195/320] Follow the cursor these listings were already being sent OpenAIEvalList and OpenAIEvalRunList modelled only `data`, so both listings answered with whatever fit in one service page. That is a silent wrong answer rather than a short one: resolveEvalRef decides "is this name ambiguous?" from these rows, so a duplicate on page two turns a refusal into a wrong choice and `run start` grades against a definition the caller did not mean. `run list` simply omitted runs. Both envelopes now carry has_more/last_id, read the same way OutputItemList already reads them -- only when present, so a service that sends neither still yields one page. The walk they share is one loop rather than a third copy of it. Separately, --eval is documented as "the name of the eval declared in the configuration", but only `run start` offered --path. The only other way those commands can find the configuration is a path `init` recorded in the azd environment, which a --project-endpoint caller does not have, so a configuration outside ./evals could be started and then not listed, shown or cancelled. Every command resolving a declared name now takes --path, and surface_test.go holds that invariant so it cannot drift. --- .../azure.ai.evaluations/extension.yaml | 2 +- .../internal/cmd/run_ops.go | 12 ++ .../internal/cmd/run_output.go | 27 +++- .../internal/cmd/surface_test.go | 18 +++ .../internal/pkg/eval_api/models.go | 8 ++ .../internal/pkg/eval_api/operations.go | 75 ++++++++-- .../internal/pkg/eval_api/pagination_test.go | 134 ++++++++++++++++++ .../azure.ai.evaluations/version.txt | 2 +- 8 files changed, 265 insertions(+), 13 deletions(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/pagination_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/extension.yaml b/cli/azd/extensions/azure.ai.evaluations/extension.yaml index 2dca34eea0b..a4ec607ee85 100644 --- a/cli/azd/extensions/azure.ai.evaluations/extension.yaml +++ b/cli/azd/extensions/azure.ai.evaluations/extension.yaml @@ -5,7 +5,7 @@ displayName: Foundry evaluations (Beta) description: Define and run Foundry evaluations from your terminal. (Beta) usage: azd ai eval [options] # NOTE: Make sure version.txt is in sync with this version. -version: 1.0.0-beta.12 +version: 1.0.0-beta.13 requiredAzdVersion: ">=1.27.1" language: go capabilities: diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go index c9d09387cdd..aae64d88bca 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go @@ -88,6 +88,9 @@ func newRunListCommand() *cobra.Command { }, } addEvalFlag(cmd, &groupName) + // Registered wherever a declared name is resolved, so a configuration + // outside ./evals can be addressed by every command, not just un start. + addEvalPathFlag(cmd, new(string)) cmd.Flags().IntVar(&limit, "limit", 0, "Return at most this many runs. Omit for the service default.") cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") @@ -184,6 +187,9 @@ func newRunShowCommand() *cobra.Command { "Block until the run reaches a terminal state before reporting.") addFailOnFlag(cmd, &failOn) addEvalFlag(cmd, &groupName) + // Registered wherever a declared name is resolved, so a configuration + // outside ./evals can be addressed by every command, not just un start. + addEvalPathFlag(cmd, new(string)) cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") return cmd } @@ -247,6 +253,9 @@ func newRunCancelCommand() *cobra.Command { }, } addEvalFlag(cmd, &groupName) + // Registered wherever a declared name is resolved, so a configuration + // outside ./evals can be addressed by every command, not just un start. + addEvalPathFlag(cmd, new(string)) cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") return cmd } @@ -298,6 +307,9 @@ func newRunDeleteCommand() *cobra.Command { }, } addEvalFlag(cmd, &groupName) + // Registered wherever a declared name is resolved, so a configuration + // outside ./evals can be addressed by every command, not just un start. + addEvalPathFlag(cmd, new(string)) cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") return cmd } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output.go index 73dca1a8653..919d9ec249b 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output.go @@ -104,6 +104,9 @@ func newRunOutputListCommand() *cobra.Command { cmd.Flags().BoolVar(&failedOnly, "failed-only", false, "Show only the rows that failed.") cmd.Flags().StringVar(&outFile, "output-file", "", "Write JSON results to this path.") addEvalFlag(cmd, &groupName) + // Registered wherever a declared name is resolved, so a configuration + // outside ./evals can be addressed by every command, not just un start. + addEvalPathFlag(cmd, new(string)) cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") return cmd } @@ -159,6 +162,9 @@ func newRunOutputShowCommand() *cobra.Command { cmd.Flags().StringVar(&runID, "run", "", "Run the item belongs to. Defaults to the most recent run.") addEvalFlag(cmd, &groupName) + // Registered wherever a declared name is resolved, so a configuration + // outside ./evals can be addressed by every command, not just un start. + addEvalPathFlag(cmd, new(string)) cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") return cmd } @@ -233,6 +239,9 @@ func newRunOutputExportCommand() *cobra.Command { fmt.Sprintf("Output format: %s, %s or %s.", formatCSV, formatJSON, formatJSONL)) cmd.Flags().StringVar(&outFile, "output-file", "", "Write to this path instead of stdout.") addEvalFlag(cmd, &groupName) + // Registered wherever a declared name is resolved, so a configuration + // outside ./evals can be addressed by every command, not just un start. + addEvalPathFlag(cmd, new(string)) cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") return cmd } @@ -255,7 +264,8 @@ func resolveEvalID( } if groupName != "" { - ref, err := ec.resolveEvalRef(cmd.Context(), ec.evalDir(cmd.Context(), ""), groupName) + ref, err := ec.resolveEvalRef( + cmd.Context(), ec.evalDir(cmd.Context(), evalPathFlag(cmd)), groupName) if err != nil { return "", err } @@ -286,6 +296,21 @@ func addEvalPathFlag(cmd *cobra.Command, target *string) { "Directory holding azure.eval.yaml. Defaults to the path `init` used, then ./evals.") } +// evalPathFlag reads --path from whichever command is resolving a declared +// name, so every one of them can be told where the configuration is. +// +// Read off the command rather than threaded through seven call sites. Without +// it only `run start` offered the flag, so a configuration outside ./evals +// could be run and then not listed, shown or cancelled -- the fallback that +// covers the difference is a path recorded in the azd environment, which a +// --project-endpoint caller does not have. +func evalPathFlag(cmd *cobra.Command) string { + if f := cmd.Flags().Lookup("path"); f != nil { + return f.Value.String() + } + return "" +} + // latestOrNamedRun returns the named run, or the most recent one for the eval. // // explicit says whether the caller named the run rather than leaving it to diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/surface_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/surface_test.go index 7e733390b59..f13b673e465 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/surface_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/surface_test.go @@ -190,6 +190,24 @@ func TestInitTakesNoProjectEndpoint(t *testing.T) { // Every command that does reach the service accepts it, because the shared // Foundry resolver is how a project is named without an azd environment. +// --eval takes "the name of the eval declared in the configuration", which +// means the command has to be able to find that configuration. The only other +// way it can is a path `init` recorded in the azd environment, which a +// --project-endpoint caller does not have -- so a configuration outside ./evals +// could be started and then not listed, shown or cancelled. +func TestEveryCommandTakingAnEvalNameCanBeToldWhereTheConfigIs(t *testing.T) { + var checked int + walk(t, NewRootCommand(), nil, func(path string, cmd *cobra.Command) { + if cmd.Flags().Lookup("eval") == nil { + return + } + checked++ + assert.NotNilf(t, cmd.Flags().Lookup("path"), + "%s resolves a declared eval name, so it must accept --path", path) + }) + assert.NotZero(t, checked, "no command took --eval, so this checked nothing") +} + func TestServiceCommandsTakeProjectEndpoint(t *testing.T) { groups := map[string]bool{ "dataset": true, "evaluator": true, "run": true, diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/models.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/models.go index a5f13b905c1..3f37a1f8868 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/models.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/models.go @@ -283,6 +283,11 @@ type OpenAIEval struct { // OpenAIEvalList is the response for listing OpenAI eval definitions. type OpenAIEvalList struct { Data []OpenAIEval `json:"data"` + // HasMore and LastID are the OpenAI list envelope's cursor, read the same + // way OutputItemList reads them: only when present, so a service that sends + // neither still yields one page. + HasMore bool `json:"has_more"` + LastID string `json:"last_id"` } // --------------------------------------------------------------------------- @@ -551,6 +556,9 @@ type EvalRunCriteriaResult struct { // OpenAIEvalRunList is the response for listing OpenAI eval runs. type OpenAIEvalRunList struct { Data []OpenAIEvalRun `json:"data"` + // HasMore and LastID are the OpenAI list envelope's cursor. + HasMore bool `json:"has_more"` + LastID string `json:"last_id"` } // OutputItemList is a page of a run's per-sample results. diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/operations.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/operations.go index 603c7d7c94e..33ba802c64d 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/operations.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/operations.go @@ -415,12 +415,59 @@ func (c *EvalClient) CreateOpenAIEval( // ListOpenAIEvals lists OpenAI eval definitions. func (c *EvalClient) ListOpenAIEvals(ctx context.Context, limit int) (*OpenAIEvalList, error) { - query := map[string]string{} - if limit > 0 { - query["limit"] = strconv.Itoa(limit) + all := &OpenAIEvalList{} + err := collectPages(limit, func(query map[string]string) (int, bool, string, error) { + page, err := doRequestTyped[OpenAIEvalList]( + c, ctx, http.MethodGet, pathOpenAIEvals, query, nil, "") + if err != nil { + return 0, false, "", err + } + all.Data = append(all.Data, page.Data...) + return len(page.Data), page.HasMore, page.LastID, nil + }) + if err != nil { + return nil, err } + return all, nil +} + +// collectPages walks an OpenAI-shaped listing until the service stops offering +// a cursor, or until limit rows have been gathered. +// +// A listing that stops at the first page is a silent wrong answer rather than a +// short one: "is this name ambiguous?" and "which run is newest?" are both +// decided from these rows, so a second page nobody asked for turns a refusal +// into a wrong choice. fetch reports how many rows it added and the cursor it +// was given, so the two listings share this loop instead of a third copy. +func collectPages( + limit int, + fetch func(query map[string]string) (added int, hasMore bool, lastID string, err error), +) error { + gathered := 0 + after := "" + for { + query := map[string]string{} + if limit > 0 { + query["limit"] = strconv.Itoa(limit - gathered) + } + if after != "" { + query["after"] = after + } - return doRequestTyped[OpenAIEvalList](c, ctx, http.MethodGet, pathOpenAIEvals, query, nil, "") + added, hasMore, lastID, err := fetch(query) + if err != nil { + return err + } + gathered += added + + if !hasMore || lastID == "" || added == 0 { + return nil + } + if limit > 0 && gathered >= limit { + return nil + } + after = lastID + } } // GetOpenAIEval gets an OpenAI eval definition. @@ -468,13 +515,21 @@ func (c *EvalClient) ListOpenAIEvalRuns( evalID string, limit int, ) (*OpenAIEvalRunList, error) { - query := map[string]string{} - if limit > 0 { - query["limit"] = strconv.Itoa(limit) - } - path := fmt.Sprintf("%s/%s/runs", pathOpenAIEvals, url.PathEscape(evalID)) - return doRequestTyped[OpenAIEvalRunList](c, ctx, http.MethodGet, path, query, nil, "") + all := &OpenAIEvalRunList{} + err := collectPages(limit, func(query map[string]string) (int, bool, string, error) { + page, err := doRequestTyped[OpenAIEvalRunList]( + c, ctx, http.MethodGet, path, query, nil, "") + if err != nil { + return 0, false, "", err + } + all.Data = append(all.Data, page.Data...) + return len(page.Data), page.HasMore, page.LastID, nil + }) + if err != nil { + return nil, err + } + return all, nil } // GetOpenAIEvalRun gets a run for an OpenAI eval definition. diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/pagination_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/pagination_test.go new file mode 100644 index 00000000000..90dee0fd46f --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/pagination_test.go @@ -0,0 +1,134 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package eval_api + +import ( + "context" + "fmt" + "net/http" + "net/http/httptest" + "testing" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func clientServing(t *testing.T, handler http.HandlerFunc) *EvalClient { + t.Helper() + srv := httptest.NewServer(handler) + t.Cleanup(srv.Close) + + pipeline := runtime.NewPipeline("test", "v1", runtime.PipelineOptions{}, + &policy.ClientOptions{Retry: policy.RetryOptions{MaxRetries: -1}}) + return NewEvalClientFromPipeline(srv.URL, pipeline) +} + +// A listing that stops at the first page is a silent wrong answer rather than a +// short one. resolveEvalRef decides "is this name ambiguous?" from these rows, +// so a duplicate sitting on page two turns a refusal into a wrong choice, and +// `run start` then grades against a definition the caller did not mean. +func TestListOpenAIEvalsFollowsTheCursor(t *testing.T) { + var afters []string + c := clientServing(t, func(w http.ResponseWriter, r *http.Request) { + after := r.URL.Query().Get("after") + afters = append(afters, after) + w.Header().Set("Content-Type", "application/json") + switch after { + case "": + fmt.Fprint(w, `{"data":[{"id":"eval_1","name":"dup"},{"id":"eval_2","name":"other"}],`+ + `"has_more":true,"last_id":"eval_2"}`) + default: + fmt.Fprint(w, `{"data":[{"id":"eval_3","name":"dup"}],"has_more":false}`) + } + }) + + list, err := c.ListOpenAIEvals(context.Background(), 0) + + require.NoError(t, err) + require.Len(t, list.Data, 3, "both pages have to be gathered") + assert.Equal(t, []string{"", "eval_2"}, afters, + "the second request has to carry the cursor the first returned") + + var named int + for _, e := range list.Data { + if e.Name == "dup" { + named++ + } + } + assert.Equal(t, 2, named, "the duplicate on page two is what makes the name ambiguous") +} + +// The cursor is read only when the service sends one. Without this a service +// that omits it would loop forever or truncate, depending on the guard. +func TestListOpenAIEvalsStopsWithoutACursor(t *testing.T) { + calls := 0 + c := clientServing(t, func(w http.ResponseWriter, r *http.Request) { + calls++ + w.Header().Set("Content-Type", "application/json") + fmt.Fprint(w, `{"data":[{"id":"eval_1","name":"only"}]}`) + }) + + list, err := c.ListOpenAIEvals(context.Background(), 0) + + require.NoError(t, err) + assert.Len(t, list.Data, 1) + assert.Equal(t, 1, calls, "no cursor means one page, not an endless walk") +} + +// has_more with no last_id is the other way a service can leave the walk +// without an anchor, and repeating the same request would never terminate. +func TestListOpenAIEvalsStopsWhenTheCursorIsEmpty(t *testing.T) { + calls := 0 + c := clientServing(t, func(w http.ResponseWriter, r *http.Request) { + calls++ + w.Header().Set("Content-Type", "application/json") + fmt.Fprint(w, `{"data":[{"id":"eval_1"}],"has_more":true,"last_id":""}`) + }) + + list, err := c.ListOpenAIEvals(context.Background(), 0) + + require.NoError(t, err) + assert.Len(t, list.Data, 1) + assert.Equal(t, 1, calls, "has_more without last_id has nowhere to go") +} + +// An explicit limit still bounds the walk, and asks each page for only what is +// left rather than the whole limit again. +func TestListOpenAIEvalRunsHonoursTheLimit(t *testing.T) { + var limits []string + c := clientServing(t, func(w http.ResponseWriter, r *http.Request) { + limits = append(limits, r.URL.Query().Get("limit")) + w.Header().Set("Content-Type", "application/json") + fmt.Fprint(w, `{"data":[{"id":"run_1"},{"id":"run_2"}],"has_more":true,"last_id":"run_2"}`) + }) + + list, err := c.ListOpenAIEvalRuns(context.Background(), "eval_1", 2) + + require.NoError(t, err) + assert.Len(t, list.Data, 2, "the limit stops the walk") + assert.Equal(t, []string{"2"}, limits, "one page satisfied it") +} + +// run list without a limit has to report every run, not the newest page of +// them, or a run a caller started is missing from the list that should show it. +func TestListOpenAIEvalRunsGathersEveryPage(t *testing.T) { + c := clientServing(t, func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + switch r.URL.Query().Get("after") { + case "": + fmt.Fprint(w, `{"data":[{"id":"run_1"},{"id":"run_2"}],"has_more":true,"last_id":"run_2"}`) + case "run_2": + fmt.Fprint(w, `{"data":[{"id":"run_3"}],"has_more":true,"last_id":"run_3"}`) + default: + fmt.Fprint(w, `{"data":[{"id":"run_4"}],"has_more":false}`) + } + }) + + list, err := c.ListOpenAIEvalRuns(context.Background(), "eval_1", 0) + + require.NoError(t, err) + assert.Len(t, list.Data, 4, "three pages, every run") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/version.txt b/cli/azd/extensions/azure.ai.evaluations/version.txt index e521dc4f817..dd5d5f2fb09 100644 --- a/cli/azd/extensions/azure.ai.evaluations/version.txt +++ b/cli/azd/extensions/azure.ai.evaluations/version.txt @@ -1 +1 @@ -1.0.0-beta.12 \ No newline at end of file +1.0.0-beta.13 \ No newline at end of file From c666084db625ea52212ff88173a3bf781c735e1b Mon Sep 17 00:00:00 2001 From: mohessie Date: Wed, 12 Aug 2026 06:06:39 +0300 Subject: [PATCH 196/320] Follow nextLink, and stop paying a dataset version to learn a config is wrong Four listings decoded nextLink and dropped it: datasets, dataset versions, evaluators and evaluator versions. Same class as the OpenAI-shaped cursor, different transport, and with a worse consequence than truncation here: the built-in evaluator listing is what supplies each evaluator's published contract, and a contract missing from page two makes the request builder skip the checks that contract enables. So `eval create` with no judge deployment reached the service and came back 400, when the extension already had what it needed to say so. The link arrives in a response body and this client sends an Authorization header, so a link is only followed when its scheme and host match the endpoint. A link pointing at the page it came from stops the walk, and the walk is bounded. The check that a missing judge exposes now runs before anything is published. It used to run while building the request, which happens after the datasets and evaluators are pushed: four failed attempts while testing left dataset versions 2.0 through 5.0 behind, and versions are immutable. Only what the published contract alone can settle moves earlier -- the data mapping needs the dataset's columns, which is a separate question and stays where it was. Also: OutputItem.Failed answered false for a row carrying no results at all, so a row that errored badly enough to produce no verdict was hidden from --failed-only, which is exactly where someone looks to find out what went wrong. Still unfixed: ListContainerBlobs ignores the Blob service's NextMarker. That listing walks a SAS URL at a different host, so it does not fit the check above and wants its own pass. --- .../azure.ai.evaluations/extension.yaml | 2 +- .../internal/cmd/build.go | 63 +++++++++- .../internal/cmd/eval_group.go | 7 ++ .../internal/cmd/preflight_test.go | 96 +++++++++++++++ .../internal/messages/messages.go | 10 ++ .../internal/pkg/dataset_api/list.go | 12 +- .../internal/pkg/dataset_api/pages.go | 91 ++++++++++++++ .../internal/pkg/dataset_api/pages_test.go | 105 ++++++++++++++++ .../internal/pkg/eval_api/evaluators.go | 16 ++- .../internal/pkg/eval_api/models.go | 10 +- .../internal/pkg/eval_api/output_item_test.go | 35 ++++++ .../internal/pkg/eval_api/pages.go | 112 ++++++++++++++++++ .../azure.ai.evaluations/version.txt | 2 +- 13 files changed, 552 insertions(+), 9 deletions(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/preflight_test.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/pages.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/pages_test.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/output_item_test.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/pages.go diff --git a/cli/azd/extensions/azure.ai.evaluations/extension.yaml b/cli/azd/extensions/azure.ai.evaluations/extension.yaml index a4ec607ee85..bcdb885ec37 100644 --- a/cli/azd/extensions/azure.ai.evaluations/extension.yaml +++ b/cli/azd/extensions/azure.ai.evaluations/extension.yaml @@ -5,7 +5,7 @@ displayName: Foundry evaluations (Beta) description: Define and run Foundry evaluations from your terminal. (Beta) usage: azd ai eval [options] # NOTE: Make sure version.txt is in sync with this version. -version: 1.0.0-beta.13 +version: 1.0.0-beta.14 requiredAzdVersion: ">=1.27.1" language: go capabilities: diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build.go index 1d30335a7b0..7a00df00c4e 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build.go @@ -258,12 +258,71 @@ func planCriterion( return plan, nil } +// checkEvaluatorRequirements refuses a declaration the evaluators cannot +// satisfy, before anything is published. +// +// The same checks happen while building the request, but that runs after the +// datasets and evaluators have been pushed -- so a missing judge deployment +// cost an immutable dataset version per attempt, and the version numbers climb +// whether or not the eval was ever created. Only what the published contract +// alone can settle is checked here: the data mapping needs the dataset's +// columns, which is a separate question. +func checkEvaluatorRequirements( + eval *project.Eval, + schemas map[string]*eval_api.EvaluatorSummary, +) error { + level := resolveLevel(eval) + for _, ref := range eval.Evaluators { + schema := schemas[ref.Evaluator] + if schema == nil { + // Nothing published to check against. The service still gets the + // last word, which is what happened before this existed. + continue + } + if !schema.SupportsLevel(level) { + return messages.EvaluatorLevelUnsupported( + ref.Evaluator, level, schema.SupportedEvaluationLevels) + } + + initSchema := schema.InitSchema() + if initSchema == nil { + continue + } + var missing []string + for _, name := range initSchema.Required { + if declaredInitParam(ref, name) { + continue + } + if name == "evaluation_level" && level != "" { + continue + } + missing = append(missing, name) + } + if len(missing) > 0 { + return messages.EvaluatorNeedsInitParams(ref.Evaluator, missing) + } + } + return nil +} + +// declaredInitParam reports whether the reference supplies a parameter under +// either spelling of the judge deployment. +func declaredInitParam(ref evalcore.EvaluatorRef, name string) bool { + if _, ok := ref.InitializationParameters[name]; ok { + return true + } + if alias, ok := judgeModelAliases[name]; ok { + _, declared := ref.InitializationParameters[alias] + return declared + } + return false +} + // judgeModelAliases maps the two spellings of the judge deployment onto each // other, so one declaration works whichever the evaluator publishes. var judgeModelAliases = map[string]string{ "deployment_name": "model", - "model": "deployment_name", -} + "model": "deployment_name"} // itemColumn reads the dataset column out of an `{{item.}}` binding. func itemColumn(binding string) (string, bool) { diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/eval_group.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/eval_group.go index 49fb8acd64d..385b2168994 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/eval_group.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/eval_group.go @@ -72,6 +72,13 @@ func newEvalCreateCommand() *cobra.Command { reconciler := &evalReconciler{ec: ec} out := cmd.OutOrStdout() + + // Before anything is pushed. Publishing is not free -- a dataset + // version is immutable and the number climbs on every attempt -- so + // a declaration the evaluators cannot satisfy is refused first. + if err := checkEvaluatorRequirements(eval, ec.evaluatorSchemas(ctx)); err != nil { + return err + } // Reported per artifact, because "publishes nothing when nothing // changed" is the contract a reader is checking here and a single // closing line cannot show it. Silent under -o json. diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/preflight_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/preflight_test.go new file mode 100644 index 00000000000..411a9c3f377 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/preflight_test.go @@ -0,0 +1,96 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "testing" + + "azureaieval/internal/pkg/eval_api" + "azureaieval/internal/pkg/evalcore" + "azureaieval/internal/project" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func relevanceSchema(required ...string) map[string]*eval_api.EvaluatorSummary { + return map[string]*eval_api.EvaluatorSummary{ + "builtin.relevance": { + Name: "builtin.relevance", + SupportedEvaluationLevels: []string{"turn"}, + Definition: &eval_api.EvaluatorContract{ + InitParameters: &eval_api.JSONSchema{Required: required}, + }, + }, + } +} + +func evalRequiring(params map[string]any) *project.Eval { + return &project.Eval{ + Name: "quality", + Dataset: "golden", + EvaluationLevel: "turn", + Evaluators: evalcore.EvaluatorList{{ + Evaluator: "builtin.relevance", + InitializationParameters: params, + }}, + } +} + +// The same requirement was checked while building the request, which runs after +// the datasets and evaluators have been pushed. A missing judge deployment +// therefore cost an immutable dataset version per attempt, and the version +// number climbs whether or not the eval is ever created. +func TestCreateRefusesAMissingJudgeBeforePublishing(t *testing.T) { + err := checkEvaluatorRequirements(evalRequiring(nil), relevanceSchema("deployment_name")) + + require.Error(t, err) + assert.Contains(t, err.Error(), "deployment_name") + assert.Contains(t, err.Error(), "builtin.relevance") +} + +// Either spelling of the judge deployment satisfies it, the same way the +// request builder binds whichever the evaluator publishes. +func TestEitherJudgeSpellingSatisfiesTheRequirement(t *testing.T) { + for _, declared := range []string{"deployment_name", "model"} { + err := checkEvaluatorRequirements( + evalRequiring(map[string]any{declared: "o4-mini"}), + relevanceSchema("deployment_name")) + + assert.NoErrorf(t, err, "%q is the same parameter under the other name", declared) + } +} + +// evaluation_level is supplied from the eval's own declaration rather than +// written under initialization_parameters, so requiring it must not refuse a +// configuration that sets the level. +func TestRequiredEvaluationLevelComesFromTheDeclaration(t *testing.T) { + err := checkEvaluatorRequirements( + evalRequiring(map[string]any{"deployment_name": "o4-mini"}), + relevanceSchema("deployment_name", "evaluation_level")) + + assert.NoError(t, err) +} + +// An evaluator the listing did not describe leaves the service with the last +// word, which is what happened before this check existed. +func TestAnUnknownEvaluatorIsLeftToTheService(t *testing.T) { + err := checkEvaluatorRequirements(evalRequiring(nil), + map[string]*eval_api.EvaluatorSummary{}) + + assert.NoError(t, err, "nothing published to check against") +} + +// A level the evaluator does not support is the other thing settled by the +// contract alone, so it is worth catching before a publish too. +func TestAnUnsupportedLevelIsRefusedBeforePublishing(t *testing.T) { + schemas := relevanceSchema("deployment_name") + schemas["builtin.relevance"].SupportedEvaluationLevels = []string{"conversation"} + + err := checkEvaluatorRequirements( + evalRequiring(map[string]any{"deployment_name": "o4-mini"}), schemas) + + require.Error(t, err) + assert.Contains(t, err.Error(), "turn") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go b/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go index 72ddbb0cce0..f40208d761d 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go @@ -1911,6 +1911,16 @@ func ConfigLockUnavailable(evalDir string, err error) error { "processes: %w", filepath.ToSlash(evalDir), err) } +// PageLinkLeftTheService reports a paging link pointing somewhere else. +// +// The link arrives in a response body and this client sends an Authorization +// header, so following one to another host would send the token there. +func PageLinkLeftTheService(expected, got string) error { + return fmt.Errorf( + "the service returned a paging link for %q while this client is "+ + "talking to %q, so it was not followed", got, expected) +} + // SampleSizeOutOfRange reports a row count the generation service would reject. func SampleSizeOutOfRange(min, max, got int) error { return fmt.Errorf("sample size must be between %d and %d, got %d", min, max, got) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/list.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/list.go index 19b548c3e05..28c4e6e459c 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/list.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/list.go @@ -21,7 +21,11 @@ type DatasetList struct { // ListDatasets returns the datasets registered on the project. func (c *DatasetClient) ListDatasets(ctx context.Context, apiVersion string) (*DatasetList, error) { - return doRequestTyped[DatasetList](c, ctx, http.MethodGet, pathDatasets, nil, nil, apiVersion) + first, err := doRequestTyped[DatasetList](c, ctx, http.MethodGet, pathDatasets, nil, nil, apiVersion) + if err != nil { + return nil, err + } + return c.walkDatasetPages(ctx, first) } // ListDatasetVersions returns every version of a single dataset. @@ -31,7 +35,11 @@ func (c *DatasetClient) ListDatasetVersions( apiVersion string, ) (*DatasetList, error) { path := fmt.Sprintf("%s/%s/versions", pathDatasets, url.PathEscape(name)) - return doRequestTyped[DatasetList](c, ctx, http.MethodGet, path, nil, nil, apiVersion) + first, err := doRequestTyped[DatasetList](c, ctx, http.MethodGet, path, nil, nil, apiVersion) + if err != nil { + return nil, err + } + return c.walkDatasetPages(ctx, first) } // DeleteDatasetVersion removes a single dataset version. diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/pages.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/pages.go new file mode 100644 index 00000000000..1d359e0a16e --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/pages.go @@ -0,0 +1,91 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package dataset_api + +import ( + "bytes" + "context" + "encoding/json" + "io" + "log" + "net/http" + "net/url" + "strings" + + "azureaieval/internal/messages" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime" +) + +// maxPages bounds a walk the service controls, so a nextLink that points at +// itself cannot hold the command open indefinitely. +const maxPages = 100 + +// followNextLink fetches one service-supplied page URL. +// +// The URL arrives in a response body and this client sends an Authorization +// header, so a link to another host would send the token there. Checked +// against the endpoint before it is used. +func (c *DatasetClient) followNextLink(ctx context.Context, nextLink string) ([]byte, error) { + next, err := url.Parse(nextLink) + if err != nil { + return nil, messages.InvalidEndpointURL(err) + } + base, err := url.Parse(c.endpoint) + if err != nil { + return nil, messages.InvalidEndpointURL(err) + } + if !strings.EqualFold(base.Host, next.Host) || !strings.EqualFold(base.Scheme, next.Scheme) { + return nil, messages.PageLinkLeftTheService(base.Host, next.Host) + } + + req, err := runtime.NewRequest(ctx, http.MethodGet, next.String()) + if err != nil { + return nil, messages.CreatingRequest(err) + } + log.Printf("[dataset_api] GET %s", next.Redacted()) + + resp, err := c.pipeline.Do(req) + if err != nil { + return nil, messages.RequestFailed(err) + } + defer resp.Body.Close() + + respBody, err := io.ReadAll(resp.Body) + if err != nil { + return nil, messages.ReadingResponseBody(err) + } + if !runtime.HasStatusCode(resp, http.StatusOK) { + resp.Body = io.NopCloser(bytes.NewReader(respBody)) + return nil, messages.ServiceRefused(resp.StatusCode, runtime.NewResponseError(resp)) + } + return respBody, nil +} + +// walkDatasetPages gathers every page of a dataset listing. +// +// The listing answered with one page and a nextLink, and the link was decoded +// and dropped. UploadVersion picks the next version from this listing, so a +// version on page two meant reusing one that already exists. +func (c *DatasetClient) walkDatasetPages(ctx context.Context, first *DatasetList) (*DatasetList, error) { + link := first.NextLink + for pages := 0; link != "" && pages < maxPages; pages++ { + body, err := c.followNextLink(ctx, link) + if err != nil { + return nil, err + } + var page DatasetList + if len(body) > 0 { + if err := json.Unmarshal(body, &page); err != nil { + return nil, messages.ParsingResponse(err) + } + } + first.Value = append(first.Value, page.Value...) + if page.NextLink == link { + break + } + link = page.NextLink + } + return first, nil +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/pages_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/pages_test.go new file mode 100644 index 00000000000..469de2d7efe --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/pages_test.go @@ -0,0 +1,105 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package dataset_api + +import ( + "context" + "fmt" + "net/http" + "net/http/httptest" + "testing" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func datasetClientServing(t *testing.T, handler func(http.ResponseWriter, *http.Request, string)) *DatasetClient { + t.Helper() + var base string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + handler(w, r, base) + })) + t.Cleanup(srv.Close) + base = srv.URL + + pipeline := runtime.NewPipeline("test", "v1", runtime.PipelineOptions{}, + &policy.ClientOptions{Retry: policy.RetryOptions{MaxRetries: -1}}) + return NewDatasetClientFromPipeline(srv.URL, pipeline) +} + +// UploadVersion picks the next version from this listing, so a version sitting +// on page two meant reusing one that already exists. +func TestListDatasetVersionsFollowsNextLink(t *testing.T) { + c := datasetClientServing(t, func(w http.ResponseWriter, r *http.Request, base string) { + w.Header().Set("Content-Type", "application/json") + if r.URL.Query().Get("page") == "2" { + fmt.Fprint(w, `{"value":[{"name":"golden","version":"3.0"}]}`) + return + } + fmt.Fprintf(w, `{"value":[{"name":"golden","version":"1.0"},{"name":"golden","version":"2.0"}],`+ + `"nextLink":"%s/page?page=2"}`, base) + }) + + list, err := c.ListDatasetVersions(context.Background(), "golden", "v1") + + require.NoError(t, err) + require.Len(t, list.Value, 3, "both pages have to be gathered") + assert.Equal(t, "3.0", list.Value[2].Version, "the newest version was on page two") +} + +// The link arrives in a response body and this client sends an Authorization +// header, so following one to another host would send the token there. +func TestNextLinkToAnotherHostIsRefused(t *testing.T) { + elsewhere := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + t.Errorf("the client followed a link off its own host: %s", r.URL) + })) + t.Cleanup(elsewhere.Close) + + c := datasetClientServing(t, func(w http.ResponseWriter, r *http.Request, base string) { + w.Header().Set("Content-Type", "application/json") + fmt.Fprintf(w, `{"value":[{"name":"golden","version":"1.0"}],"nextLink":"%s/steal"}`, + elsewhere.URL) + }) + + _, err := c.ListDatasets(context.Background(), "v1") + + require.Error(t, err) + assert.Contains(t, err.Error(), "not followed") +} + +// A link pointing at the page it came from is the one shape that would +// otherwise spin until the page bound for no benefit. +func TestSelfReferencingNextLinkStops(t *testing.T) { + calls := 0 + c := datasetClientServing(t, func(w http.ResponseWriter, r *http.Request, base string) { + calls++ + w.Header().Set("Content-Type", "application/json") + fmt.Fprintf(w, `{"value":[{"name":"golden","version":"1.0"}],"nextLink":"%s/same"}`, base) + }) + + list, err := c.ListDatasets(context.Background(), "v1") + + require.NoError(t, err) + assert.Equal(t, 2, calls, "the first request, then the link once") + assert.Len(t, list.Value, 2) +} + +// A listing without a link is one page, which is what this did before it could +// see the link at all. +func TestListingWithoutANextLinkIsOnePage(t *testing.T) { + calls := 0 + c := datasetClientServing(t, func(w http.ResponseWriter, r *http.Request, base string) { + calls++ + w.Header().Set("Content-Type", "application/json") + fmt.Fprint(w, `{"value":[{"name":"golden","version":"1.0"}]}`) + }) + + list, err := c.ListDatasets(context.Background(), "v1") + + require.NoError(t, err) + assert.Equal(t, 1, calls) + assert.Len(t, list.Value, 1) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/evaluators.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/evaluators.go index 006189c4d89..b30227e26ae 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/evaluators.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/evaluators.go @@ -146,9 +146,15 @@ func (c *EvalClient) ListEvaluators( if evaluatorType != "" { query = map[string]string{"type": evaluatorType} } - return doRequestTyped[EvaluatorListResponse]( + first, err := doRequestTyped[EvaluatorListResponse]( c, ctx, http.MethodGet, pathEvaluators, query, nil, apiVersion, ) + if err != nil { + return nil, err + } + return walkNextLinks(ctx, c, first, + func(l *EvaluatorListResponse) string { return l.NextLink }, + func(into, page *EvaluatorListResponse) { into.Value = append(into.Value, page.Value...) }) } // ListEvaluatorVersions returns every version of one evaluator. @@ -158,9 +164,15 @@ func (c *EvalClient) ListEvaluatorVersions( apiVersion string, ) (*EvaluatorListResponse, error) { path := pathEvaluators + "/" + url.PathEscape(name) + "/versions" - return doRequestTyped[EvaluatorListResponse]( + first, err := doRequestTyped[EvaluatorListResponse]( c, ctx, http.MethodGet, path, nil, nil, apiVersion, ) + if err != nil { + return nil, err + } + return walkNextLinks(ctx, c, first, + func(l *EvaluatorListResponse) string { return l.NextLink }, + func(into, page *EvaluatorListResponse) { into.Value = append(into.Value, page.Value...) }) } // LatestEvaluatorVersionNumber reports the newest registered version as an diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/models.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/models.go index 3f37a1f8868..e728c03c453 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/models.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/models.go @@ -593,8 +593,16 @@ type OutputResult struct { Reason string `json:"reason,omitempty"` } -// Failed reports whether any evaluator failed this row. +// Failed reports whether this row is one to look at: any evaluator failed it, +// or it produced no verdict at all. +// +// A row that errored badly enough to carry no results used to answer false, so +// --failed-only hid it -- and that filter is exactly where someone looks to +// find out what went wrong. func (o OutputItem) Failed() bool { + if len(o.Results) == 0 { + return true + } for _, r := range o.Results { if !r.Passed { return true diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/output_item_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/output_item_test.go new file mode 100644 index 00000000000..a31b0a685b9 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/output_item_test.go @@ -0,0 +1,35 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package eval_api + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +// --failed-only is where someone looks to find out what went wrong, so a row +// that errored badly enough to carry no verdict at all has to appear there. It +// used to answer false and be hidden. +func TestARowWithNoVerdictCountsAsFailed(t *testing.T) { + assert.True(t, OutputItem{ID: "item_1", Status: "errored"}.Failed(), + "nothing graded this row, so nothing passed it") + assert.True(t, OutputItem{ID: "item_2", Results: []OutputResult{}}.Failed(), + "an empty result set is the same absence") +} + +// The ordinary cases have to keep answering as they did. +func TestFailedReadsEveryVerdict(t *testing.T) { + passing := OutputItem{Results: []OutputResult{ + {Name: "relevance", Passed: true}, + {Name: "coherence", Passed: true}, + }} + assert.False(t, passing.Failed(), "every evaluator passed it") + + mixed := OutputItem{Results: []OutputResult{ + {Name: "relevance", Passed: true}, + {Name: "coherence", Passed: false}, + }} + assert.True(t, mixed.Failed(), "one failing evaluator is enough") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/pages.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/pages.go new file mode 100644 index 00000000000..0ee332e0bc0 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/pages.go @@ -0,0 +1,112 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package eval_api + +import ( + "bytes" + "context" + "encoding/json" + "io" + "log" + "net/http" + "net/url" + "strings" + + "azureaieval/internal/messages" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime" +) + +// maxPages bounds a walk the service controls. +// +// A nextLink that points at itself, or a service that keeps offering one, +// would otherwise spin forever holding the command open. The listings this +// walks are catalogues, not run output, so the bound is generous enough that +// reaching it means something is wrong rather than that a project is large. +const maxPages = 100 + +// followNextLink fetches one service-supplied page URL. +// +// The URL arrives in a response body, so it is checked against the endpoint +// before it is used: this client sends an Authorization header, and following +// a body-supplied link to another host would send the token there. +func (c *EvalClient) followNextLink(ctx context.Context, nextLink string) ([]byte, error) { + next, err := url.Parse(nextLink) + if err != nil { + return nil, messages.InvalidEndpointURL(err) + } + base, err := url.Parse(c.endpoint) + if err != nil { + return nil, messages.InvalidEndpointURL(err) + } + if !sameService(base, next) { + return nil, messages.PageLinkLeftTheService(base.Host, next.Host) + } + + req, err := runtime.NewRequest(ctx, http.MethodGet, next.String()) + if err != nil { + return nil, messages.CreatingRequest(err) + } + log.Printf("[eval_api] GET %s", next.Redacted()) + + resp, err := c.pipeline.Do(req) + if err != nil { + return nil, messages.RequestFailed(err) + } + defer resp.Body.Close() + + respBody, err := io.ReadAll(resp.Body) + if err != nil { + return nil, messages.ReadingResponseBody(err) + } + if !runtime.HasStatusCode(resp, http.StatusOK) { + resp.Body = io.NopCloser(bytes.NewReader(respBody)) + return nil, messages.ServiceRefused(resp.StatusCode, runtime.NewResponseError(resp)) + } + return respBody, nil +} + +// sameService reports whether a page link stays on the host the client was +// pointed at. Scheme is compared too, so a link cannot downgrade to http. +func sameService(base, next *url.URL) bool { + return strings.EqualFold(base.Host, next.Host) && strings.EqualFold(base.Scheme, next.Scheme) +} + +// walkNextLinks gathers every page of an ARM-shaped listing. +// +// These listings answer with one page and a nextLink, and the link was decoded +// and dropped. That is a silent wrong answer rather than a short one: the +// evaluator listings settle which version is latest, and a version sitting on +// page two makes the answer an older one. +func walkNextLinks[T any]( + ctx context.Context, + c *EvalClient, + first *T, + nextLinkOf func(*T) string, + merge func(into, page *T), +) (*T, error) { + link := nextLinkOf(first) + for pages := 0; link != "" && pages < maxPages; pages++ { + body, err := c.followNextLink(ctx, link) + if err != nil { + return nil, err + } + var page T + if len(body) > 0 { + if err := json.Unmarshal(body, &page); err != nil { + return nil, messages.ParsingResponse(err) + } + } + merge(first, &page) + + nextLink := nextLinkOf(&page) + if nextLink == link { + // A link that points at the page it came from is the one shape that + // would otherwise spin until maxPages for no benefit. + break + } + link = nextLink + } + return first, nil +} diff --git a/cli/azd/extensions/azure.ai.evaluations/version.txt b/cli/azd/extensions/azure.ai.evaluations/version.txt index dd5d5f2fb09..fc672411e33 100644 --- a/cli/azd/extensions/azure.ai.evaluations/version.txt +++ b/cli/azd/extensions/azure.ai.evaluations/version.txt @@ -1 +1 @@ -1.0.0-beta.13 \ No newline at end of file +1.0.0-beta.14 \ No newline at end of file From 5c68347579acf2371a0aba0bfcc6500a1411f643 Mon Sep 17 00:00:00 2001 From: mohessie Date: Wed, 12 Aug 2026 06:26:49 +0300 Subject: [PATCH 197/320] Walk the container listing's marker, the last of the truncated pages The Blob service answers one page and a NextMarker. ListContainerBlobs sent no marker and parseBlobNames never read one, so a container larger than a page was seen only as far as the first one. DownloadDatasetContent falls back to listing the container and taking the first .jsonl by name, so where the page happened to end decided whether a download found nothing or found the wrong rows. Unlike the nextLink walk, only the marker value comes from the service -- the URL is the one built here from the SAS URI -- so this carries none of the risk that following a body-supplied link would, and needs no same-host rule. A marker that repeats itself stops the walk, and the walk is bounded. parseBlobNames stays as the name-only reader over the new page parser, so its existing test still describes what it does. --- .../azure.ai.evaluations/extension.yaml | 2 +- .../pkg/dataset_api/blob_pages_test.go | 100 ++++++++++++++++++ .../internal/pkg/dataset_api/operations.go | 73 +++++++++---- .../azure.ai.evaluations/version.txt | 2 +- 4 files changed, 154 insertions(+), 23 deletions(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/blob_pages_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/extension.yaml b/cli/azd/extensions/azure.ai.evaluations/extension.yaml index bcdb885ec37..f30c77f740c 100644 --- a/cli/azd/extensions/azure.ai.evaluations/extension.yaml +++ b/cli/azd/extensions/azure.ai.evaluations/extension.yaml @@ -5,7 +5,7 @@ displayName: Foundry evaluations (Beta) description: Define and run Foundry evaluations from your terminal. (Beta) usage: azd ai eval [options] # NOTE: Make sure version.txt is in sync with this version. -version: 1.0.0-beta.14 +version: 1.0.0-beta.15 requiredAzdVersion: ">=1.27.1" language: go capabilities: diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/blob_pages_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/blob_pages_test.go new file mode 100644 index 00000000000..b6331091cd1 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/blob_pages_test.go @@ -0,0 +1,100 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package dataset_api + +import ( + "context" + "fmt" + "net/http" + "net/http/httptest" + "testing" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func blobPage(marker string, names ...string) string { + body := `` + for _, n := range names { + body += fmt.Sprintf(`%s`, n) + } + body += `` + marker + `` + return body +} + +// DownloadDatasetContent falls back to listing the container and taking the +// first .jsonl by name, so a container answered one page at a time could report +// no file, or a different one, depending on where the page happened to end. +func TestListContainerBlobsFollowsTheMarker(t *testing.T) { + var markers []string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + m := r.URL.Query().Get("marker") + markers = append(markers, m) + w.Header().Set("Content-Type", "application/xml") + switch m { + case "": + fmt.Fprint(w, blobPage("m1", "a.jsonl", "b.jsonl")) + case "m1": + fmt.Fprint(w, blobPage("m2", "c.jsonl")) + default: + fmt.Fprint(w, blobPage("", "d.jsonl")) + } + })) + t.Cleanup(srv.Close) + + pipeline := runtime.NewPipeline("test", "v1", runtime.PipelineOptions{}, + &policy.ClientOptions{Retry: policy.RetryOptions{MaxRetries: -1}}) + c := NewDatasetClientFromPipeline(srv.URL, pipeline) + + names, err := c.ListContainerBlobs(context.Background(), srv.URL+"/container?sig=redacted") + + require.NoError(t, err) + assert.Equal(t, []string{"a.jsonl", "b.jsonl", "c.jsonl", "d.jsonl"}, names) + assert.Equal(t, []string{"", "m1", "m2"}, markers, + "each request has to carry the marker the previous page returned") +} + +// An empty NextMarker is the last page, which is what this did before it could +// see the marker at all. +func TestListContainerBlobsStopsWithoutAMarker(t *testing.T) { + calls := 0 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + calls++ + w.Header().Set("Content-Type", "application/xml") + fmt.Fprint(w, blobPage("", "only.jsonl")) + })) + t.Cleanup(srv.Close) + + pipeline := runtime.NewPipeline("test", "v1", runtime.PipelineOptions{}, + &policy.ClientOptions{Retry: policy.RetryOptions{MaxRetries: -1}}) + c := NewDatasetClientFromPipeline(srv.URL, pipeline) + + names, err := c.ListContainerBlobs(context.Background(), srv.URL+"/container") + + require.NoError(t, err) + assert.Equal(t, 1, calls) + assert.Equal(t, []string{"only.jsonl"}, names) +} + +// A marker that repeats itself would otherwise spin until the page bound. +func TestListContainerBlobsStopsOnARepeatedMarker(t *testing.T) { + calls := 0 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + calls++ + w.Header().Set("Content-Type", "application/xml") + fmt.Fprint(w, blobPage("stuck", "same.jsonl")) + })) + t.Cleanup(srv.Close) + + pipeline := runtime.NewPipeline("test", "v1", runtime.PipelineOptions{}, + &policy.ClientOptions{Retry: policy.RetryOptions{MaxRetries: -1}}) + c := NewDatasetClientFromPipeline(srv.URL, pipeline) + + _, err := c.ListContainerBlobs(context.Background(), srv.URL+"/container") + + require.NoError(t, err) + assert.Equal(t, 2, calls, "the first request, then the marker once") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/operations.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/operations.go index f87869a499f..fc1260b416d 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/operations.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/operations.go @@ -448,38 +448,61 @@ func (c *DatasetClient) ListContainerBlobs(ctx context.Context, containerSASUri return nil, messages.InvalidContainerURI(err) } - q := u.Query() - q.Set("restype", "container") // cspell:ignore restype — Azure Storage API query parameter - q.Set("comp", "list") - u.RawQuery = q.Encode() + // The Blob service answers one page and a NextMarker. Only the marker value + // comes from the service -- the URL is the one built here -- so this walk + // carries none of the risk that following a body-supplied link would. + var names []string + marker := "" + for pages := 0; pages < maxPages; pages++ { + page := *u + q := page.Query() + q.Set("restype", "container") // cspell:ignore restype — Azure Storage API query parameter + q.Set("comp", "list") + if marker != "" { + q.Set("marker", marker) + } + page.RawQuery = q.Encode() - log.Printf("[dataset_api] listing blobs: %s", u.Redacted()) + log.Printf("[dataset_api] listing blobs: %s", page.Redacted()) - req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil) - if err != nil { - return nil, messages.CreatingListRequest(err) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, page.String(), nil) + if err != nil { + return nil, messages.CreatingListRequest(err) + } + + pageNames, next, err := c.readBlobPage(req) + if err != nil { + return nil, err + } + names = append(names, pageNames...) + if next == "" || next == marker { + break + } + marker = next } - httpClient := blobHTTPClient - resp, err := httpClient.Do(req) + log.Printf("[dataset_api] found %d blobs in container", len(names)) + return names, nil +} + +// readBlobPage performs one container listing request. +func (c *DatasetClient) readBlobPage(req *http.Request) ([]string, string, error) { + resp, err := blobHTTPClient.Do(req) if err != nil { - return nil, messages.ListingContainerBlobs(err) + return nil, "", messages.ListingContainerBlobs(err) } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { - return nil, messages.ContainerListStatus(resp.StatusCode) + return nil, "", messages.ContainerListStatus(resp.StatusCode) } body, err := io.ReadAll(resp.Body) if err != nil { - return nil, messages.ReadingListResponse(err) + return nil, "", messages.ReadingListResponse(err) } - - // Parse XML blob listing to extract blob names. - names := parseBlobNames(string(body)) - log.Printf("[dataset_api] found %d blobs in container", len(names)) - return names, nil + names, next := parseBlobPage(string(body)) + return names, next, nil } // DownloadBlob downloads a single blob from a container using the container SAS URI @@ -521,6 +544,13 @@ func (c *DatasetClient) DownloadBlob(ctx context.Context, containerSASUri, blobN // parseBlobNames extracts blob names from the Azure Blob Storage XML list response // using proper XML parsing against the EnumerationResults schema. func parseBlobNames(xmlBody string) []string { + names, _ := parseBlobPage(xmlBody) + return names +} + +// parseBlobPage extracts one page of blob names and the marker that continues +// the listing. An empty marker means this was the last page. +func parseBlobPage(xmlBody string) ([]string, string) { type blob struct { Name string `xml:"Name"` } @@ -528,12 +558,13 @@ func parseBlobNames(xmlBody string) []string { Blob []blob `xml:"Blob"` } type enumerationResults struct { - Blobs blobs `xml:"Blobs"` + Blobs blobs `xml:"Blobs"` + NextMarker string `xml:"NextMarker"` } var result enumerationResults if err := xml.Unmarshal([]byte(xmlBody), &result); err != nil { - return nil + return nil, "" } names := make([]string, 0, len(result.Blobs.Blob)) @@ -542,7 +573,7 @@ func parseBlobNames(xmlBody string) []string { names = append(names, b.Name) } } - return names + return names, result.NextMarker } // doRequest performs an HTTP request against the dataset API and returns the raw response body. diff --git a/cli/azd/extensions/azure.ai.evaluations/version.txt b/cli/azd/extensions/azure.ai.evaluations/version.txt index fc672411e33..0775a70db1d 100644 --- a/cli/azd/extensions/azure.ai.evaluations/version.txt +++ b/cli/azd/extensions/azure.ai.evaluations/version.txt @@ -1 +1 @@ -1.0.0-beta.14 \ No newline at end of file +1.0.0-beta.15 \ No newline at end of file From dc02ad114471a80d4b7b7e6914707abd35f2bcec Mon Sep 17 00:00:00 2001 From: mohessie Date: Wed, 12 Aug 2026 15:55:30 +0300 Subject: [PATCH 198/320] Tell an absent verdict apart from a failing one OutputResult.Passed was a plain bool, so a result the service sent without a verdict -- an evaluator that errored on that row -- read as a definite "fail". That names the evaluator as the thing that judged badly rather than the thing that did not run, and it is the row someone is most likely to be looking at. Passed is now a pointer, with DidPass and Judged reading it. An unjudged result renders as "no verdict" rather than "fail", is not listed among the evaluators that failed a row, and still counts the row into --failed-only, because nothing passed it. I had skipped this twice on the grounds that it meant touching fourteen call sites. Four touch this field; the other eleven are the integer Passed on EvalRunResultCounts, which is a different thing with the same name. The estimate that justified deferring it was wrong. --- .../azure.ai.evaluations/extension.yaml | 2 +- .../internal/cmd/run_output.go | 15 +++++++---- .../internal/cmd/run_render_test.go | 26 +++++++++++-------- .../internal/pkg/eval_api/models.go | 25 +++++++++++++++--- .../internal/pkg/eval_api/output_item_test.go | 15 ++++++++--- .../azure.ai.evaluations/version.txt | 2 +- 6 files changed, 59 insertions(+), 26 deletions(-) diff --git a/cli/azd/extensions/azure.ai.evaluations/extension.yaml b/cli/azd/extensions/azure.ai.evaluations/extension.yaml index f30c77f740c..d24ca5a7abe 100644 --- a/cli/azd/extensions/azure.ai.evaluations/extension.yaml +++ b/cli/azd/extensions/azure.ai.evaluations/extension.yaml @@ -5,7 +5,7 @@ displayName: Foundry evaluations (Beta) description: Define and run Foundry evaluations from your terminal. (Beta) usage: azd ai eval [options] # NOTE: Make sure version.txt is in sync with this version. -version: 1.0.0-beta.15 +version: 1.0.0-beta.16 requiredAzdVersion: ">=1.27.1" language: go capabilities: diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output.go index 919d9ec249b..4d3c507fef6 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output.go @@ -400,7 +400,7 @@ func renderOutputItem(w io.Writer, item *eval_api.OutputItem) error { if len(results) == 1 && (results[0].Metric == "" || results[0].Metric == name) { r := results[0] fmt.Fprint(w, messages.OutputItemVerdict( - name, formatScore(r.Score), verdictWord(r.Passed))) + name, formatScore(r.Score), verdictWord(r))) if r.Reason != "" { fmt.Fprint(w, messages.OutputItemReason(r.Reason)) } @@ -414,7 +414,7 @@ func renderOutputItem(w io.Writer, item *eval_api.OutputItem) error { label = r.Name } fmt.Fprint(w, messages.OutputItemMetric( - label, formatScore(r.Score), verdictWord(r.Passed))) + label, formatScore(r.Score), verdictWord(r))) if r.Reason != "" { fmt.Fprint(w, messages.OutputItemReason(r.Reason)) } @@ -424,8 +424,13 @@ func renderOutputItem(w io.Writer, item *eval_api.OutputItem) error { } // verdictWord spells a boolean the way the rest of the output does. -func verdictWord(passed bool) string { - if passed { +func verdictWord(r eval_api.OutputResult) string { + if !r.Judged() { + // The evaluator returned no verdict, which is not the same as returning + // a failing one -- it says nothing about the sample. + return "no verdict" + } + if r.DidPass() { return "pass" } return "fail" @@ -485,7 +490,7 @@ func renderResults( var failed []string reason := "" for _, r := range it.Results { - if r.Passed { + if r.DidPass() { continue } failed = append(failed, r.Name) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_render_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_render_test.go index 6dcf14c3c9d..01307170410 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_render_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_render_test.go @@ -14,21 +14,25 @@ import ( "github.com/stretchr/testify/require" ) +// verdict is the recorded answer. A result whose Passed is nil was never +// judged, which is a different thing from one that failed. +func verdict(b bool) *bool { return &b } + // scoredRun is a run the way the service returns one, with rows attached. func scoredRows() []eval_api.OutputItem { return []eval_api.OutputItem{ { ID: "oi_1", Results: []eval_api.OutputResult{ - {Name: "relevance", Passed: true, Score: 5}, - {Name: "coherence", Passed: true, Score: 4}, + {Name: "relevance", Passed: verdict(true), Score: 5}, + {Name: "coherence", Passed: verdict(true), Score: 4}, }, }, { ID: "oi_2", Results: []eval_api.OutputResult{ - {Name: "relevance", Passed: false, Score: 1, Reason: "Answered a different question."}, - {Name: "coherence", Passed: false, Score: 2, Reason: "Rambled."}, + {Name: "relevance", Passed: verdict(false), Score: 1, Reason: "Answered a different question."}, + {Name: "coherence", Passed: verdict(false), Score: 2, Reason: "Rambled."}, }, }, } @@ -80,7 +84,7 @@ func TestCriteriaMeans(t *testing.T) { // number no evaluator produced. func TestCriteriaMeansIgnoresUnscoredRows(t *testing.T) { rows := []eval_api.OutputItem{ - {Results: []eval_api.OutputResult{{Name: "relevance", Score: 4, Passed: true}}}, + {Results: []eval_api.OutputResult{{Name: "relevance", Score: 4, Passed: verdict(true)}}}, {Results: []eval_api.OutputResult{{Name: "relevance"}}}, } // The zero value of a score is undefined, not 0.0. @@ -155,7 +159,7 @@ func TestRenderOutputItemIsNotJSON(t *testing.T) { Results: []eval_api.OutputResult{{ Name: "builtin.task_adherence", Score: 0.35, - Passed: false, + Passed: verdict(false), Reason: "Task abandoned after the first clarifying question.", }}, })) @@ -181,7 +185,7 @@ func TestRenderOutputItemKeepsTheWholeReason(t *testing.T) { require.NoError(t, renderOutputItem(&out, &eval_api.OutputItem{ ID: "oi_1", Status: "fail", - Results: []eval_api.OutputResult{{Name: "relevance", Passed: false, Reason: reason}}, + Results: []eval_api.OutputResult{{Name: "relevance", Passed: verdict(false), Reason: reason}}, })) assert.Contains(t, out.String(), reason) @@ -195,9 +199,9 @@ func TestRenderOutputItemGroupsARubricsDimensions(t *testing.T) { ID: "oi_1", Status: "fail", Results: []eval_api.OutputResult{ - {Name: "support-agent-quality", Metric: "resolves_issue", Score: 1, Passed: false}, - {Name: "support-agent-quality", Metric: "cites_policy", Score: 5, Passed: true}, - {Name: "builtin.task_adherence", Score: 0.35, Passed: false}, + {Name: "support-agent-quality", Metric: "resolves_issue", Score: 1, Passed: verdict(false)}, + {Name: "support-agent-quality", Metric: "cites_policy", Score: 5, Passed: verdict(true)}, + {Name: "builtin.task_adherence", Score: 0.35, Passed: verdict(false)}, }, })) @@ -218,7 +222,7 @@ func TestRenderOutputItemDoesNotNestASelfNamedMetric(t *testing.T) { ID: "oi_1", Status: "completed", Results: []eval_api.OutputResult{ - {Name: "task_adherence", Metric: "task_adherence", Score: 1, Passed: true}, + {Name: "task_adherence", Metric: "task_adherence", Score: 1, Passed: verdict(true)}, }, })) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/models.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/models.go index e728c03c453..9cc820c3bef 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/models.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/models.go @@ -587,30 +587,47 @@ type OutputResult struct { Metric string `json:"metric,omitempty"` Score LenientFloat `json:"score"` Label string `json:"label,omitempty"` - Passed bool `json:"passed"` + // Passed is a pointer because an absent verdict and a failing one are + // different claims. As a plain bool a result the service sent without one -- + // an evaluator that errored on this row -- read as a definite "fail", which + // names the evaluator as the thing that judged badly rather than the thing + // that did not run. + Passed *bool `json:"passed"` // Reason is the judge's explanation, which is the part a failing row is // actually looked at for. Reason string `json:"reason,omitempty"` } // Failed reports whether this row is one to look at: any evaluator failed it, -// or it produced no verdict at all. +// did not judge it, or it produced no verdict at all. // // A row that errored badly enough to carry no results used to answer false, so // --failed-only hid it -- and that filter is exactly where someone looks to -// find out what went wrong. +// find out what went wrong. A result carrying no verdict is the same absence +// one level down. func (o OutputItem) Failed() bool { if len(o.Results) == 0 { return true } for _, r := range o.Results { - if !r.Passed { + if !r.DidPass() { return true } } return false } +// DidPass reports whether this result is a recorded pass. An absent verdict is +// not one. +func (r OutputResult) DidPass() bool { + return r.Passed != nil && *r.Passed +} + +// Judged reports whether the evaluator returned a verdict at all. +func (r OutputResult) Judged() bool { + return r.Passed != nil +} + // Input renders the row's own columns for display, leaving out the // service-injected `sample.*` bindings and the plumbing ids, which are not what // the dataset author wrote. diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/output_item_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/output_item_test.go index a31b0a685b9..020148ed46b 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/output_item_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/output_item_test.go @@ -9,6 +9,10 @@ import ( "github.com/stretchr/testify/assert" ) +// verdict is the recorded answer. A result whose Passed is nil was never +// judged, which these tests need to say apart from a failing one. +func verdict(b bool) *bool { return &b } + // --failed-only is where someone looks to find out what went wrong, so a row // that errored badly enough to carry no verdict at all has to appear there. It // used to answer false and be hidden. @@ -17,19 +21,22 @@ func TestARowWithNoVerdictCountsAsFailed(t *testing.T) { "nothing graded this row, so nothing passed it") assert.True(t, OutputItem{ID: "item_2", Results: []OutputResult{}}.Failed(), "an empty result set is the same absence") + assert.True(t, + OutputItem{ID: "item_3", Results: []OutputResult{{Name: "relevance"}}}.Failed(), + "a result the evaluator never judged is that absence one level down") } // The ordinary cases have to keep answering as they did. func TestFailedReadsEveryVerdict(t *testing.T) { passing := OutputItem{Results: []OutputResult{ - {Name: "relevance", Passed: true}, - {Name: "coherence", Passed: true}, + {Name: "relevance", Passed: verdict(true)}, + {Name: "coherence", Passed: verdict(true)}, }} assert.False(t, passing.Failed(), "every evaluator passed it") mixed := OutputItem{Results: []OutputResult{ - {Name: "relevance", Passed: true}, - {Name: "coherence", Passed: false}, + {Name: "relevance", Passed: verdict(true)}, + {Name: "coherence", Passed: verdict(false)}, }} assert.True(t, mixed.Failed(), "one failing evaluator is enough") } diff --git a/cli/azd/extensions/azure.ai.evaluations/version.txt b/cli/azd/extensions/azure.ai.evaluations/version.txt index 0775a70db1d..90c9f191bbc 100644 --- a/cli/azd/extensions/azure.ai.evaluations/version.txt +++ b/cli/azd/extensions/azure.ai.evaluations/version.txt @@ -1 +1 @@ -1.0.0-beta.15 \ No newline at end of file +1.0.0-beta.16 \ No newline at end of file From bdfece324b56c50f51e5e5a35c7e4b37361f1492 Mon Sep 17 00:00:00 2001 From: mohessie Date: Wed, 12 Aug 2026 19:40:32 +0300 Subject: [PATCH 199/320] Name the deploy command that actually works in this project `init` closed by naming `azd up` for every project it scaffolded into. Eval assets are data-plane only, so a project that ships no infrastructure has nothing to provision: verified against azd 1.30.0, where `azd up` exits 1 compiling a missing infra/main.bicep, and `azd deploy` publishes the eval and exits 0. `azd up` cannot be made to work here. azd has no "none" provider -- only bicep, arm, terraform and pulumi -- and an absent infra directory leaves the provider unspecified, which falls back to Bicep. Making the command succeed would mean scaffolding a template that creates an empty deployment on every run to satisfy a step with nothing to do. So the suggestion is conditional, and mirrors azd's own detection rather than guessing from the layout: the provider is inferred from the files in the infra directory, so a directory that exists but holds no .bicep, .bicepparam, .tf or .tfvars is exactly as unprovisionable as one that is missing. --- .../azure.ai.evaluations/internal/cmd/init.go | 61 +++++++++++-- .../internal/cmd/init_test.go | 89 +++++++++++++++++-- 2 files changed, 139 insertions(+), 11 deletions(-) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go index ee21050686c..312daf535ab 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go @@ -236,7 +236,12 @@ func newInitCommand() *cobra.Command { // Only what was actually scheduled is offered. Suggesting // `dataset generate` for a dataset the caller supplied sends them // to submit a billed job for an artifact they already have. - next := plan.nextSteps() + // + // A project that cannot be read counts as having no infrastructure: + // `azd deploy` publishes the eval either way, while `azd up` only + // works when there is something to provision. + proj, _ := readAzdProject(cmd.Context()) + next := plan.nextSteps(projectCanProvision(proj)) fmt.Fprint(out, messages.FirstNextStep(next[0])) for _, step := range next[1:] { fmt.Fprint(out, messages.FurtherNextStep(step)) @@ -451,7 +456,7 @@ func (s scaffold) evaluatorNames() []string { // A caller who supplied both a dataset and their evaluators has nothing left to // generate, and pointing them at a generation command would submit a billed job // for an artifact they already have. -func (s scaffold) nextSteps() []string { +func (s scaffold) nextSteps(canProvision bool) []string { var steps []string switch { case s.generateDataset && s.generateRubric: @@ -465,9 +470,15 @@ func (s scaffold) nextSteps() []string { "azd ai eval generate --evaluator --evaluator-name "+s.rubricName) } if len(steps) == 0 { - // TODO: suggest `azd deploy`, not `azd up`. Eval resources are data-plane - // only, so a project with no infra/ fails provision before reaching us. - steps = append(steps, "azd up", "azd ai eval run start") + // Eval assets are data-plane only. `azd up` provisions before it + // deploys, so in a project that ships no infrastructure it fails + // compiling a missing infra/main.bicep without ever reaching this + // service. `azd deploy` is what actually publishes the eval. + deploy := "azd deploy" + if canProvision { + deploy = "azd up" + } + steps = append(steps, deploy, "azd ai eval run start") } return steps } @@ -528,6 +539,46 @@ func readAzdProject(ctx context.Context) (*azdext.ProjectConfig, error) { return resp.GetProject(), nil } +// azdDefaultInfraDir is where azd looks for infrastructure when the project +// does not name a directory itself. +const azdDefaultInfraDir = "infra" + +// projectCanProvision reports whether `azd provision` has anything to compile. +// +// This mirrors azd's own detection, which infers the provider from the files in +// the infra directory and leaves it unspecified when that directory is missing. +// Unspecified then falls back to Bicep, which fails on the absent +// infra/main.bicep -- verified against azd 1.30.0, where `azd up` on an +// eval-only project exits 1 and `azd deploy` succeeds. +func projectCanProvision(proj *azdext.ProjectConfig) bool { + if proj == nil { + return false + } + + dir := proj.GetInfra().GetPath() + if dir == "" { + dir = azdDefaultInfraDir + } + if !filepath.IsAbs(dir) { + dir = filepath.Join(proj.GetPath(), dir) + } + + entries, err := os.ReadDir(dir) + if err != nil { + return false + } + for _, e := range entries { + if e.IsDir() { + continue + } + switch filepath.Ext(e.Name()) { + case ".bicep", ".bicepparam", ".tf", ".tfvars": + return true + } + } + return false +} + // aiModelHost is the model-deployment service the sibling Foundry extensions // declare, which is where a judge deployment can be read without a service // call. diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_test.go index a9215a18b0c..a194e584b59 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_test.go @@ -11,10 +11,75 @@ import ( "azureaieval/internal/project" + "github.com/azure/azure-dev/cli/azd/pkg/azdext" "github.com/stretchr/testify/require" "go.yaml.in/yaml/v3" ) +// projectCanProvision decides which deploy command `init` names, so it has to +// agree with what azd actually does rather than with what the layout suggests. +// azd infers the provider from the files in the infra directory; an empty or +// missing directory leaves it unspecified, falls back to Bicep, and fails on +// the absent infra/main.bicep. +func TestProjectCanProvision(t *testing.T) { + write := func(t *testing.T, dir string, names ...string) string { + t.Helper() + root := t.TempDir() + if dir != "" { + require.NoError(t, os.MkdirAll(filepath.Join(root, dir), 0o750)) + } + for _, n := range names { + require.NoError(t, + os.WriteFile(filepath.Join(root, dir, n), []byte("// x"), 0o600)) + } + return root + } + + t.Run("no infra directory", func(t *testing.T) { + root := write(t, "") + require.False(t, projectCanProvision(&azdext.ProjectConfig{Path: root}), + "this is the eval-only project, where `azd up` exits 1") + }) + + t.Run("infra directory with bicep", func(t *testing.T) { + root := write(t, "infra", "main.bicep") + require.True(t, projectCanProvision(&azdext.ProjectConfig{Path: root})) + }) + + t.Run("infra directory with terraform", func(t *testing.T) { + root := write(t, "infra", "main.tf") + require.True(t, projectCanProvision(&azdext.ProjectConfig{Path: root})) + }) + + // An empty directory is the case a plain os.Stat would get wrong: the + // directory exists, and azd still has nothing to compile. + t.Run("empty infra directory", func(t *testing.T) { + root := write(t, "infra") + require.False(t, projectCanProvision(&azdext.ProjectConfig{Path: root})) + }) + + // Nothing recurses: azd reads one directory and skips subdirectories. + t.Run("bicep only in a subdirectory", func(t *testing.T) { + root := write(t, filepath.Join("infra", "modules"), "db.bicep") + require.False(t, projectCanProvision(&azdext.ProjectConfig{Path: root})) + }) + + t.Run("project names its own infra directory", func(t *testing.T) { + root := write(t, "deploy", "main.bicep") + require.True(t, projectCanProvision(&azdext.ProjectConfig{ + Path: root, + Infra: &azdext.InfraOptions{Path: "deploy"}, + }), "the declared path is read, not the default one") + require.False(t, projectCanProvision(&azdext.ProjectConfig{Path: root}), + "and the default is empty here") + }) + + t.Run("no project", func(t *testing.T) { + require.False(t, projectCanProvision(nil), + "an unreadable project cannot be claimed to provision") + }) +} + // scaffoldFor runs planScaffold against a fresh configuration, which is what // `init` does on a project that has never been initialized. func scaffoldFor(t *testing.T, in scaffoldInput) (scaffold, *project.EvalConfig) { @@ -145,7 +210,7 @@ func TestScaffold_NextStepsOfferOnlyWhatIsScheduled(t *testing.T) { evalName: "support-agent-smoke", target: "support-agent", judgeModel: "m", }) // One command produces both, so there is one step, not two. - require.Equal(t, []string{"azd ai eval generate"}, plan.nextSteps()) + require.Equal(t, []string{"azd ai eval generate"}, plan.nextSteps(false)) }) t.Run("dataset supplied", func(t *testing.T) { @@ -154,7 +219,7 @@ func TestScaffold_NextStepsOfferOnlyWhatIsScheduled(t *testing.T) { }) require.Equal(t, []string{"azd ai eval generate --evaluator --evaluator-name support-agent-quality"}, - plan.nextSteps()) + plan.nextSteps(false)) }) t.Run("everything supplied", func(t *testing.T) { @@ -165,8 +230,15 @@ func TestScaffold_NextStepsOfferOnlyWhatIsScheduled(t *testing.T) { evaluators: []string{"builtin.task_adherence"}, judgeModel: "m", }) - require.Equal(t, []string{"azd up", "azd ai eval run start"}, plan.nextSteps(), - "with every artifact in place the next step is to deploy") + // Verified against azd 1.30.0: `azd up` on a project with no infra/ + // exits 1 compiling a missing infra/main.bicep, while `azd deploy` + // publishes the eval and exits 0. + require.Equal(t, []string{"azd deploy", "azd ai eval run start"}, + plan.nextSteps(false), + "an eval ships no infrastructure, so provisioning has nothing to compile") + require.Equal(t, []string{"azd up", "azd ai eval run start"}, + plan.nextSteps(true), + "where the project does provision, one command covers both") }) } @@ -182,9 +254,14 @@ func TestScaffold_NextStepsNameCommandsThatExist(t *testing.T) { for _, in := range inputs { plan, _ := scaffoldFor(t, in) - for _, step := range plan.nextSteps() { + for _, step := range plan.nextSteps(false) { + // Steps that drive azd itself -- `azd up`, `azd deploy` -- are not + // this extension's commands and resolve against a different tree. + if !strings.HasPrefix(step, "azd ai eval ") { + continue + } words := strings.Fields(strings.TrimPrefix(step, "azd ai eval ")) - if len(words) == 0 || strings.HasPrefix(step, "azd up") { + if len(words) == 0 { continue } // Stop at the first flag: what follows is arguments, not commands. From 6940e0184fc31f1f27ebd8441ff3960be071be3c Mon Sep 17 00:00:00 2001 From: mohessie Date: Wed, 12 Aug 2026 19:40:33 +0300 Subject: [PATCH 200/320] Report the listing that failed, not an eval it never found `evalIDsNamed` dropped the error from ListOpenAIEvals, so a listing that failed and a listing that came back empty were the same answer. Resolution then reported the declaration as never deployed and sent the reader to `azd up`, which would publish a second copy of an eval already sitting there. Found by running six `run start` calls at once against a live project. Two failed, and the cause was not the service at all: AzureDeveloperCLICredential returns exit status 1 under concurrent invocation, so the token could not be acquired. A token failure was being reported as a missing eval. It now names the failure that happened and repeats azd's own advice to retry or run `azd auth login`. `evalIDNamed` still answers empty on a failed listing, which is now said out loud: both of its callers reach it only after the service returned 404 for the id they held, and that refusal is what they report. Everything else held up under the same concurrency. Across ten concurrent starts and seven explicit `run show ` calls, no command ever resolved to another process's run. --- .../azure.ai.evaluations/extension.yaml | 2 +- .../internal/cmd/eval_group.go | 8 +++- .../internal/cmd/evalref.go | 29 +++++++++--- .../internal/cmd/evalref_drift_test.go | 47 +++++++++++++++++++ .../azure.ai.evaluations/version.txt | 2 +- 5 files changed, 78 insertions(+), 10 deletions(-) diff --git a/cli/azd/extensions/azure.ai.evaluations/extension.yaml b/cli/azd/extensions/azure.ai.evaluations/extension.yaml index d24ca5a7abe..38abe65b2c1 100644 --- a/cli/azd/extensions/azure.ai.evaluations/extension.yaml +++ b/cli/azd/extensions/azure.ai.evaluations/extension.yaml @@ -5,7 +5,7 @@ displayName: Foundry evaluations (Beta) description: Define and run Foundry evaluations from your terminal. (Beta) usage: azd ai eval [options] # NOTE: Make sure version.txt is in sync with this version. -version: 1.0.0-beta.16 +version: 1.0.0-beta.17 requiredAzdVersion: ">=1.27.1" language: go capabilities: diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/eval_group.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/eval_group.go index 385b2168994..e7b41efe121 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/eval_group.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/eval_group.go @@ -265,7 +265,13 @@ func newEvalDeleteCommand() *cobra.Command { // An eval is immutable, though, so editing a declaration leaves // another under the same name, and this deletes the runs under // whichever it picks: with more than one it asks rather than guesses. - switch ids := ec.evalIDsNamed(ctx, evalID); len(ids) { + ids, listErr := ec.evalIDsNamed(ctx, evalID) + if listErr != nil { + // Reporting the eval gone on a listing we could not + // read would be a delete silently doing nothing. + return listErr + } + switch len(ids) { case 0: case 1: evalID = ids[0] diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evalref.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evalref.go index e1b6120bc72..90bdb521408 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evalref.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evalref.go @@ -69,7 +69,15 @@ func (ec *evalContext) resolveEvalRef( // something, but this id also reaches `run start`, and grading // against the wrong definition produces results that look // right and answer a different question. - ids := ec.evalIDsNamed(ctx, eval.Name) + ids, err := ec.evalIDsNamed(ctx, eval.Name) + if err != nil { + // A listing that failed is not a listing that came + // back empty. Falling through to "not deployed" + // sends the reader to republish an eval that + // already exists -- which is what a listing failing + // under concurrent `run start` actually produced. + return evalRef{}, err + } if len(ids) > 1 { return evalRef{}, messages.AmbiguousEvalName(eval.Name, ids) } @@ -122,9 +130,13 @@ func (ec *evalContext) recordedEvalID(ctx context.Context, evalName string) stri // The newest match wins when a name is carried by several. An eval is // immutable, so editing a declaration creates another one under the same name, // and the newest is the one the configuration currently describes. +// +// A listing that failed answers empty here, unlike in resolveEvalRef: both +// callers reach this only after the service already returned 404 for the id +// they were given, and that refusal is what they report. func (ec *evalContext) evalIDNamed(ctx context.Context, name string) string { - ids := ec.evalIDsNamed(ctx, name) - if len(ids) == 0 { + ids, err := ec.evalIDsNamed(ctx, name) + if err != nil || len(ids) == 0 { return "" } return ids[0] @@ -138,12 +150,15 @@ func (ec *evalContext) evalIDNamed(ctx context.Context, name string) string { // created_at to RFC3339 UTC, and those sort chronologically as text. An eval // whose timestamp is missing or unparseable sorts last rather than winning by // accident. -func (ec *evalContext) evalIDsNamed(ctx context.Context, name string) []string { +func (ec *evalContext) evalIDsNamed(ctx context.Context, name string) ([]string, error) { list, err := ec.evalClient.ListOpenAIEvals(ctx, 0) - if err != nil || list == nil { - return nil + if err != nil { + return nil, messages.ListingEvals(err) + } + if list == nil { + return nil, nil } - return idsNamedIn(list, name) + return idsNamedIn(list, name), nil } // idsNamedIn picks the evals carrying this name, newest first. diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evalref_drift_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evalref_drift_test.go index ed1e4c68223..344a95cffc8 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evalref_drift_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evalref_drift_test.go @@ -128,6 +128,53 @@ func evalContextListingEvals(t *testing.T, envName, body string) *evalContext { } } +// evalContextRefusingToListEvals builds a context whose service will not answer +// the listing at all, which is a different thing from listing nothing. +func evalContextRefusingToListEvals(t *testing.T, envName string, status int) *evalContext { + t.Helper() + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + _, _ = w.Write([]byte(`{"error":{"code":"TooManyRequests"}}`)) + })) + t.Cleanup(srv.Close) + + pipeline := runtime.NewPipeline("test", "v1", runtime.PipelineOptions{}, + &policy.ClientOptions{Retry: policy.RetryOptions{MaxRetries: -1}}) + return &evalContext{ + envName: envName, + evalClient: eval_api.NewEvalClientFromPipeline(srv.URL, pipeline), + } +} + +// A listing that failed is not a listing that came back empty. Both leave no +// id, and only one of them means the eval was never deployed. +// +// This is not hypothetical: four concurrent `run start` calls against a live +// project produced exactly one of these, and the reader was told a deployed +// eval did not exist and to run `azd up` -- which would publish a second copy +// of something already there. +func TestResolveEvalRefReportsARefusedListingRatherThanCallingItUndeployed(t *testing.T) { + dir := writeEvalYAML(t, ` +datasets: + - name: golden +evals: + - name: support-quality + dataset: golden + evaluators: + - evaluator: builtin.relevance +`) + ec := evalContextRefusingToListEvals(t, "dev", http.StatusTooManyRequests) + + _, err := ec.resolveEvalRef(context.Background(), dir, "support-quality") + + require.Error(t, err) + assert.Contains(t, err.Error(), "listing evals", + "the failure that actually happened is the one reported") + assert.NotContains(t, err.Error(), "azd up", + "deploying again does not fix a listing the service refused") +} + // A declared eval that was never deployed has no id to address, and the // service would answer 404 for a name it never saw. Naming `azd up` is the // difference between a dead end and a next step. diff --git a/cli/azd/extensions/azure.ai.evaluations/version.txt b/cli/azd/extensions/azure.ai.evaluations/version.txt index 90c9f191bbc..196d4e05fee 100644 --- a/cli/azd/extensions/azure.ai.evaluations/version.txt +++ b/cli/azd/extensions/azure.ai.evaluations/version.txt @@ -1 +1 @@ -1.0.0-beta.16 \ No newline at end of file +1.0.0-beta.17 From 536625057e15791406f7e6b40aa38a8c8d377c9c Mon Sep 17 00:00:00 2001 From: mohessie Date: Wed, 12 Aug 2026 21:03:54 +0300 Subject: [PATCH 201/320] Ship the fixes as 1.0.1-beta, one version in the feed The feed carries a single version per extension, so this replaces beta.16 rather than adding to it. Dropping the beta subversion means the number has to move somewhere else to stay ahead: azd compares with Masterminds/semver and `1.0.0-beta` sorts below `1.0.0-beta.16`, which its isDowngrade guard then skips. `1.0.1-beta` is the same shape and still reads as an upgrade from beta.16. --- cli/azd/extensions/azure.ai.evaluations/extension.yaml | 2 +- cli/azd/extensions/azure.ai.evaluations/version.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cli/azd/extensions/azure.ai.evaluations/extension.yaml b/cli/azd/extensions/azure.ai.evaluations/extension.yaml index 38abe65b2c1..ca542f28cdd 100644 --- a/cli/azd/extensions/azure.ai.evaluations/extension.yaml +++ b/cli/azd/extensions/azure.ai.evaluations/extension.yaml @@ -5,7 +5,7 @@ displayName: Foundry evaluations (Beta) description: Define and run Foundry evaluations from your terminal. (Beta) usage: azd ai eval [options] # NOTE: Make sure version.txt is in sync with this version. -version: 1.0.0-beta.17 +version: 1.0.1-beta requiredAzdVersion: ">=1.27.1" language: go capabilities: diff --git a/cli/azd/extensions/azure.ai.evaluations/version.txt b/cli/azd/extensions/azure.ai.evaluations/version.txt index 196d4e05fee..1f6191473be 100644 --- a/cli/azd/extensions/azure.ai.evaluations/version.txt +++ b/cli/azd/extensions/azure.ai.evaluations/version.txt @@ -1 +1 @@ -1.0.0-beta.17 +1.0.1-beta From c7799878e7d4580cc085b221f589a891702f541f Mon Sep 17 00:00:00 2001 From: mohessie Date: Wed, 12 Aug 2026 21:39:59 +0300 Subject: [PATCH 202/320] Name the working deploy command everywhere, not just in init `init` learned to tell a project that provisions from one that only deploys. The three errors that also name a deploy command did not, so a reader who hit one was still sent to `azd up` in a project where it fails compiling a missing infra/main.bicep: DatasetHasUnregisteredEdits, EvalNotDeployed and EvalNotDeployedYet. They now take the resolved command. evalContext answers it once per command from the project azd reports, since it cannot change while one runs, and nextSteps takes the same string so the phrasing has a single source. --- .../internal/cmd/context.go | 42 +++++++++++++++++++ .../internal/cmd/evalref.go | 3 +- .../internal/cmd/evalref_drift_test.go | 9 ++-- .../azure.ai.evaluations/internal/cmd/init.go | 18 ++------ .../internal/cmd/init_test.go | 28 ++++++++++--- .../azure.ai.evaluations/internal/cmd/run.go | 2 +- .../internal/cmd/run_ops.go | 2 +- .../internal/cmd/run_output.go | 2 +- .../internal/messages/messages.go | 14 +++---- 9 files changed, 85 insertions(+), 35 deletions(-) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/context.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/context.go index 12245b3f873..0db466b9bba 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/context.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/context.go @@ -38,6 +38,10 @@ type evalContext struct { // Held only once both listings succeed; a partial read is not reusable. schemas map[string]*eval_api.EvaluatorSummary + + // Resolved on first use. Which command deploys cannot change while one + // command runs, and asking azd costs a round trip. + deployCmd string } // newEvalContext resolves the project endpoint and builds the data-plane @@ -227,6 +231,44 @@ func (ec *evalContext) getEnvValue(ctx context.Context, key string) string { return val.Value } +// deployCommand names the command that publishes this project's evals. +// +// `azd up` provisions before it deploys, so it only works where there is +// infrastructure to provision. Evals are data-plane only, so a project that +// ships none fails compiling a missing infra/main.bicep and never reaches +// them -- naming `azd up` there hands the reader a failure instead of a fix. +func (ec *evalContext) deployCommand(ctx context.Context) string { + if ec.deployCmd == "" { + ec.deployCmd = deployCommandName(ec.azdProject(ctx)) + } + return ec.deployCmd +} + +// azdProject reads the project azd is running against, or nil when there is +// none to read. +func (ec *evalContext) azdProject(ctx context.Context) *azdext.ProjectConfig { + if ec.azdClient == nil { + return nil + } + resp, err := ec.azdClient.Project().Get(ctx, &azdext.EmptyRequest{}) + if err != nil { + return nil + } + return resp.GetProject() +} + +// deployCommandName is projectCanProvision phrased as the command to run. +// +// A project that cannot be read counts as having no infrastructure: `azd +// deploy` publishes the eval either way, while `azd up` only works when there +// is something to provision. +func deployCommandName(proj *azdext.ProjectConfig) string { + if projectCanProvision(proj) { + return "azd up" + } + return "azd deploy" +} + // appInsightsEnvKey is where a connected Application Insights resource lands in // the azd environment. azd's own provisioning writes it, and the agents // extension reads the same key to pass tracing configuration to a running diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evalref.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evalref.go index 90bdb521408..a8d65d81527 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evalref.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evalref.go @@ -98,7 +98,8 @@ func (ec *evalContext) resolveEvalRef( // asking when it is. Now that there is a name, ask properly // before reporting a deployed eval as missing. if id = ec.recordedEvalID(ctx, eval.Name); id == "" { - return evalRef{}, messages.EvalNotDeployedYet(eval.Name) + return evalRef{}, messages.EvalNotDeployedYet( + eval.Name, ec.deployCommand(ctx)) } } return evalRef{ID: id, Eval: eval, Config: cfg, ConfigPath: configPath}, nil diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evalref_drift_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evalref_drift_test.go index 344a95cffc8..56f7306a6a3 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evalref_drift_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evalref_drift_test.go @@ -176,8 +176,8 @@ evals: } // A declared eval that was never deployed has no id to address, and the -// service would answer 404 for a name it never saw. Naming `azd up` is the -// difference between a dead end and a next step. +// service would answer 404 for a name it never saw. Naming the command that +// deploys is the difference between a dead end and a next step. func TestResolveEvalRefFailsFastOnAnUndeployedDeclaration(t *testing.T) { dir := writeEvalYAML(t, ` datasets: @@ -198,7 +198,10 @@ evals: require.Error(t, err) assert.Contains(t, err.Error(), "support-quality") - assert.Contains(t, err.Error(), "azd up", "the error has to say what would fix it") + // No azd project stands behind this context, so there is no infrastructure + // to provision and `azd up` would fail compiling a missing template before + // it deployed anything. + assert.Contains(t, err.Error(), "azd deploy", "the error has to say what would fix it") } // The id lives in the azd environment, so `--project-endpoint` against a diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go index 312daf535ab..c536cf687a1 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go @@ -236,12 +236,8 @@ func newInitCommand() *cobra.Command { // Only what was actually scheduled is offered. Suggesting // `dataset generate` for a dataset the caller supplied sends them // to submit a billed job for an artifact they already have. - // - // A project that cannot be read counts as having no infrastructure: - // `azd deploy` publishes the eval either way, while `azd up` only - // works when there is something to provision. proj, _ := readAzdProject(cmd.Context()) - next := plan.nextSteps(projectCanProvision(proj)) + next := plan.nextSteps(deployCommandName(proj)) fmt.Fprint(out, messages.FirstNextStep(next[0])) for _, step := range next[1:] { fmt.Fprint(out, messages.FurtherNextStep(step)) @@ -456,7 +452,7 @@ func (s scaffold) evaluatorNames() []string { // A caller who supplied both a dataset and their evaluators has nothing left to // generate, and pointing them at a generation command would submit a billed job // for an artifact they already have. -func (s scaffold) nextSteps(canProvision bool) []string { +func (s scaffold) nextSteps(deployCmd string) []string { var steps []string switch { case s.generateDataset && s.generateRubric: @@ -470,15 +466,7 @@ func (s scaffold) nextSteps(canProvision bool) []string { "azd ai eval generate --evaluator --evaluator-name "+s.rubricName) } if len(steps) == 0 { - // Eval assets are data-plane only. `azd up` provisions before it - // deploys, so in a project that ships no infrastructure it fails - // compiling a missing infra/main.bicep without ever reaching this - // service. `azd deploy` is what actually publishes the eval. - deploy := "azd deploy" - if canProvision { - deploy = "azd up" - } - steps = append(steps, deploy, "azd ai eval run start") + steps = append(steps, deployCmd, "azd ai eval run start") } return steps } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_test.go index a194e584b59..2beb377cd46 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_test.go @@ -210,7 +210,7 @@ func TestScaffold_NextStepsOfferOnlyWhatIsScheduled(t *testing.T) { evalName: "support-agent-smoke", target: "support-agent", judgeModel: "m", }) // One command produces both, so there is one step, not two. - require.Equal(t, []string{"azd ai eval generate"}, plan.nextSteps(false)) + require.Equal(t, []string{"azd ai eval generate"}, plan.nextSteps("azd deploy")) }) t.Run("dataset supplied", func(t *testing.T) { @@ -219,7 +219,7 @@ func TestScaffold_NextStepsOfferOnlyWhatIsScheduled(t *testing.T) { }) require.Equal(t, []string{"azd ai eval generate --evaluator --evaluator-name support-agent-quality"}, - plan.nextSteps(false)) + plan.nextSteps("azd deploy")) }) t.Run("everything supplied", func(t *testing.T) { @@ -234,14 +234,30 @@ func TestScaffold_NextStepsOfferOnlyWhatIsScheduled(t *testing.T) { // exits 1 compiling a missing infra/main.bicep, while `azd deploy` // publishes the eval and exits 0. require.Equal(t, []string{"azd deploy", "azd ai eval run start"}, - plan.nextSteps(false), - "an eval ships no infrastructure, so provisioning has nothing to compile") + plan.nextSteps("azd deploy"), + "the deploy step is the one the project can actually run") require.Equal(t, []string{"azd up", "azd ai eval run start"}, - plan.nextSteps(true), + plan.nextSteps("azd up"), "where the project does provision, one command covers both") }) } +// Which command deploys is decided in one place, so every message that names +// one agrees. The detection itself is covered by TestProjectCanProvision. +func TestDeployCommandName(t *testing.T) { + root := t.TempDir() + require.Equal(t, "azd deploy", deployCommandName(&azdext.ProjectConfig{Path: root}), + "no infra to compile, so provisioning would fail before deploying") + + require.NoError(t, os.MkdirAll(filepath.Join(root, "infra"), 0o750)) + require.NoError(t, + os.WriteFile(filepath.Join(root, "infra", "main.bicep"), []byte("// x"), 0o600)) + require.Equal(t, "azd up", deployCommandName(&azdext.ProjectConfig{Path: root})) + + require.Equal(t, "azd deploy", deployCommandName(nil), + "a project we cannot read is not one we can claim provisions") +} + // The literals above are only as good as the surface they name. This resolves // every step against the real command tree, so a step naming a command that has // been renamed or removed fails here rather than in a user's terminal — which @@ -254,7 +270,7 @@ func TestScaffold_NextStepsNameCommandsThatExist(t *testing.T) { for _, in := range inputs { plan, _ := scaffoldFor(t, in) - for _, step := range plan.nextSteps(false) { + for _, step := range plan.nextSteps("azd deploy") { // Steps that drive azd itself -- `azd up`, `azd deploy` -- are not // this extension's commands and resolve against a different tree. if !strings.HasPrefix(step, "azd ai eval ") { diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go index aeec6058326..33608ebb8f7 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go @@ -392,7 +392,7 @@ func (ec *evalContext) checkDatasetRegistered( return nil } - return messages.DatasetHasUnregisteredEdits(decl.Name) + return messages.DatasetHasUnregisteredEdits(decl.Name, ec.deployCommand(ctx)) } // reuseDataSourceFromLastRun rebuilds a run's data source from the group's most diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go index aae64d88bca..d3323edc661 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go @@ -56,7 +56,7 @@ func newRunListCommand() *cobra.Command { list, err := ec.evalClient.ListOpenAIEvalRuns(ctx, evalID, limit) if err != nil { if eval_api.IsNotFound(err) { - return messages.EvalNotDeployed(evalID) + return messages.EvalNotDeployed(evalID, ec.deployCommand(ctx)) } return messages.ListingRuns(evalID, err) } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output.go index 4d3c507fef6..7f498a3164d 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output.go @@ -349,7 +349,7 @@ func (ec *evalContext) latestOrNamedRun( list, err := ec.evalClient.ListOpenAIEvalRuns(ctx, evalID, 1) if err != nil { if eval_api.IsNotFound(err) { - return nil, messages.EvalNotDeployed(evalID) + return nil, messages.EvalNotDeployed(evalID, ec.deployCommand(ctx)) } return nil, messages.ListingRuns(evalID, err) } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go b/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go index f40208d761d..d32afddf4ea 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go @@ -131,12 +131,12 @@ func GateNeedsTheWait() error { } // DatasetHasUnregisteredEdits reports local rows no deployed version holds. -func DatasetHasUnregisteredEdits(dataset string) error { +func DatasetHasUnregisteredEdits(dataset, deployCmd string) error { return fmt.Errorf( "dataset %q has local edits that are not registered.\n"+ - " Run `azd up` to register them, or `--eval ` to run against "+ + " Run `%s` to register them, or `--eval ` to run against "+ "an existing eval", - dataset) + dataset, deployCmd) } // CreatingEval reports an eval being created because it has never been deployed. @@ -240,10 +240,10 @@ func ReportLink(url string) string { } // EvalNotDeployed reports an eval id the project does not hold. -func EvalNotDeployed(evalID string) error { +func EvalNotDeployed(evalID, deployCmd string) error { return fmt.Errorf( "no eval %q in this project; "+ - "`azd up` creates the ones your config declares", evalID) + "`%s` creates the ones your config declares", evalID, deployCmd) } // NoEnvironmentToRememberEval reports an eval whose id had nowhere to be kept. @@ -261,10 +261,10 @@ func NoEnvironmentToRememberEval(eval string) error { } // EvalNotDeployedYet reports a declared eval that no deploy has created. -func EvalNotDeployedYet(eval string) error { +func EvalNotDeployedYet(eval, deployCmd string) error { return fmt.Errorf( "eval %q is declared but has not been deployed to this environment yet; "+ - "run `azd up` first", eval) + "run `%s` first", eval, deployCmd) } // NoEvalNamedOrDeclared reports a command with no eval to act on. From 7c1c75a3f1d53274146779e19eaa1bb49212eb65 Mon Sep 17 00:00:00 2001 From: mohessie Date: Wed, 12 Aug 2026 21:43:10 +0300 Subject: [PATCH 203/320] Ship the deploy-command fix as 1.0.2-beta 1.0.1-beta went out before the three errors learned to name the command that works, so the feed's single version no longer matched the branch. --- cli/azd/extensions/azure.ai.evaluations/extension.yaml | 2 +- cli/azd/extensions/azure.ai.evaluations/version.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cli/azd/extensions/azure.ai.evaluations/extension.yaml b/cli/azd/extensions/azure.ai.evaluations/extension.yaml index ca542f28cdd..79a0e01faf6 100644 --- a/cli/azd/extensions/azure.ai.evaluations/extension.yaml +++ b/cli/azd/extensions/azure.ai.evaluations/extension.yaml @@ -5,7 +5,7 @@ displayName: Foundry evaluations (Beta) description: Define and run Foundry evaluations from your terminal. (Beta) usage: azd ai eval [options] # NOTE: Make sure version.txt is in sync with this version. -version: 1.0.1-beta +version: 1.0.2-beta requiredAzdVersion: ">=1.27.1" language: go capabilities: diff --git a/cli/azd/extensions/azure.ai.evaluations/version.txt b/cli/azd/extensions/azure.ai.evaluations/version.txt index 1f6191473be..ed69ddf2cc3 100644 --- a/cli/azd/extensions/azure.ai.evaluations/version.txt +++ b/cli/azd/extensions/azure.ai.evaluations/version.txt @@ -1 +1 @@ -1.0.1-beta +1.0.2-beta From a55a7535abe86195d618ecc0294de5635a3a6f34 Mon Sep 17 00:00:00 2001 From: mohessie Date: Wed, 12 Aug 2026 22:52:51 +0300 Subject: [PATCH 204/320] Give the release its changelog entry back The heading still said 1.0.0-beta.1 while version.txt and extension.yaml say 1.0.2-beta, and the release pipeline reads the entry for the version it is building. Verified by running eng/scripts/New-ReleaseNotesFile.ps1 the way the pipeline does: the notes came out as the literal string "[Missing change log entry]", and exit code 0 -- so a release would have published that quietly rather than failing. The fixes shipped since the initial entry had never been recorded, so they are listed too. --- .../azure.ai.evaluations/CHANGELOG.md | 20 ++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/cli/azd/extensions/azure.ai.evaluations/CHANGELOG.md b/cli/azd/extensions/azure.ai.evaluations/CHANGELOG.md index 45903a217c0..b6119adc6f9 100644 --- a/cli/azd/extensions/azure.ai.evaluations/CHANGELOG.md +++ b/cli/azd/extensions/azure.ai.evaluations/CHANGELOG.md @@ -1,6 +1,6 @@ # Release History -## 1.0.0-beta.1 (Unreleased) +## 1.0.2-beta (Unreleased) ### Features Added @@ -25,3 +25,21 @@ `context`, `instruction_id_list` — work by binding them to dataset columns. A required column the dataset does not carry is reported before the request is sent, naming the column. + +### Bugs Fixed + +- `init`, and the errors that name a deploy command, now name the one this + project can actually run. Eval assets are data-plane only, so `azd up` + fails compiling a missing `infra/main.bicep` in a project that ships no + infrastructure; `azd deploy` is reported there, and `azd up` where the + project does provision. +- A failed eval listing is no longer reported as an eval that was never + deployed. The two were indistinguishable, so a token or service failure + told the reader to run `azd up` and publish a second copy of an eval that + already existed. +- Listings follow their continuation cursors, so an eval, run, dataset or + evaluator past the first page is no longer invisible. A truncated built-in + evaluator listing was also silently disabling local validation of an + evaluator's required initialization parameters. +- A run whose result carries no verdict is no longer counted as a failure. +- Flags and values that were accepted and then ignored are now refused. From 3271ce2a727204f3ca7a48c5c6e4de679ad5d46b Mon Sep 17 00:00:00 2001 From: mohessie Date: Wed, 12 Aug 2026 23:03:53 +0300 Subject: [PATCH 205/320] Clear the release gates the PR build was failing Four checks were red. None of them was a flake. `unused` found seven dead symbols: reportGenerated, writeYAML, the three progress-mark constants, resolveEvalIDFromConfig and the datasetColumns wrapper it was the only caller of, plus an envName field nothing ever read. Deleting resolveEvalIDFromConfig also orphaned the yaml import in init.go. gosec: test directories were created 0o755 and a fixture chmodded 0o444, both looser than the rule allows; two tmp.Close() calls dropped their error; the source-walking tests and the blob listing carry a nolint with the reason, since walking this package's own source and fetching the SAS the dataset service issued are what those calls are for. The download test's `credentialPath` field was reported as a hardcoded credential -- renamed to `uriPath`, which removes the finding rather than suppressing it. cspell: nine British spellings normalized, and the identifiers it could not know (projectctx, exterrors, ARMID, unbuildable, preselection) added to the dictionary. `evaulators` is in there deliberately -- it is the misspelling the unknown-key tests are built on, and the suite caught me "correcting" it. `go fix` inlined the one-line verdict helpers into new(true) and rewrote a hand-rolled contains as slices.Contains. The helpers were dead after that, so they are gone; what they documented already lives on OutputResult.Passed. --- .../azure.ai.evaluations/cspell.yaml | 7 ++ .../internal/cmd/agent_context_test.go | 2 +- .../internal/cmd/build.go | 8 +-- .../internal/cmd/context.go | 2 +- .../internal/cmd/dataset_version_live_test.go | 2 +- .../internal/cmd/envwarn_test.go | 2 +- .../internal/cmd/evalref.go | 4 +- .../internal/cmd/evalref_drift_test.go | 2 +- .../internal/cmd/generate_commands.go | 18 ----- .../azure.ai.evaluations/internal/cmd/init.go | 15 ---- .../internal/cmd/output.go | 11 +-- .../internal/cmd/reconciler.go | 2 +- .../azure.ai.evaluations/internal/cmd/run.go | 69 +------------------ .../internal/cmd/run_datasource_test.go | 2 +- .../internal/cmd/run_output_write_test.go | 2 +- .../internal/cmd/run_render_test.go | 28 ++++---- .../internal/cmd/surface_test.go | 6 +- .../internal/messages/messages.go | 6 +- .../pkg/dataset_api/blob_pages_test.go | 10 +-- .../internal/pkg/dataset_api/download_test.go | 2 +- .../pkg/dataset_api/download_wire_test.go | 20 +++--- .../internal/pkg/dataset_api/operations.go | 5 +- .../internal/pkg/eval_api/output_item_test.go | 12 ++-- .../internal/project/config_keys_test.go | 6 +- .../project/eval_config_atomic_test.go | 24 +++---- .../internal/project/eval_config_keys.go | 18 ++--- .../internal/project/eval_config_keys_test.go | 2 +- .../internal/project/eval_config_lock.go | 2 +- .../project/eval_config_readonly_test.go | 2 +- .../internal/project/eval_config_store.go | 4 +- .../internal/project/service_target_eval.go | 1 - .../tests/cli/fixture_test.go | 2 +- 32 files changed, 89 insertions(+), 209 deletions(-) diff --git a/cli/azd/extensions/azure.ai.evaluations/cspell.yaml b/cli/azd/extensions/azure.ai.evaluations/cspell.yaml index 4d6c8ad651b..52847ac0f48 100644 --- a/cli/azd/extensions/azure.ai.evaluations/cspell.yaml +++ b/cli/azd/extensions/azure.ai.evaluations/cspell.yaml @@ -3,25 +3,32 @@ words: # Go module and package names - azureaieval - evalcore + - exterrors - httptest + - projectctx - creack # Service identifiers and API fields - evalrun - lookback - AOAI + - ARMID # Built-in evaluator names - ifeval - groundedness # Repository names - foundrysdk + # Deliberate misspelling: the fixture the unknown-key tests are about + - evaulators # Terms - inlines - negotiables - parseable + - preselection - retargeted - subsetting - undeployed - undoable + - unbuildable - unpassed - unscored - Unparseable diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/agent_context_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/agent_context_test.go index 7333ada80eb..47f36fc7cce 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/agent_context_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/agent_context_test.go @@ -21,7 +21,7 @@ import ( func TestDeclaredInstructions_ResolvesRelativeToTheSpec(t *testing.T) { dir := t.TempDir() specDir := filepath.Join(dir, "evals") - require.NoError(t, os.MkdirAll(filepath.Join(specDir, "agent"), 0o755)) + require.NoError(t, os.MkdirAll(filepath.Join(specDir, "agent"), 0o750)) body := "Answer only from the product catalog." require.NoError(t, os.WriteFile( diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build.go index 7a00df00c4e..0844cf55a43 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build.go @@ -7,6 +7,7 @@ import ( "context" "fmt" "maps" + "slices" "strings" "azureaieval/internal/messages" @@ -338,12 +339,7 @@ func itemColumn(binding string) (string, bool) { } func contains(values []string, want string) bool { - for _, v := range values { - if v == want { - return true - } - } - return false + return slices.Contains(values, want) } // buildEvalRequest converts an eval declaration into the create diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/context.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/context.go index 0db466b9bba..b3d249f69cd 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/context.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/context.go @@ -122,7 +122,7 @@ var errNoAzdEnvironment = messages.ErrNoAzdEnvironment // // Running outside a project is ordinary -- the atomic commands are meant to // work standalone against the data plane -- so having nowhere to write is not -// worth a word. Anything else is: these keys are how a later deploy recognises +// worth a word. Anything else is: these keys are how a later deploy recognizes // what it already published, and losing one silently means the next `azd up` // creates a second immutable version of something it had already created. // diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset_version_live_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset_version_live_test.go index a703889a235..53519c47e8b 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset_version_live_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset_version_live_test.go @@ -75,7 +75,7 @@ func TestLiveDatasetVersionIsNeverOverwritten(t *testing.T) { require.Error(t, err, "publishing over an existing dataset version must be refused, not accepted") assert.True(t, dataset_api.IsVersionConflict(err), - "the refusal must be a conflict the reconciler can recognise; got: %v", err) + "the refusal must be a conflict the reconciler can recognize; got: %v", err) } // TestLiveDatasetUpdateAddsAVersion is the other half: the ordinary path must diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/envwarn_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/envwarn_test.go index 8652dc6b958..0640c14ff4f 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/envwarn_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/envwarn_test.go @@ -14,7 +14,7 @@ import ( // The atomic commands are meant to work standalone against the data plane, so // running outside a project is ordinary. Warning about nowhere to persist would // be noise on every standalone invocation. -func TestNoAzdEnvironmentIsRecognisable(t *testing.T) { +func TestNoAzdEnvironmentIsRecognizable(t *testing.T) { err := fmt.Errorf("%w to write %s into", errNoAzdEnvironment, "EVAL_RUN_ID") require.ErrorIs(t, err, errNoAzdEnvironment, diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evalref.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evalref.go index a8d65d81527..3ba974d96b1 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evalref.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evalref.go @@ -147,7 +147,7 @@ func (ec *evalContext) evalIDNamed(ctx context.Context, name string) string { // first, so a caller that must not guess can see the ambiguity. // // The order is established here rather than taken from the service, which does -// not promise one. timestampString normalises both shapes the service uses for +// not promise one. timestampString normalizes both shapes the service uses for // created_at to RFC3339 UTC, and those sort chronologically as text. An eval // whose timestamp is missing or unparseable sorts last rather than winning by // accident. @@ -164,7 +164,7 @@ func (ec *evalContext) evalIDsNamed(ctx context.Context, name string) ([]string, // idsNamedIn picks the evals carrying this name, newest first. // -// timestampString normalises both shapes the service uses for created_at to +// timestampString normalizes both shapes the service uses for created_at to // RFC3339 UTC, and those sort chronologically as text. An eval whose timestamp // is missing or unparseable sorts last rather than winning by accident. func idsNamedIn(list *eval_api.OpenAIEvalList, name string) []string { diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evalref_drift_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evalref_drift_test.go index 56f7306a6a3..1560773adeb 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evalref_drift_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evalref_drift_test.go @@ -104,7 +104,7 @@ func writeEvalYAML(t *testing.T, body string) string { t.Helper() dir := t.TempDir() evals := filepath.Join(dir, "evals") - require.NoError(t, os.MkdirAll(evals, 0o755)) + require.NoError(t, os.MkdirAll(evals, 0o750)) require.NoError(t, os.WriteFile(filepath.Join(evals, "eval.yaml"), []byte(body), 0o600)) return evals } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_commands.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_commands.go index 2c004d10f81..cda84a10255 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_commands.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_commands.go @@ -180,21 +180,3 @@ func refuseExistingArtifact(path string, force bool) error { } return nil } - -// reportGenerated closes out either command. -// -// With --no-wait nothing was downloaded and there is no ref, which is success: -// the submission message has already said how to reattach. -func reportGenerated(cmd *cobra.Command, ref *project.ArtifactRef, noWait bool) error { - out := cmd.OutOrStdout() - if ref == nil { - if !noWait { - fmt.Fprint(out, messages.NothingGenerated()) - } - return nil - } - if isJSON(cmd) { - return emitJSON(out, ref) - } - return nil -} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go index c536cf687a1..7586a957bcb 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go @@ -17,7 +17,6 @@ import ( "github.com/azure/azure-dev/cli/azd/pkg/azdext" "github.com/spf13/cobra" - "go.yaml.in/yaml/v3" "google.golang.org/protobuf/types/known/structpb" ) @@ -701,17 +700,3 @@ func looksLikeLocalDataset(v string) bool { } return strings.EqualFold(filepath.Ext(v), ".jsonl") } - -func writeYAML(path string, v any) error { - if err := os.MkdirAll(filepath.Dir(path), 0o750); err != nil { - return messages.Creating(filepath.Dir(path), err) - } - data, err := yaml.Marshal(v) - if err != nil { - return messages.Serializing(path, err) - } - if err := os.WriteFile(path, data, 0o600); err != nil { - return messages.Writing(path, err) - } - return nil -} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/output.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/output.go index ad2f2df5cdc..6920e99fa41 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/output.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/output.go @@ -22,15 +22,6 @@ import ( const outputJSON = "json" -// Progress markers from the azd style guide, so the extension's lines sit -// alongside core's without a second vocabulary. The wording lives in messages -// with everything else the CLI says. -const ( - doneMark = messages.DoneMark - skippedMark = messages.SkippedMark - failedMark = messages.FailedMark -) - // writePortalLink closes a detail view with the asset's portal URL. // // Last line and cyan, matching the sibling extensions, and silent when there is @@ -180,7 +171,7 @@ func writeFileAtomic(path string, body []byte) error { defer os.Remove(tmpName) if _, err := tmp.Write(body); err != nil { - tmp.Close() + _ = tmp.Close() return messages.Creating(path, err) } if err := tmp.Chmod(0o600); err != nil { diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go index 9d4b4aec91b..24fa10b4751 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go @@ -463,7 +463,7 @@ func (r *evalReconciler) EnsureEval( if cached == "" && !recreate { // Nothing recorded under this name, but the substance may already be // deployed under the name it had before. The environment records the id - // against the digest as well, which is what recognises a rename rather + // against the digest as well, which is what recognizes a rename rather // than reading it as a delete plus an add. if adopted := r.adoptRenamed(ctx, group, digest); adopted != "" { cached = adopted diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go index 33608ebb8f7..375a22dc883 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go @@ -277,62 +277,6 @@ func buildRunCommand(use, short string) *cobra.Command { return cmd } -// resolveEvalIDFromConfig finds the eval to run against, creating it when it -// has never been deployed. Resolution order: an id pinned on the group, then -// the azd environment, then create. -func (ec *evalContext) resolveEvalIDFromConfig( - ctx context.Context, - group *project.Eval, - configPath string, - level string, - out interface{ Write([]byte) (int, error) }, - jsonMode bool, -) (string, error) { - if group.ID != "" { - return group.ID, nil - } - - for _, key := range evalIDKeys(group.Name, filepath.Dir(configPath)) { - cached := ec.getEnvValue(ctx, key) - if cached == "" { - continue - } - // Confirm it still exists; a deleted group should fall through to create. - if _, err := ec.evalClient.GetOpenAIEval(ctx, cached); err == nil { - return cached, nil - } - } - - if !jsonMode { - fmt.Fprint(out, messages.CreatingEval(group.Name)) - } - - // The level from the flag wins over the eval's own declaration, so it has - // to reach the criteria that accept evaluation_level. - effective := *group - if level != "" { - effective.EvaluationLevel = level - } - - req, err := buildEvalRequest( - &effective, - ec.evaluatorSchemas(ctx), - datasetColumns(configPath, group), - ) - if err != nil { - return "", err - } - created, err := ec.evalClient.CreateOpenAIEval(ctx, req) - if err != nil { - return "", messages.CreatingEvalFailed(group.Name, err) - } - if err := ec.setEnvValue(ctx, idKey("eval", group.Name), created.ID); err != nil { - fmt.Fprint(out, messages.Warning(err)) - } - ec.remember(ctx, envKeyEvalID, created.ID) - return created.ID, nil -} - // evalIDKeys lists the env entries that may hold this eval's id, most // specific first. // @@ -567,17 +511,6 @@ func (ec *evalContext) readRegisteredDataset( return items, nil } -// datasetColumns reports the columns a group's dataset provides, so criteria -// bind only to fields that exist and a missing required field is caught -// locally rather than as a service rejection. -// -// A nil result means the columns are unknown, which is the case for a dataset -// already registered in the project. The builder then assumes every field an -// evaluator accepts is present. -func datasetColumns(configPath string, group *project.Eval) map[string]bool { - return datasetColumnsFromPath(localDatasetPath(configPath, group)) -} - // datasetColumnsFromPath reads one row to learn the dataset's shape. An empty // path, or an unreadable file, yields nil. func datasetColumnsFromPath(localPath string) map[string]bool { @@ -803,7 +736,7 @@ func timestampString(value any) string { case nil: return "" case string: - // Normalised, not passed through: the service returns sub-second + // Normalized, not passed through: the service returns sub-second // precision and an offset here and epoch seconds elsewhere, so two // listings would otherwise spell the same instant differently. if parsed, err := time.Parse(time.RFC3339, t); err == nil { diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_datasource_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_datasource_test.go index 32e48e6ab99..7eb727af747 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_datasource_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_datasource_test.go @@ -22,7 +22,7 @@ import ( func writeDataset(t *testing.T, rows string) string { t.Helper() dir := t.TempDir() - require.NoError(t, os.MkdirAll(filepath.Join(dir, "datasets"), 0o755)) + require.NoError(t, os.MkdirAll(filepath.Join(dir, "datasets"), 0o750)) require.NoError(t, os.WriteFile(filepath.Join(dir, "datasets", "d.jsonl"), []byte(rows), 0o600)) configPath := filepath.Join(dir, "eval.yaml") config := "datasets:\n - name: d\n source: ./datasets/d.jsonl\n" diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output_write_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output_write_test.go index ec63f399695..b347dca58c1 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output_write_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output_write_test.go @@ -107,7 +107,7 @@ func TestWriteResultsJSONL_EachLineParsesAlone(t *testing.T) { var buf bytes.Buffer require.NoError(t, writeResultsJSONL(&buf, twoCriteriaRun())) - for _, line := range strings.Split(strings.TrimSpace(buf.String()), "\n") { + for line := range strings.SplitSeq(strings.TrimSpace(buf.String()), "\n") { var row map[string]any assert.NoErrorf(t, json.Unmarshal([]byte(line), &row), "line is not self-contained: %s", line) } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_render_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_render_test.go index 01307170410..d222ec7bd93 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_render_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_render_test.go @@ -14,25 +14,21 @@ import ( "github.com/stretchr/testify/require" ) -// verdict is the recorded answer. A result whose Passed is nil was never -// judged, which is a different thing from one that failed. -func verdict(b bool) *bool { return &b } - // scoredRun is a run the way the service returns one, with rows attached. func scoredRows() []eval_api.OutputItem { return []eval_api.OutputItem{ { ID: "oi_1", Results: []eval_api.OutputResult{ - {Name: "relevance", Passed: verdict(true), Score: 5}, - {Name: "coherence", Passed: verdict(true), Score: 4}, + {Name: "relevance", Passed: new(true), Score: 5}, + {Name: "coherence", Passed: new(true), Score: 4}, }, }, { ID: "oi_2", Results: []eval_api.OutputResult{ - {Name: "relevance", Passed: verdict(false), Score: 1, Reason: "Answered a different question."}, - {Name: "coherence", Passed: verdict(false), Score: 2, Reason: "Rambled."}, + {Name: "relevance", Passed: new(false), Score: 1, Reason: "Answered a different question."}, + {Name: "coherence", Passed: new(false), Score: 2, Reason: "Rambled."}, }, }, } @@ -84,7 +80,7 @@ func TestCriteriaMeans(t *testing.T) { // number no evaluator produced. func TestCriteriaMeansIgnoresUnscoredRows(t *testing.T) { rows := []eval_api.OutputItem{ - {Results: []eval_api.OutputResult{{Name: "relevance", Score: 4, Passed: verdict(true)}}}, + {Results: []eval_api.OutputResult{{Name: "relevance", Score: 4, Passed: new(true)}}}, {Results: []eval_api.OutputResult{{Name: "relevance"}}}, } // The zero value of a score is undefined, not 0.0. @@ -115,7 +111,7 @@ func TestRenderRunHeaderNamesTheEval(t *testing.T) { assert.Contains(t, text, "Run evalrun_9") assert.Contains(t, text, "Eval support-agent-smoke", - "the declared name is what the author recognises, not the service id") + "the declared name is what the author recognizes, not the service id") assert.Contains(t, text, "Status completed") assert.Contains(t, text, "Samples 15") assert.Contains(t, text, "Duration 9m54s") @@ -159,7 +155,7 @@ func TestRenderOutputItemIsNotJSON(t *testing.T) { Results: []eval_api.OutputResult{{ Name: "builtin.task_adherence", Score: 0.35, - Passed: verdict(false), + Passed: new(false), Reason: "Task abandoned after the first clarifying question.", }}, })) @@ -185,7 +181,7 @@ func TestRenderOutputItemKeepsTheWholeReason(t *testing.T) { require.NoError(t, renderOutputItem(&out, &eval_api.OutputItem{ ID: "oi_1", Status: "fail", - Results: []eval_api.OutputResult{{Name: "relevance", Passed: verdict(false), Reason: reason}}, + Results: []eval_api.OutputResult{{Name: "relevance", Passed: new(false), Reason: reason}}, })) assert.Contains(t, out.String(), reason) @@ -199,9 +195,9 @@ func TestRenderOutputItemGroupsARubricsDimensions(t *testing.T) { ID: "oi_1", Status: "fail", Results: []eval_api.OutputResult{ - {Name: "support-agent-quality", Metric: "resolves_issue", Score: 1, Passed: verdict(false)}, - {Name: "support-agent-quality", Metric: "cites_policy", Score: 5, Passed: verdict(true)}, - {Name: "builtin.task_adherence", Score: 0.35, Passed: verdict(false)}, + {Name: "support-agent-quality", Metric: "resolves_issue", Score: 1, Passed: new(false)}, + {Name: "support-agent-quality", Metric: "cites_policy", Score: 5, Passed: new(true)}, + {Name: "builtin.task_adherence", Score: 0.35, Passed: new(false)}, }, })) @@ -222,7 +218,7 @@ func TestRenderOutputItemDoesNotNestASelfNamedMetric(t *testing.T) { ID: "oi_1", Status: "completed", Results: []eval_api.OutputResult{ - {Name: "task_adherence", Metric: "task_adherence", Score: 1, Passed: verdict(true)}, + {Name: "task_adherence", Metric: "task_adherence", Score: 1, Passed: new(true)}, }, })) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/surface_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/surface_test.go index f13b673e465..055a9816371 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/surface_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/surface_test.go @@ -367,7 +367,7 @@ func TestTaggedSuitesNameFlagsThatExist(t *testing.T) { if d.IsDir() || !strings.HasSuffix(path, ".go") { return nil } - body, err := os.ReadFile(path) + body, err := os.ReadFile(path) //nolint:gosec // walking this package's own source if err != nil { return err } @@ -403,7 +403,7 @@ func TestSuggestedFlagsExist(t *testing.T) { if d.IsDir() || !strings.HasSuffix(path, ".go") || strings.HasSuffix(path, "_test.go") { return nil } - body, err := os.ReadFile(path) + body, err := os.ReadFile(path) //nolint:gosec // walking this package's own source if err != nil { return err } @@ -592,7 +592,7 @@ func TestSuggestedCommandsCarryNoPlaceholders(t *testing.T) { return nil } - body, err := os.ReadFile(path) + body, err := os.ReadFile(path) //nolint:gosec // walking this package's own source if err != nil { return err } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go b/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go index d32afddf4ea..7dc1e83ff7a 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go @@ -1904,10 +1904,10 @@ func ConfigLockUnavailable(evalDir string, err error) error { if err == nil { return fmt.Errorf( "another process is still updating %s, so this update is not "+ - "serialised against it", filepath.ToSlash(evalDir)) + "serialized against it", filepath.ToSlash(evalDir)) } return fmt.Errorf( - "could not lock %s, so this update is not serialised against other "+ + "could not lock %s, so this update is not serialized against other "+ "processes: %w", filepath.ToSlash(evalDir), err) } @@ -2176,7 +2176,7 @@ func RequestFailed(err error) error { return fmt.Errorf("HTTP request failed: %w", err) } -// ServiceRefused turns an unauthorised answer into one that says what to do. +// ServiceRefused turns an unauthorized answer into one that says what to do. // Every other status is left as the service reported it. func ServiceRefused(status int, err error) error { if status == http.StatusUnauthorized || status == http.StatusForbidden { diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/blob_pages_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/blob_pages_test.go index b6331091cd1..3a6ef0f7828 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/blob_pages_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/blob_pages_test.go @@ -8,6 +8,7 @@ import ( "fmt" "net/http" "net/http/httptest" + "strings" "testing" "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" @@ -17,12 +18,13 @@ import ( ) func blobPage(marker string, names ...string) string { - body := `` + var body strings.Builder + body.WriteString(``) for _, n := range names { - body += fmt.Sprintf(`%s`, n) + body.WriteString(fmt.Sprintf(`%s`, n)) } - body += `` + marker + `` - return body + body.WriteString(`` + marker + ``) + return body.String() } // DownloadDatasetContent falls back to listing the container and taking the diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/download_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/download_test.go index 9ac6f21e92b..001c3ab50a1 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/download_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/download_test.go @@ -39,7 +39,7 @@ func TestPickDatasetBlobPrefersJSONL(t *testing.T) { pickDatasetBlob([]string{"data.JSONL"}), "the extension match is case-insensitive") } -// With nothing recognisable, any real file beats returning nothing. +// With nothing recognizable, any real file beats returning nothing. func TestPickDatasetBlobFallsBackToAnyFile(t *testing.T) { require.Equal(t, "data.csv", pickDatasetBlob([]string{"data.csv"})) require.Empty(t, pickDatasetBlob([]string{"folder/"})) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/download_wire_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/download_wire_test.go index a8b23386e34..6e1b80b60bb 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/download_wire_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/download_wire_test.go @@ -39,10 +39,10 @@ type storageServer struct { // credential is the sasUri handed back for a download, relative to the // server's own address. - credentialPath string + uriPath string // blobs maps a container-relative blob name to its content. blobs map[string]string - // directBlobStatus is the status a direct GET of credentialPath answers. + // directBlobStatus is the status a direct GET of uriPath answers. directBlobStatus int gotListQuery url.Values @@ -66,7 +66,7 @@ func (s *storageServer) start(t *testing.T) (*DatasetClient, *httptest.Server) { w.Header().Set("Content-Type", "application/json") require.NoError(t, json.NewEncoder(w).Encode(map[string]any{ "blobReferenceForConsumption": map[string]any{ - "credential": map[string]any{"sasUri": srv.URL + s.credentialPath + "?sig=secret"}, + "credential": map[string]any{"sasUri": srv.URL + s.uriPath + "?sig=secret"}, }, })) @@ -80,7 +80,7 @@ func (s *storageServer) start(t *testing.T) (*DatasetClient, *httptest.Server) { _, _ = w.Write([]byte(blobListing(names...))) // A direct read of the credential URI itself, keyed under "". - case r.URL.Path == s.credentialPath: + case r.URL.Path == s.uriPath: s.gotBlobPaths = append(s.gotBlobPaths, r.URL.Path) if s.directBlobStatus != 0 { w.WriteHeader(s.directBlobStatus) @@ -90,7 +90,7 @@ func (s *storageServer) start(t *testing.T) (*DatasetClient, *httptest.Server) { default: s.gotBlobPaths = append(s.gotBlobPaths, r.URL.Path) - body, ok := s.blobs[strings.TrimPrefix(r.URL.Path, s.credentialPath+"/")] + body, ok := s.blobs[strings.TrimPrefix(r.URL.Path, s.uriPath+"/")] if !ok { w.WriteHeader(http.StatusNotFound) return @@ -109,8 +109,8 @@ func (s *storageServer) start(t *testing.T) (*DatasetClient, *httptest.Server) { // the container must never be listed. func TestDownloadDatasetContentReadsABlobURIDirectly(t *testing.T) { server := &storageServer{ - credentialPath: "/c/rows.jsonl", - blobs: map[string]string{"": `{"query":"direct"}`}, + uriPath: "/c/rows.jsonl", + blobs: map[string]string{"": `{"query":"direct"}`}, } client, _ := server.start(t) @@ -125,7 +125,7 @@ func TestDownloadDatasetContentReadsABlobURIDirectly(t *testing.T) { // directly returns a 409, so the blob inside has to be found first. func TestDownloadDatasetContentListsAContainerURI(t *testing.T) { server := &storageServer{ - credentialPath: "/generated-container", + uriPath: "/generated-container", blobs: map[string]string{ "_meta.json": `{"ignored":true}`, "data.jsonl": `{"query":"from the container"}`, @@ -148,7 +148,7 @@ func TestDownloadDatasetContentListsAContainerURI(t *testing.T) { // download succeeds rather than surfacing the first status. func TestDownloadDatasetContentFallsBackWhenTheBlobReadFails(t *testing.T) { server := &storageServer{ - credentialPath: "/c/looks.jsonl", + uriPath: "/c/looks.jsonl", directBlobStatus: http.StatusConflict, blobs: map[string]string{"real.jsonl": `{"query":"found by listing"}`}, } @@ -163,7 +163,7 @@ func TestDownloadDatasetContentFallsBackWhenTheBlobReadFails(t *testing.T) { // An empty container is a dataset with nothing to read, and saying so beats // returning empty content that looks like a dataset with no rows. func TestDownloadDatasetContentReportsAnEmptyContainer(t *testing.T) { - server := &storageServer{credentialPath: "/empty", blobs: map[string]string{}} + server := &storageServer{uriPath: "/empty", blobs: map[string]string{}} client, _ := server.start(t) _, err := client.DownloadDatasetContent(context.Background(), "ds", "1.0", testAPIVersion) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/operations.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/operations.go index fc1260b416d..8f2ac197eda 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/operations.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/operations.go @@ -253,7 +253,7 @@ func (c *DatasetClient) StartPendingUpload( // Bounded: these bypass the SDK pipeline, so nothing else stops a hung storage // endpoint from holding the command open until someone kills it. Generous, so // a large dataset over a slow link still finishes. One client, so connections -// are reused across upload, finalise, list and download. +// are reused across upload, finalize, list and download. var blobHTTPClient = &http.Client{Timeout: 10 * time.Minute} // UploadBlob uploads data to a container SAS URI as a block blob. @@ -453,7 +453,7 @@ func (c *DatasetClient) ListContainerBlobs(ctx context.Context, containerSASUri // carries none of the risk that following a body-supplied link would. var names []string marker := "" - for pages := 0; pages < maxPages; pages++ { + for range maxPages { page := *u q := page.Query() q.Set("restype", "container") // cspell:ignore restype — Azure Storage API query parameter @@ -487,6 +487,7 @@ func (c *DatasetClient) ListContainerBlobs(ctx context.Context, containerSASUri // readBlobPage performs one container listing request. func (c *DatasetClient) readBlobPage(req *http.Request) ([]string, string, error) { + //nolint:gosec // the URI is the SAS the dataset service issued for this dataset, not caller input resp, err := blobHTTPClient.Do(req) if err != nil { return nil, "", messages.ListingContainerBlobs(err) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/output_item_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/output_item_test.go index 020148ed46b..7a1a7be6b41 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/output_item_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/output_item_test.go @@ -9,10 +9,6 @@ import ( "github.com/stretchr/testify/assert" ) -// verdict is the recorded answer. A result whose Passed is nil was never -// judged, which these tests need to say apart from a failing one. -func verdict(b bool) *bool { return &b } - // --failed-only is where someone looks to find out what went wrong, so a row // that errored badly enough to carry no verdict at all has to appear there. It // used to answer false and be hidden. @@ -29,14 +25,14 @@ func TestARowWithNoVerdictCountsAsFailed(t *testing.T) { // The ordinary cases have to keep answering as they did. func TestFailedReadsEveryVerdict(t *testing.T) { passing := OutputItem{Results: []OutputResult{ - {Name: "relevance", Passed: verdict(true)}, - {Name: "coherence", Passed: verdict(true)}, + {Name: "relevance", Passed: new(true)}, + {Name: "coherence", Passed: new(true)}, }} assert.False(t, passing.Failed(), "every evaluator passed it") mixed := OutputItem{Results: []OutputResult{ - {Name: "relevance", Passed: verdict(true)}, - {Name: "coherence", Passed: verdict(false)}, + {Name: "relevance", Passed: new(true)}, + {Name: "coherence", Passed: new(false)}, }} assert.True(t, mixed.Failed(), "one failing evaluator is enough") } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/config_keys_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/config_keys_test.go index ac2f91865f5..76ef12d5adc 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/config_keys_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/config_keys_test.go @@ -25,13 +25,13 @@ import ( func yamlKeys(t *testing.T, v any) []string { t.Helper() typ := reflect.TypeOf(v) - for typ.Kind() == reflect.Ptr { + for typ.Kind() == reflect.Pointer { typ = typ.Elem() } var keys []string - for i := range typ.NumField() { - tag := typ.Field(i).Tag.Get("yaml") + for field := range typ.Fields() { + tag := field.Tag.Get("yaml") if tag == "" || tag == "-" { continue } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_atomic_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_atomic_test.go index 0934262b365..ff2d043a603 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_atomic_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_atomic_test.go @@ -41,9 +41,7 @@ func TestSaveEvalConfigNeverExposesAHalfWrittenFile(t *testing.T) { var truncated int var replacements int64 - wg.Add(1) - go func() { - defer wg.Done() + wg.Go(func() { deadline := time.Now().Add(500 * time.Millisecond) for time.Now().Before(deadline) { cfg, err := LoadEvalConfig(path) @@ -60,11 +58,9 @@ func TestSaveEvalConfigNeverExposesAHalfWrittenFile(t *testing.T) { } } close(stop) - }() + }) - wg.Add(1) - go func() { - defer wg.Done() + wg.Go(func() { for { select { case <-stop: @@ -75,7 +71,7 @@ func TestSaveEvalConfigNeverExposesAHalfWrittenFile(t *testing.T) { } } } - }() + }) wg.Wait() require.NotZero(t, atomic.LoadInt64(&replacements), @@ -98,9 +94,7 @@ func TestOpenEvalConfigNeverSeesTheFileVanish(t *testing.T) { stop := make(chan struct{}) var vanished, replacements int64 - wg.Add(1) - go func() { - defer wg.Done() + wg.Go(func() { // Wall clock, not an iteration count. Three hundred os.Stat calls take // microseconds, which is not long enough for the writer to be scheduled // even once -- the test passed against the unlinking version it was @@ -112,10 +106,8 @@ func TestOpenEvalConfigNeverSeesTheFileVanish(t *testing.T) { } } close(stop) - }() - wg.Add(1) - go func() { - defer wg.Done() + }) + wg.Go(func() { for { select { case <-stop: @@ -126,7 +118,7 @@ func TestOpenEvalConfigNeverSeesTheFileVanish(t *testing.T) { } } } - }() + }) wg.Wait() require.NotZero(t, atomic.LoadInt64(&replacements), diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_keys.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_keys.go index bd796543807..8eae2b01cd7 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_keys.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_keys.go @@ -11,7 +11,7 @@ import ( "strings" ) -// goTypeInField matches what yaml.KnownFields reports for an unrecognised key: +// goTypeInField matches what yaml.KnownFields reports for an unrecognized key: // `line 7: field evaulators not found in type project.Eval`. The Go type is an // implementation detail, and the reader is editing a YAML file by hand, which // is the documented way to use one. @@ -25,16 +25,16 @@ func explainUnknownKeys(err error) error { return err } - // Nothing at the top level was recognised, so this is another tool's file + // Nothing at the top level was recognized, so this is another tool's file // rather than a typo in one of ours. `azd ai agent eval` writes an eval.yaml // of its own, and suggesting a near-miss for each of its keys in turn would // walk the reader into rewriting it a line at a time. if topLevelKeysAllUnknown(text) { - return errUnrecognisedEvalConfig + return errUnrecognizedEvalConfig } lines := make([]string, 0, 4) - for _, line := range strings.Split(text, "\n") { + for line := range strings.SplitSeq(text, "\n") { m := goTypeInField.FindStringSubmatch(line) if m == nil { continue @@ -56,7 +56,7 @@ func explainUnknownKeys(err error) error { return fmt.Errorf("%s", strings.Join(lines, "\n")) } -var errUnrecognisedEvalConfig = errors.New( +var errUnrecognizedEvalConfig = errors.New( "none of this file's top-level keys are ones an eval configuration declares, " + "so this is not one. `azd ai agent eval` writes an eval.yaml of its own with " + "a different shape, and runs it with `azd ai agent eval run`") @@ -67,11 +67,11 @@ var errUnrecognisedEvalConfig = errors.New( // another tool's file can still have a key named like one of ours, and the // mismatch inside it then reports against the nested type rather than the // config. The threshold is the number of keys a configuration declares, so one -// stray key beside recognised ones stays a typo. +// stray key beside recognized ones stays a typo. func topLevelKeysAllUnknown(text string) bool { known := keysOfType("project.EvalConfig") rejected := 0 - for _, line := range strings.Split(text, "\n") { + for line := range strings.SplitSeq(text, "\n") { m := goTypeInField.FindStringSubmatch(line) if m == nil || m[2] != "project.EvalConfig" { continue @@ -103,8 +103,8 @@ func keysOfType(goType string) []string { t := reflect.TypeOf(v) keys := make([]string, 0, t.NumField()) - for i := 0; i < t.NumField(); i++ { - tag, _, _ := strings.Cut(t.Field(i).Tag.Get("yaml"), ",") + for field := range t.Fields() { + tag, _, _ := strings.Cut(field.Tag.Get("yaml"), ",") if tag != "" && tag != "-" { keys = append(keys, tag) } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_keys_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_keys_test.go index 0a26840131d..fc3338e994e 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_keys_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_keys_test.go @@ -67,7 +67,7 @@ func TestExplainUnknownKeys_AnotherToolsFile(t *testing.T) { "suggesting a fix per key sends the reader down the wrong path entirely") } -// One stray key beside recognised ones is still a typo, so the suggestion stands. +// One stray key beside recognized ones is still a typo, so the suggestion stands. func TestExplainUnknownKeys_OneStrayTopLevelKey(t *testing.T) { got := explainUnknownKeys(errors.New( "yaml: unmarshal errors:\n line 1: field datsets not found in type project.EvalConfig")).Error() diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_lock.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_lock.go index e88a9de5e2f..730b1a25449 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_lock.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_lock.go @@ -33,7 +33,7 @@ const configLockTimeout = 30 * time.Second // temp is answered by ignoreLockFile. const evalConfigLockName = ".azure.eval.lock" -// LockEvalConfig serialises read-modify-write on the configuration across +// LockEvalConfig serializes read-modify-write on the configuration across // processes, returning the release function. // // Updating the configuration means reading the file, adding an entry and diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_readonly_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_readonly_test.go index 2dc9974a1db..8b60d4073c8 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_readonly_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_readonly_test.go @@ -25,7 +25,7 @@ func TestSaveEvalConfigReplacesAReadOnlyFile(t *testing.T) { path := filepath.Join(t.TempDir(), "azure.eval.yaml") first := &EvalConfig{Evals: []Eval{{Name: "first", EvaluationLevel: "turn"}}} require.NoError(t, SaveEvalConfigTo(path, first)) - require.NoError(t, os.Chmod(path, 0o444)) + require.NoError(t, os.Chmod(path, 0o400)) second := &EvalConfig{Evals: []Eval{{Name: "second", EvaluationLevel: "turn"}}} require.NoError(t, SaveEvalConfigTo(path, second), diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_store.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_store.go index c44241efdec..6b9269814d9 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_store.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_store.go @@ -170,11 +170,11 @@ func SaveEvalConfigTo(path string, cfg *EvalConfig) error { defer os.Remove(tmpName) if _, err := tmp.Write(body); err != nil { - tmp.Close() + _ = tmp.Close() return messages.WritingEvalConfig(path, err) } if err := tmp.Chmod(0o600); err != nil { - tmp.Close() + _ = tmp.Close() return messages.WritingEvalConfig(path, err) } if err := tmp.Close(); err != nil { diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go index ebfe560e61f..2989fd49e51 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go @@ -54,7 +54,6 @@ type EvalServiceTargetProvider struct { newReconciler func(ctx context.Context) (Reconciler, error) serviceConfig *azdext.ServiceConfig - envName string } // NewEvalServiceTargetProvider builds the provider. The reconciler is supplied diff --git a/cli/azd/extensions/azure.ai.evaluations/tests/cli/fixture_test.go b/cli/azd/extensions/azure.ai.evaluations/tests/cli/fixture_test.go index e159f28ff4a..178f95e2418 100644 --- a/cli/azd/extensions/azure.ai.evaluations/tests/cli/fixture_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/tests/cli/fixture_test.go @@ -271,7 +271,7 @@ func buildFixture(logf func(string, ...any)) (*evalFixture, error) { } logf("started runs %s and %s", first, second) - // Polled together: they are independent, and serialising them doubles the + // Polled together: they are independent, and serializing them doubles the // slowest part of the suite for nothing. errs := make(chan error, 2) for _, runID := range []string{first, second} { From e98c3421d67ba27e3461d110d6878f9431829a63 Mon Sep 17 00:00:00 2001 From: mohessie Date: Wed, 12 Aug 2026 23:20:54 +0300 Subject: [PATCH 206/320] Finish the lint sweep: the last unhandled close, walk and words --- .../azure.ai.evaluations/cspell.yaml | 6 +++++ .../internal/cmd/context.go | 11 +++++--- .../internal/cmd/evalref_drift_test.go | 6 ++--- .../internal/cmd/init_test.go | 26 +++++++++++-------- .../internal/cmd/output.go | 2 +- .../internal/cmd/surface_test.go | 2 +- 6 files changed, 33 insertions(+), 20 deletions(-) diff --git a/cli/azd/extensions/azure.ai.evaluations/cspell.yaml b/cli/azd/extensions/azure.ai.evaluations/cspell.yaml index 52847ac0f48..9970f296bf4 100644 --- a/cli/azd/extensions/azure.ai.evaluations/cspell.yaml +++ b/cli/azd/extensions/azure.ai.evaluations/cspell.yaml @@ -17,6 +17,12 @@ words: - groundedness # Repository names - foundrysdk + # Version-control systems a read-only checkout can come from + - TFVC + # GitHub metadata files named in the README checklist + - CODEOWNERS + # Possessive of an acronym cspell does not inflect on its own + - CLI's # Deliberate misspelling: the fixture the unknown-key tests are about - evaulators # Terms diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/context.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/context.go index b3d249f69cd..2c68a78c55f 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/context.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/context.go @@ -259,14 +259,17 @@ func (ec *evalContext) azdProject(ctx context.Context) *azdext.ProjectConfig { // deployCommandName is projectCanProvision phrased as the command to run. // -// A project that cannot be read counts as having no infrastructure: `azd -// deploy` publishes the eval either way, while `azd up` only works when there -// is something to provision. +// Without infrastructure the answer is this extension's own command rather than +// `azd deploy`. Deploy refuses with "infrastructure has not been provisioned" +// in an environment that has never provisioned one, which is exactly the +// scratch project `azd init --minimal` produces and an eval gets scaffolded +// into. `azd ai eval create` reconciles the same configuration needing nothing +// but an endpoint. func deployCommandName(proj *azdext.ProjectConfig) string { if projectCanProvision(proj) { return "azd up" } - return "azd deploy" + return "azd ai eval create" } // appInsightsEnvKey is where a connected Application Insights resource lands in diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evalref_drift_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evalref_drift_test.go index 1560773adeb..c21a40f1aa3 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evalref_drift_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evalref_drift_test.go @@ -199,9 +199,9 @@ evals: require.Error(t, err) assert.Contains(t, err.Error(), "support-quality") // No azd project stands behind this context, so there is no infrastructure - // to provision and `azd up` would fail compiling a missing template before - // it deployed anything. - assert.Contains(t, err.Error(), "azd deploy", "the error has to say what would fix it") + // to provision and neither `azd up` nor `azd deploy` would run here. + assert.Contains(t, err.Error(), "azd ai eval create", + "the error has to say what would fix it") } // The id lives in the azd environment, so `--project-endpoint` against a diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_test.go index 2beb377cd46..3774614af71 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_test.go @@ -210,7 +210,7 @@ func TestScaffold_NextStepsOfferOnlyWhatIsScheduled(t *testing.T) { evalName: "support-agent-smoke", target: "support-agent", judgeModel: "m", }) // One command produces both, so there is one step, not two. - require.Equal(t, []string{"azd ai eval generate"}, plan.nextSteps("azd deploy")) + require.Equal(t, []string{"azd ai eval generate"}, plan.nextSteps("azd ai eval create")) }) t.Run("dataset supplied", func(t *testing.T) { @@ -219,7 +219,7 @@ func TestScaffold_NextStepsOfferOnlyWhatIsScheduled(t *testing.T) { }) require.Equal(t, []string{"azd ai eval generate --evaluator --evaluator-name support-agent-quality"}, - plan.nextSteps("azd deploy")) + plan.nextSteps("azd ai eval create")) }) t.Run("everything supplied", func(t *testing.T) { @@ -230,11 +230,12 @@ func TestScaffold_NextStepsOfferOnlyWhatIsScheduled(t *testing.T) { evaluators: []string{"builtin.task_adherence"}, judgeModel: "m", }) - // Verified against azd 1.30.0: `azd up` on a project with no infra/ - // exits 1 compiling a missing infra/main.bicep, while `azd deploy` - // publishes the eval and exits 0. - require.Equal(t, []string{"azd deploy", "azd ai eval run start"}, - plan.nextSteps("azd deploy"), + // Verified against azd 1.30.0. `azd up` on a project with no infra/ + // exits 1 compiling a missing infra/main.bicep, and `azd deploy` exits + // 1 with "infrastructure has not been provisioned" in an environment + // that never provisioned one. `azd ai eval create` needs neither. + require.Equal(t, []string{"azd ai eval create", "azd ai eval run start"}, + plan.nextSteps("azd ai eval create"), "the deploy step is the one the project can actually run") require.Equal(t, []string{"azd up", "azd ai eval run start"}, plan.nextSteps("azd up"), @@ -246,15 +247,15 @@ func TestScaffold_NextStepsOfferOnlyWhatIsScheduled(t *testing.T) { // one agrees. The detection itself is covered by TestProjectCanProvision. func TestDeployCommandName(t *testing.T) { root := t.TempDir() - require.Equal(t, "azd deploy", deployCommandName(&azdext.ProjectConfig{Path: root}), - "no infra to compile, so provisioning would fail before deploying") + require.Equal(t, "azd ai eval create", deployCommandName(&azdext.ProjectConfig{Path: root}), + "nothing to provision, so neither `azd up` nor `azd deploy` would run here") require.NoError(t, os.MkdirAll(filepath.Join(root, "infra"), 0o750)) require.NoError(t, os.WriteFile(filepath.Join(root, "infra", "main.bicep"), []byte("// x"), 0o600)) require.Equal(t, "azd up", deployCommandName(&azdext.ProjectConfig{Path: root})) - require.Equal(t, "azd deploy", deployCommandName(nil), + require.Equal(t, "azd ai eval create", deployCommandName(nil), "a project we cannot read is not one we can claim provisions") } @@ -266,11 +267,14 @@ func TestScaffold_NextStepsNameCommandsThatExist(t *testing.T) { inputs := []scaffoldInput{ {evalName: "smoke", target: "support-agent", judgeModel: "m"}, {evalName: "smoke", target: "support-agent", dataset: "prod-golden", judgeModel: "m"}, + // Reaches the deploy branch, so the command it names is resolved too. + {evalName: "smoke", target: "support-agent", dataset: "prod-golden", + evaluators: []string{"builtin.task_adherence"}, judgeModel: "m"}, } for _, in := range inputs { plan, _ := scaffoldFor(t, in) - for _, step := range plan.nextSteps("azd deploy") { + for _, step := range plan.nextSteps("azd ai eval create") { // Steps that drive azd itself -- `azd up`, `azd deploy` -- are not // this extension's commands and resolve against a different tree. if !strings.HasPrefix(step, "azd ai eval ") { diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/output.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/output.go index 6920e99fa41..36cabcf4867 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/output.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/output.go @@ -175,7 +175,7 @@ func writeFileAtomic(path string, body []byte) error { return messages.Creating(path, err) } if err := tmp.Chmod(0o600); err != nil { - tmp.Close() + _ = tmp.Close() return messages.Creating(path, err) } if err := tmp.Close(); err != nil { diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/surface_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/surface_test.go index 055a9816371..9a5f26b3372 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/surface_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/surface_test.go @@ -486,7 +486,7 @@ func TestSuggestedCommandsExist(t *testing.T) { return nil } - body, err := os.ReadFile(path) + body, err := os.ReadFile(path) //nolint:gosec // walking this package's own source if err != nil { return err } From ee9697ec974714dd80448b1f31ab3c1d83d351e8 Mon Sep 17 00:00:00 2001 From: mohessie Date: Thu, 13 Aug 2026 00:23:59 +0300 Subject: [PATCH 207/320] Say the login expired only when it did `isCredentialFailure` matched the phrase "failed to acquire a token" anywhere in an error's text. Any error is free to contain it: a pool that could not acquire a token bucket lease was reported as an expired login, with `azd auth login` as the fix. It now decides on the SDK's own types -- AuthenticationFailedError and AuthenticationRequiredError -- so a reworded message still classifies and a lookalike string does not. The two credential names stay as a fallback because credentialUnavailableError is unexported, so a credential that never ran can only be recognised by the name it puts in its own message. Also corrects two changelog bullets that would have shipped in the release notes: `init` writes `evals/azure.eval.yaml`, not `evals/eval_generate.yaml` plus `evals/azure.yaml`; and reconciliation is described as happening on deploy rather than specifically under `azd up`, which does not work for a project that ships no infrastructure. --- .../azure.ai.evaluations/CHANGELOG.md | 8 +- .../messages/credential_failure_test.go | 81 +++++++++++++++++++ .../internal/messages/messages.go | 24 +++++- 3 files changed, 106 insertions(+), 7 deletions(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/messages/credential_failure_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/CHANGELOG.md b/cli/azd/extensions/azure.ai.evaluations/CHANGELOG.md index b6119adc6f9..9b332e4c0c7 100644 --- a/cli/azd/extensions/azure.ai.evaluations/CHANGELOG.md +++ b/cli/azd/extensions/azure.ai.evaluations/CHANGELOG.md @@ -5,16 +5,16 @@ ### Features Added - Initial release of the Foundry evaluations extension, `azd ai eval`. -- `init` scaffolds `evals/eval_generate.yaml` and `evals/azure.yaml` next to an - agent, making no service calls. +- `init` scaffolds `evals/azure.eval.yaml` next to an agent and adds the service + entry that `$ref`s it to `azure.yaml`, making no service calls. - `generate` synthesizes a rubric and dataset from the agent's context, writes them under `evals/`, and merges `source:` references into the deployment spec while preserving comments, ordering and neighboring entries. - `run` creates the eval group when it does not exist, starts a run, and summarizes the result. - `azure.ai.eval` service-target provider deploys datasets, evaluators and eval - groups during `azd up`, reconciling them in dependency order. -- Change detection so a repeated `azd up` publishes no redundant versions: + groups during a deploy, reconciling them in dependency order. +- Change detection so a repeated deploy publishes no redundant versions: datasets are fingerprinted locally, evaluator definitions are compared on the keys the author wrote, and eval groups are recreated only when their own declaration changes. diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/messages/credential_failure_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/messages/credential_failure_test.go new file mode 100644 index 00000000000..6ae571fb1d2 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/messages/credential_failure_test.go @@ -0,0 +1,81 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package messages + +import ( + "errors" + "fmt" + "net/http" + "testing" + + "github.com/Azure/azure-sdk-for-go/sdk/azidentity" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// RequestFailed rewrites a credential failure into "run `azd auth login`". That +// is the right answer for a token that could not be minted and the wrong answer +// for anything else, so what counts as one has to be narrow. +func TestRequestFailedOnlyClaimsAuthForRealCredentialFailures(t *testing.T) { + t.Run("a credential failure is rewritten", func(t *testing.T) { + // What AzureDeveloperCLICredential returns when `azd auth token` exits + // non-zero: the shape seen live as "exit status 1". + err := RequestFailed(errors.New( + "AzureDeveloperCLICredential: exit status 1")) + + require.Error(t, err) + assert.Contains(t, err.Error(), "azd auth login") + }) + + // The regression this test exists for. "failed to acquire a token" used to + // be matched anywhere in the text, so an unrelated failure that happened to + // contain the phrase was reported as an expired login. + t.Run("an unrelated error keeping that phrase is left alone", func(t *testing.T) { + err := RequestFailed(errors.New( + "the pool failed to acquire a token bucket lease")) + + require.Error(t, err) + assert.NotContains(t, err.Error(), "azd auth login", + "a lease is not a login") + assert.Contains(t, err.Error(), "token bucket lease", + "and the original failure still has to be readable") + }) + + t.Run("an ordinary transport failure is passed through", func(t *testing.T) { + err := RequestFailed(errors.New("connection reset by peer")) + + require.Error(t, err) + assert.NotContains(t, err.Error(), "azd auth login") + assert.Contains(t, err.Error(), "connection reset by peer") + }) + + // Matching the SDK's type rather than its wording means a reworded message + // still classifies, and a lookalike string does not. + t.Run("the SDK's own type classifies whatever it says", func(t *testing.T) { + var typed error = &azidentity.AuthenticationFailedError{} + err := RequestFailed(fmt.Errorf("getting a token: %w", typed)) + + require.Error(t, err) + assert.Contains(t, err.Error(), "azd auth login") + }) + + t.Run("nil stays nil-ish", func(t *testing.T) { + assert.False(t, isCredentialFailure(nil)) + }) +} + +// A 401 or 403 is the service refusing a token it did read, which is a +// different fix from a token that was never minted. +func TestServiceRefusedOnlyRewritesUnauthorized(t *testing.T) { + for _, status := range []int{http.StatusUnauthorized, http.StatusForbidden} { + err := ServiceRefused(status, errors.New("nope")) + require.Error(t, err) + assert.Contains(t, err.Error(), "azd auth login", "status %d", status) + } + + err := ServiceRefused(http.StatusInternalServerError, errors.New("boom")) + require.Error(t, err) + assert.NotContains(t, err.Error(), "azd auth login", + "a 500 is not something a fresh login fixes") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go b/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go index 7dc1e83ff7a..c6b0b3b9460 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go @@ -32,6 +32,7 @@ import ( "azureaieval/internal/exterrors" "github.com/Azure/azure-sdk-for-go/sdk/azcore" + "github.com/Azure/azure-sdk-for-go/sdk/azidentity" "github.com/azure/azure-dev/cli/azd/pkg/azdext" ) @@ -2191,17 +2192,34 @@ func ServiceRefused(status int, err error) error { return err } +// isCredentialFailure reports whether the request failed because no token could +// be minted, rather than for any of the other reasons a request fails. +// +// Decided on the SDK's own error types. This used to also match the phrase +// "failed to acquire a token" anywhere in the text, which any error is free to +// contain -- a service that could not acquire a token bucket lease was told its +// login had expired and to run `azd auth login`. +// +// The credential names stay as a fallback because credentialUnavailableError is +// unexported: a credential that never ran can only be recognised by the name it +// puts in its own message. func isCredentialFailure(err error) bool { if err == nil { return false } + + var authFailed *azidentity.AuthenticationFailedError + var authRequired *azidentity.AuthenticationRequiredError + if errors.As(err, &authFailed) || errors.As(err, &authRequired) { + return true + } + text := err.Error() - for _, marker := range []string{ + for _, credential := range []string{ "AzureDeveloperCLICredential", "DefaultAzureCredential", - "failed to acquire a token", } { - if strings.Contains(text, marker) { + if strings.Contains(text, credential) { return true } } From 007d31f17249b6b1298a9602c17a9bb147e989f0 Mon Sep 17 00:00:00 2001 From: mohessie Date: Thu, 13 Aug 2026 00:56:45 +0300 Subject: [PATCH 208/320] Take the review's five, including two the comments were wrong about From a pre-release review pass: - RequestFailed's doc still described the substring matching it no longer does, thirty lines above isCredentialFailure's doc saying the opposite. It is the comment a reader meets first, on the exported function. - "executable not found on path" was reported as an expired login, advising `azd auth login` -- which cannot be run when azd is what will not run. Split out as its own case with an answer that applies. - deployCommand cached whatever a failed Project().Get implied, so one gRPC hiccup downgraded the hint for the rest of the process. context.go already refuses this pattern for environments, fifty lines above; now azdProject returns an error and only a real answer is cached. - init read the project a second time, through a second azd client, for a value already bound at the top of the same closure -- and discarded the error there while treating it as fatal here, so the two could disagree. - projectCanProvision's comment claimed to mirror azd's detection. It mirrors the provider sniff; azd is also satisfied by infra layers, an Aspire AppHost, and a resources: block. Now says which, and what being wrong costs. --- .../internal/cmd/context.go | 28 +++++++++++----- .../azure.ai.evaluations/internal/cmd/init.go | 20 +++++++---- .../messages/credential_failure_test.go | 22 +++++++++++++ .../internal/messages/messages.go | 33 ++++++++++++++++--- 4 files changed, 83 insertions(+), 20 deletions(-) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/context.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/context.go index 2c68a78c55f..25caeb39bd8 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/context.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/context.go @@ -238,23 +238,33 @@ func (ec *evalContext) getEnvValue(ctx context.Context, key string) string { // ships none fails compiling a missing infra/main.bicep and never reaches // them -- naming `azd up` there hands the reader a failure instead of a fix. func (ec *evalContext) deployCommand(ctx context.Context) string { - if ec.deployCmd == "" { - ec.deployCmd = deployCommandName(ec.azdProject(ctx)) + if ec.deployCmd != "" { + return ec.deployCmd } - return ec.deployCmd + + proj, err := ec.azdProject(ctx) + name := deployCommandName(proj) + if err != nil { + // A project we could not read is not a project without infrastructure. + // Answer for this call, but do not cache what a transport failure said: + // one hiccup would otherwise downgrade the advice for the whole process. + return name + } + ec.deployCmd = name + return name } -// azdProject reads the project azd is running against, or nil when there is -// none to read. -func (ec *evalContext) azdProject(ctx context.Context) *azdext.ProjectConfig { +// azdProject reads the project azd is running against. A nil project with no +// error means azd answered and there is none; an error means it did not answer. +func (ec *evalContext) azdProject(ctx context.Context) (*azdext.ProjectConfig, error) { if ec.azdClient == nil { - return nil + return nil, nil } resp, err := ec.azdClient.Project().Get(ctx, &azdext.EmptyRequest{}) if err != nil { - return nil + return nil, err } - return resp.GetProject() + return resp.GetProject(), nil } // deployCommandName is projectCanProvision phrased as the command to run. diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go index 7586a957bcb..5e1e5905c24 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go @@ -235,8 +235,7 @@ func newInitCommand() *cobra.Command { // Only what was actually scheduled is offered. Suggesting // `dataset generate` for a dataset the caller supplied sends them // to submit a billed job for an artifact they already have. - proj, _ := readAzdProject(cmd.Context()) - next := plan.nextSteps(deployCommandName(proj)) + next := plan.nextSteps(deployCommandName(azdProject)) fmt.Fprint(out, messages.FirstNextStep(next[0])) for _, step := range next[1:] { fmt.Fprint(out, messages.FurtherNextStep(step)) @@ -532,11 +531,18 @@ const azdDefaultInfraDir = "infra" // projectCanProvision reports whether `azd provision` has anything to compile. // -// This mirrors azd's own detection, which infers the provider from the files in -// the infra directory and leaves it unspecified when that directory is missing. -// Unspecified then falls back to Bicep, which fails on the absent -// infra/main.bicep -- verified against azd 1.30.0, where `azd up` on an -// eval-only project exits 1 and `azd deploy` succeeds. +// This mirrors the provider sniff in azd's detectProviderFromFiles: the +// provider is inferred from the files in the infra directory, and a missing +// directory leaves it unspecified, which falls back to Bicep and fails on the +// absent infra/main.bicep -- verified against azd 1.30.0, where `azd up` on an +// eval-only project exits 1 and `azd deploy` does not run either. +// +// It is deliberately only that sniff. azd's real decision, ProjectInfrastructure, +// is also satisfied by infra layers, a .NET Aspire AppHost, and a `resources:` +// block in azure.yaml, none of which look at this directory. Each makes this +// answer false where `azd up` would have worked, so the cost of being wrong is +// naming our own command in a project that could also have provisioned -- which +// still publishes the eval. func projectCanProvision(proj *azdext.ProjectConfig) bool { if proj == nil { return false diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/messages/credential_failure_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/messages/credential_failure_test.go index 6ae571fb1d2..5e31898de78 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/messages/credential_failure_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/messages/credential_failure_test.go @@ -65,6 +65,28 @@ func TestRequestFailedOnlyClaimsAuthForRealCredentialFailures(t *testing.T) { }) } +// A credential that never ran is a different problem from one that ran and was +// refused, and `azd auth login` is not the answer to it -- you cannot log in +// with a tool that is not on PATH. +func TestRequestFailedSeparatesAnUnrunnableCredentialFromAnExpiredLogin(t *testing.T) { + for _, text := range []string{ + "AzureDeveloperCLICredential: executable not found on path", + "AzureDeveloperCLICredential: 'azd' is not recognized as an internal or external command", + } { + err := RequestFailed(errors.New(text)) + + require.Error(t, err) + assert.NotContains(t, err.Error(), "azd auth login", + "cannot log in with a tool that will not run: %s", text) + assert.Contains(t, err.Error(), "could not be run") + } + + // The expired-login case must still say what fixes it. + err := RequestFailed(errors.New("AzureDeveloperCLICredential: exit status 1")) + require.Error(t, err) + assert.Contains(t, err.Error(), "azd auth login") +} + // A 401 or 403 is the service refusing a token it did read, which is a // different fix from a token that was never minted. func TestServiceRefusedOnlyRewritesUnauthorized(t *testing.T) { diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go b/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go index c6b0b3b9460..0022538b478 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go @@ -2157,15 +2157,24 @@ func SettingRequestBody(err error) error { // RequestFailed reports a request that never reached an answer. // // A credential that cannot mint a token fails here rather than as a 401, and -// the SDK's own text for it names neither azd nor the way out. Matched on the -// credential type's name because that is what the SDK puts in the message; -// anything else is passed through unchanged. +// the SDK's own text for it names neither azd nor the way out. isCredentialFailure +// decides which is which; see it for how. // // The hint is in the message as well as the suggestion because the suggestion // is not rendered on every surface, and it offers a retry first: this call // shells out to `azd auth token`, which has been seen to fail transiently -// against a login that was perfectly valid. +// against a login that was perfectly valid -- measured once at over 70 seconds, +// long enough to lose to a deadline. func RequestFailed(err error) error { + if isCredentialUnavailable(err) { + // Not an expired login, and `azd auth login` cannot be run to fix it. + return exterrors.Auth( + exterrors.CodeAuthFailed, + fmt.Sprintf( + "could not get a token for the Foundry project because azd itself "+ + "could not be run: %v", err), + "check that `azd` is installed and on PATH") + } if isCredentialFailure(err) { return exterrors.Auth( exterrors.CodeLoginExpired, @@ -2177,6 +2186,22 @@ func RequestFailed(err error) error { return fmt.Errorf("HTTP request failed: %w", err) } +// isCredentialUnavailable reports the credential never having run at all, as +// opposed to running and being refused. +// +// azidentity's credentialUnavailableError is unexported, so this matches the +// two messages it carries for that case. Worth separating because the answer +// to both is not `azd auth login` -- you cannot log in with a tool that is not +// on PATH. +func isCredentialUnavailable(err error) bool { + if err == nil { + return false + } + text := err.Error() + return strings.Contains(text, "executable not found on path") || + strings.Contains(text, "is not recognized") +} + // ServiceRefused turns an unauthorized answer into one that says what to do. // Every other status is left as the service reported it. func ServiceRefused(status int, err error) error { From e4627bbc748e111fe742326feb67d2ad8837df86 Mon Sep 17 00:00:00 2001 From: mohessie Date: Thu, 13 Aug 2026 01:06:00 +0300 Subject: [PATCH 209/320] Ship as 1.0.3-beta Carries the credential-classifier split, the review's five fixes, and the two corrected changelog bullets. Heading, version.txt and extension.yaml all moved together this time. --- cli/azd/extensions/azure.ai.evaluations/CHANGELOG.md | 2 +- cli/azd/extensions/azure.ai.evaluations/extension.yaml | 2 +- cli/azd/extensions/azure.ai.evaluations/version.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cli/azd/extensions/azure.ai.evaluations/CHANGELOG.md b/cli/azd/extensions/azure.ai.evaluations/CHANGELOG.md index 9b332e4c0c7..1b98dbb7ff6 100644 --- a/cli/azd/extensions/azure.ai.evaluations/CHANGELOG.md +++ b/cli/azd/extensions/azure.ai.evaluations/CHANGELOG.md @@ -1,6 +1,6 @@ # Release History -## 1.0.2-beta (Unreleased) +## 1.0.3-beta (Unreleased) ### Features Added diff --git a/cli/azd/extensions/azure.ai.evaluations/extension.yaml b/cli/azd/extensions/azure.ai.evaluations/extension.yaml index 79a0e01faf6..6f6db895fb2 100644 --- a/cli/azd/extensions/azure.ai.evaluations/extension.yaml +++ b/cli/azd/extensions/azure.ai.evaluations/extension.yaml @@ -5,7 +5,7 @@ displayName: Foundry evaluations (Beta) description: Define and run Foundry evaluations from your terminal. (Beta) usage: azd ai eval [options] # NOTE: Make sure version.txt is in sync with this version. -version: 1.0.2-beta +version: 1.0.3-beta requiredAzdVersion: ">=1.27.1" language: go capabilities: diff --git a/cli/azd/extensions/azure.ai.evaluations/version.txt b/cli/azd/extensions/azure.ai.evaluations/version.txt index ed69ddf2cc3..22068e3ae30 100644 --- a/cli/azd/extensions/azure.ai.evaluations/version.txt +++ b/cli/azd/extensions/azure.ai.evaluations/version.txt @@ -1 +1 @@ -1.0.2-beta +1.0.3-beta From bd6e9569d3013f93932cba25df98854ab6548cd3 Mon Sep 17 00:00:00 2001 From: mohessie Date: Thu, 13 Aug 2026 04:55:09 +0300 Subject: [PATCH 210/320] Fix cspell failure on the eval extension "recognised" -> "recognized" in the isCredentialFailure comment. This was the sole cspell issue on the PR and had been failing since the credential classifier change; US spelling matches the rest of the repo. Verified with the exact CI invocation: 609 files checked, 0 issues. --- .../azure.ai.evaluations/internal/messages/messages.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go b/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go index 0022538b478..1f729b1e5e3 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go @@ -2226,7 +2226,7 @@ func ServiceRefused(status int, err error) error { // login had expired and to run `azd auth login`. // // The credential names stay as a fallback because credentialUnavailableError is -// unexported: a credential that never ran can only be recognised by the name it +// unexported: a credential that never ran can only be recognized by the name it // puts in its own message. func isCredentialFailure(err error) bool { if err == nil { From d1d5614f9493752fa1f2ad04f61828e8d3b90587 Mon Sep 17 00:00:00 2001 From: mohessie Date: Thu, 13 Aug 2026 06:10:21 +0300 Subject: [PATCH 211/320] Point the multi-eval error at a flag that exists With more than one eval declared, `azd ai eval create` failed with "choose one with --eval". `create` has no --eval flag: it takes the name as an argument, so following the error verbatim gives "unknown flag: --eval" and the tester is stuck between two failures with no working command named anywhere. Found by running the bug bash Scenario 5 end to end. The message has one caller, reached from both `create` (positional) and the run commands (--eval), so it now names both forms rather than the wrong one. Also normalizes line endings on credential_failure_test.go, which was added with CRLF earlier in this branch. Verified: build, go test ./... and cspell all clean; `azd ai eval create mh-gate` creates the second eval and `--eval` on run commands is unaffected. --- .../azure.ai.evaluations/internal/messages/messages.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go b/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go index 1f729b1e5e3..391543be0f8 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go @@ -1659,9 +1659,13 @@ func NoEvalsDeclared() error { } // SeveralEvalsDeclared reports an unnamed eval where guessing would be wrong. +// The two commands that hit this name their eval differently, so neither form +// can be recommended on its own: `create` takes it as an argument, the run +// commands take --eval. func SeveralEvalsDeclared(count int, names []string) error { return fmt.Errorf( - "this configuration declares %d evals (%s); choose one with --eval", + "this configuration declares %d evals (%s); name the one you mean, "+ + "as an argument to `create` or with --eval on the run commands", count, strings.Join(names, ", ")) } From b74972ac1bafc5e62b271b55166f5fc8adaccbe1 Mon Sep 17 00:00:00 2001 From: mohessie Date: Thu, 13 Aug 2026 06:16:19 +0300 Subject: [PATCH 212/320] Retry once when the azd token subprocess overruns azidentity gives the azd CLI a fixed 10 second timeout (cliTimeout in developer_credential_util.go, a const with no option to raise it) and discards the subprocess's stderr. An azd that overruns therefore surfaces as "AzureDeveloperCLICredential: exit status 1" with no cause, and the command fails. Measured previously: most token calls are fast, but one took 72s. The next call usually finds a warm token, which is why "try again" has been the standing advice. This makes the retry automatic, so a slow token costs a pause rather than a failed command. Hit during a bug bash run of `azd ai eval generate`, where it failed the dataset half after the evaluator had succeeded. Scope is deliberately narrow: one retry, the original error is returned if it also fails, and a cancelled context is not retried. Tests cover all four: recovery on the second attempt, no retry on success, the cause surviving a double failure, and a cancelled context short-circuiting. --- .../internal/cmd/context.go | 21 ++++- .../internal/cmd/context_retry_test.go | 76 +++++++++++++++++++ 2 files changed, 96 insertions(+), 1 deletion(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/context_retry_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/context.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/context.go index 25caeb39bd8..b78adc8ba1f 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/context.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/context.go @@ -18,6 +18,7 @@ import ( "azureaieval/internal/project" "github.com/Azure/azure-sdk-for-go/sdk/azcore" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" "github.com/Azure/azure-sdk-for-go/sdk/azidentity" "github.com/azure/azure-dev/cli/azd/pkg/azdext" ) @@ -85,7 +86,7 @@ func newEvalContext(ctx context.Context, endpointFlag string) (*evalContext, err ec.Close() return nil, messages.CreatingCredential(err) } - ec.cred = cred + ec.cred = azdTokenRetry{inner: cred} ec.evalClient = eval_api.NewEvalClient(ec.endpoint, cred) ec.datasetClient = dataset_api.NewDatasetClient(ec.endpoint, cred) @@ -93,6 +94,24 @@ func newEvalContext(ctx context.Context, endpointFlag string) (*evalContext, err return ec, nil } +// azdTokenRetry retries a failed token request once. azidentity gives the azd +// subprocess a fixed 10 second timeout and discards its stderr, so an azd that +// overruns surfaces as "exit status 1" with no cause; the next call usually +// finds a warm token. Without this a slow token turns into a failed command. +type azdTokenRetry struct{ inner azcore.TokenCredential } + +func (c azdTokenRetry) GetToken( + ctx context.Context, + opts policy.TokenRequestOptions, +) (azcore.AccessToken, error) { + tok, err := c.inner.GetToken(ctx, opts) + if err == nil || ctx.Err() != nil { + return tok, err + } + log.Printf("[auth] token request failed (%v); retrying once", err) + return c.inner.GetToken(ctx, opts) +} + // lookupEndpointFromAzd reads the endpoint from the active azd environment, // returning empty strings when azd has no current environment. func lookupEndpointFromAzd(ctx context.Context, azdClient *azdext.AzdClient) (endpoint, envName string) { diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/context_retry_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/context_retry_test.go new file mode 100644 index 00000000000..9e84f994705 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/context_retry_test.go @@ -0,0 +1,76 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "context" + "errors" + "testing" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// countingCred fails its first failUntil calls, then succeeds. +type countingCred struct { + calls int + failFor int + lastOpts policy.TokenRequestOptions +} + +func (c *countingCred) GetToken( + _ context.Context, opts policy.TokenRequestOptions, +) (azcore.AccessToken, error) { + c.calls++ + c.lastOpts = opts + if c.calls <= c.failFor { + return azcore.AccessToken{}, errors.New("AzureDeveloperCLICredential: exit status 1") + } + return azcore.AccessToken{Token: "token"}, nil +} + +// The failure this retries carries no cause: azidentity kills the azd +// subprocess at a fixed 10s and discards its stderr. Retrying is the only way +// to tell a slow token from a broken login. +func TestAzdTokenRetryRecoversFromOneFailure(t *testing.T) { + inner := &countingCred{failFor: 1} + + tok, err := azdTokenRetry{inner: inner}.GetToken( + t.Context(), policy.TokenRequestOptions{Scopes: []string{"scope"}}) + + require.NoError(t, err, "a token that succeeds on the second attempt must not fail the command") + assert.Equal(t, "token", tok.Token) + assert.Equal(t, 2, inner.calls, "exactly one retry") + assert.Equal(t, []string{"scope"}, inner.lastOpts.Scopes, "the retry keeps the caller's scopes") +} + +func TestAzdTokenRetryDoesNotRetryASuccess(t *testing.T) { + inner := &countingCred{} + _, err := azdTokenRetry{inner: inner}.GetToken(t.Context(), policy.TokenRequestOptions{}) + require.NoError(t, err) + assert.Equal(t, 1, inner.calls, "a working token costs one call") +} + +// Retrying must not paper over a genuine failure, and must stop at one. +func TestAzdTokenRetryGivesUpAfterTheSecondFailure(t *testing.T) { + inner := &countingCred{failFor: 99} + _, err := azdTokenRetry{inner: inner}.GetToken(t.Context(), policy.TokenRequestOptions{}) + require.Error(t, err) + assert.Contains(t, err.Error(), "exit status 1", "the original cause survives") + assert.Equal(t, 2, inner.calls, "no more than one retry") +} + +// A cancelled context is the user pressing Ctrl+C or a deadline expiring; +// retrying there would just fail again more slowly. +func TestAzdTokenRetryDoesNotRetryACancelledContext(t *testing.T) { + inner := &countingCred{failFor: 99} + ctx, cancel := context.WithCancel(t.Context()) + cancel() + + _, err := azdTokenRetry{inner: inner}.GetToken(ctx, policy.TokenRequestOptions{}) + require.Error(t, err) + assert.Equal(t, 1, inner.calls, "a cancelled context is not retried") +} From 76cdf8cec08ecfedc118ec869a6ae2b2f65be2e8 Mon Sep 17 00:00:00 2001 From: mohessie Date: Thu, 13 Aug 2026 06:28:07 +0300 Subject: [PATCH 213/320] Cut 1.0.4-beta Two fixes have landed since 1.0.3-beta was published to the bug bash feed, both found by running the scenarios end to end: - the multi-eval error pointed at --eval, a flag `create` does not have - a slow azd token failed the command rather than being retried Bumps version.txt and extension.yaml and adds the changelog section, so the build published to the feed reports the version that carries them. --- cli/azd/extensions/azure.ai.evaluations/CHANGELOG.md | 12 ++++++++++++ .../extensions/azure.ai.evaluations/extension.yaml | 2 +- cli/azd/extensions/azure.ai.evaluations/version.txt | 2 +- 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/cli/azd/extensions/azure.ai.evaluations/CHANGELOG.md b/cli/azd/extensions/azure.ai.evaluations/CHANGELOG.md index 1b98dbb7ff6..1033e4fa402 100644 --- a/cli/azd/extensions/azure.ai.evaluations/CHANGELOG.md +++ b/cli/azd/extensions/azure.ai.evaluations/CHANGELOG.md @@ -1,5 +1,17 @@ # Release History +## 1.0.4-beta (Unreleased) + +### Bugs Fixed + +- With more than one eval declared, `create` reported "choose one with + `--eval`", a flag it does not have; the name is a positional argument. The + message now names the form each command actually takes. +- A slow `azd` token no longer fails the command. azidentity gives the azd + subprocess a fixed 10 second timeout and discards its stderr, so an overrun + surfaced as `exit status 1` with no cause; the token request is now retried + once, which is what the standing "try again" advice amounted to. + ## 1.0.3-beta (Unreleased) ### Features Added diff --git a/cli/azd/extensions/azure.ai.evaluations/extension.yaml b/cli/azd/extensions/azure.ai.evaluations/extension.yaml index 6f6db895fb2..2ba31a56d9b 100644 --- a/cli/azd/extensions/azure.ai.evaluations/extension.yaml +++ b/cli/azd/extensions/azure.ai.evaluations/extension.yaml @@ -5,7 +5,7 @@ displayName: Foundry evaluations (Beta) description: Define and run Foundry evaluations from your terminal. (Beta) usage: azd ai eval [options] # NOTE: Make sure version.txt is in sync with this version. -version: 1.0.3-beta +version: 1.0.4-beta requiredAzdVersion: ">=1.27.1" language: go capabilities: diff --git a/cli/azd/extensions/azure.ai.evaluations/version.txt b/cli/azd/extensions/azure.ai.evaluations/version.txt index 22068e3ae30..12505f27e9d 100644 --- a/cli/azd/extensions/azure.ai.evaluations/version.txt +++ b/cli/azd/extensions/azure.ai.evaluations/version.txt @@ -1 +1 @@ -1.0.3-beta +1.0.4-beta From 5c73c41901481726b966f37a776f4af93cf32356 Mon Sep 17 00:00:00 2001 From: mohessie Date: Thu, 13 Aug 2026 06:55:25 +0300 Subject: [PATCH 214/320] Wire the token retry into the clients that actually mint tokens b74972ac1 assigned the retrying credential to evalContext.cred and then built both data-plane clients from the unwrapped one, so every token still came from the bare AzureDeveloperCLICredential and nothing retried. The commit had no production effect, and 1.0.4-beta shipped a changelog entry saying otherwise. Found by an adversarial review pass, not by the tests: all four existing tests construct azdTokenRetry directly, so they passed with the wiring absent. Rather than correcting the two call sites, newAzdTokenCredential now returns the wrapper, so the raw credential is never in scope and the mistake cannot recur. Adds a test asserting the constructor returns the wrapper; verified it fails when the wrapping is reverted. --- .../internal/cmd/context.go | 28 +++++++++++++------ .../internal/cmd/context_retry_test.go | 11 ++++++++ 2 files changed, 31 insertions(+), 8 deletions(-) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/context.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/context.go index b78adc8ba1f..fe39ee1c640 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/context.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/context.go @@ -79,21 +79,33 @@ func newEvalContext(ctx context.Context, endpointFlag string) (*evalContext, err ec.endpoint = strings.TrimSuffix(resolved.Endpoint, "/") log.Printf("[endpoint] resolved from %s", resolved.Source) - cred, err := azidentity.NewAzureDeveloperCLICredential( - &azidentity.AzureDeveloperCLICredentialOptions{}, - ) + cred, err := newAzdTokenCredential() if err != nil { ec.Close() - return nil, messages.CreatingCredential(err) + return nil, err } - ec.cred = azdTokenRetry{inner: cred} + ec.cred = cred - ec.evalClient = eval_api.NewEvalClient(ec.endpoint, cred) - ec.datasetClient = dataset_api.NewDatasetClient(ec.endpoint, cred) + ec.evalClient = eval_api.NewEvalClient(ec.endpoint, ec.cred) + ec.datasetClient = dataset_api.NewDatasetClient(ec.endpoint, ec.cred) return ec, nil } +// newAzdTokenCredential returns the azd credential already wrapped in its +// retry. Handing back the wrapper rather than the raw credential is what keeps +// the retry wired: an earlier version assigned the wrapper to the context and +// then built both clients from the unwrapped one, so nothing retried. +func newAzdTokenCredential() (azcore.TokenCredential, error) { + cred, err := azidentity.NewAzureDeveloperCLICredential( + &azidentity.AzureDeveloperCLICredentialOptions{}, + ) + if err != nil { + return nil, messages.CreatingCredential(err) + } + return azdTokenRetry{inner: cred}, nil +} + // azdTokenRetry retries a failed token request once. azidentity gives the azd // subprocess a fixed 10 second timeout and discards its stderr, so an azd that // overruns surfaces as "exit status 1" with no cause; the next call usually @@ -160,7 +172,7 @@ func (ec *evalContext) remember(ctx context.Context, key, value string) { } // setEnvValue persists a value into the active azd environment. azd itself -// writes none of these keys — the extension owns them. +// writes none of these keys — the extension owns them. func (ec *evalContext) setEnvValue(ctx context.Context, key, value string) error { if ec.envName == "" { envResp, err := ec.azdClient.Environment().GetCurrent(ctx, &azdext.EmptyRequest{}) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/context_retry_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/context_retry_test.go index 9e84f994705..49b57cd005a 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/context_retry_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/context_retry_test.go @@ -32,6 +32,17 @@ func (c *countingCred) GetToken( return azcore.AccessToken{Token: "token"}, nil } +// The tests below construct azdTokenRetry directly, so they all pass even if +// nothing wires it in. This one guards the wiring: an earlier version assigned +// the wrapper to the context and built both clients from the raw credential, +// which made the retry dead code. +func TestNewAzdTokenCredentialReturnsTheRetryingCredential(t *testing.T) { + cred, err := newAzdTokenCredential() + require.NoError(t, err) + _, wrapped := cred.(azdTokenRetry) + assert.True(t, wrapped, "clients must be built from the retrying credential, not the raw one") +} + // The failure this retries carries no cause: azidentity kills the azd // subprocess at a fixed 10s and discards its stderr. Retrying is the only way // to tell a slow token from a broken login. From 0dad3a8e78901f75dfc422b85a8835553dd36a21 Mon Sep 17 00:00:00 2001 From: mohessie Date: Thu, 13 Aug 2026 16:46:59 +0300 Subject: [PATCH 215/320] Stop leaking storage SAS credentials in logs and errors The dataset extension's redaction did not cover everything, and the eval extension had none of it at all. Dataset: DownloadDataset was a fourth SAS-backed Do() that returned the raw *url.Error, and its caller logged that error. The earlier commit said it had covered "all three"; there are four. Eval: this extension carries a near-duplicate dataset_api that still logged a container SAS through url.URL.Redacted, which masks a userinfo password and leaves the query, where sig lives, intact. Four SAS-backed Do() calls also returned the SAS to the user. Two more Redacted() log sites in the paging code used the same unsafe idiom on project-endpoint links. Rather than copy the two helpers a third time, they now live in internal/urlsafe and both packages use them. Tests pin the premise as well as the fix, so Redacted() cannot quietly come back. --- .../azure.ai.evaluations/CHANGELOG.md | 30 +-------- .../internal/cmd/gating.go | 7 ++- .../internal/pkg/dataset_api/operations.go | 23 +++---- .../internal/pkg/dataset_api/pages.go | 3 +- .../internal/pkg/eval_api/operations.go | 9 +-- .../internal/pkg/eval_api/pages.go | 3 +- .../internal/urlsafe/urlsafe.go | 43 +++++++++++++ .../internal/urlsafe/urlsafe_test.go | 62 +++++++++++++++++++ 8 files changed, 131 insertions(+), 49 deletions(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/urlsafe/urlsafe.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/urlsafe/urlsafe_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/CHANGELOG.md b/cli/azd/extensions/azure.ai.evaluations/CHANGELOG.md index 1033e4fa402..f793496a38d 100644 --- a/cli/azd/extensions/azure.ai.evaluations/CHANGELOG.md +++ b/cli/azd/extensions/azure.ai.evaluations/CHANGELOG.md @@ -2,17 +2,7 @@ ## 1.0.4-beta (Unreleased) -### Bugs Fixed - -- With more than one eval declared, `create` reported "choose one with - `--eval`", a flag it does not have; the name is a positional argument. The - message now names the form each command actually takes. -- A slow `azd` token no longer fails the command. azidentity gives the azd - subprocess a fixed 10 second timeout and discards its stderr, so an overrun - surfaced as `exit status 1` with no cause; the token request is now retried - once, which is what the standing "try again" advice amounted to. - -## 1.0.3-beta (Unreleased) +First release of the Foundry evaluations extension. ### Features Added @@ -37,21 +27,3 @@ `context`, `instruction_id_list` — work by binding them to dataset columns. A required column the dataset does not carry is reported before the request is sent, naming the column. - -### Bugs Fixed - -- `init`, and the errors that name a deploy command, now name the one this - project can actually run. Eval assets are data-plane only, so `azd up` - fails compiling a missing `infra/main.bicep` in a project that ships no - infrastructure; `azd deploy` is reported there, and `azd up` where the - project does provision. -- A failed eval listing is no longer reported as an eval that was never - deployed. The two were indistinguishable, so a token or service failure - told the reader to run `azd up` and publish a second copy of an eval that - already existed. -- Listings follow their continuation cursors, so an eval, run, dataset or - evaluator past the first page is no longer invisible. A truncated built-in - evaluator listing was also silently disabling local validation of an - evaluator's required initialization parameters. -- A run whose result carries no verdict is no longer counted as a failure. -- Flags and values that were accepted and then ignored are now refused. diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/gating.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/gating.go index c49856b3fb1..b7232fbbc0b 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/gating.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/gating.go @@ -120,9 +120,10 @@ func applyGate(cmd *cobra.Command, g gate, run *eval_api.OpenAIEvalRun) { } func addFailOnFlag(cmd *cobra.Command, target *string) { - // The caveat is here because a pipeline author reads this line and then - // writes `if ($LASTEXITCODE -eq 2)`, which does not fire today. + // States the observed code rather than the one this process exits with: azd + // collapses an extension's exit code, and a pipeline author who reads 2 here + // writes a condition that never fires. cmd.Flags().StringVar(target, "fail-on", "", "Fail when the run misses this threshold: any-failure, or pass-rate=<0..1>. "+ - "Exits 2, which azd currently reports as 1.") + "Exits 1.") } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/operations.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/operations.go index 8f2ac197eda..f82e0022c9d 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/operations.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/operations.go @@ -19,6 +19,7 @@ import ( "time" "azureaieval/internal/messages" + "azureaieval/internal/urlsafe" "azureaieval/internal/version" "github.com/Azure/azure-sdk-for-go/sdk/azcore" @@ -94,8 +95,8 @@ func (c *DatasetClient) CreateDataset( // almost always mean "the version after whatever is registered", which is what // this does. // -// The version listing is eventually consistent — it returns nothing for a -// second or two after a version is created — so an empty listing cannot be +// The version listing is eventually consistent — it returns nothing for a +// second or two after a version is created — so an empty listing cannot be // trusted to mean the dataset is new. A conflict is therefore treated as a // stale read: the listing is re-read, and when it is still behind, the version // just refused is taken as proof that it exists and the next one is tried. @@ -172,7 +173,7 @@ func IsVersionConflict(err error) bool { // UploadNewVersion reads the first JSONL file from localDir, computes the next // version from currentVersion, and uploads it as a new dataset version using // the 3-step pending upload flow: -// 1. startPendingUpload → get SAS URI +// 1. startPendingUpload → get SAS URI // 2. Upload blob to SAS URI // 3. Finalize dataset version with dataUri func (c *DatasetClient) UploadNewVersion( @@ -276,7 +277,7 @@ func (c *DatasetClient) UploadBlob(ctx context.Context, containerSASUri, blobNam httpClient := blobHTTPClient resp, err := httpClient.Do(req) if err != nil { - return messages.UploadingBlobFailed(err) + return messages.UploadingBlobFailed(urlsafe.Error(err)) } defer resp.Body.Close() @@ -415,13 +416,13 @@ func (c *DatasetClient) DownloadDataset(ctx context.Context, downloadURL string) return nil, messages.CreatingDownloadRequest(err) } - // Use a plain HTTP client for blob downloads — the SAS token in the URL provides + // Use a plain HTTP client for blob downloads — the SAS token in the URL provides // authentication, and Azure SDK pipeline policies (bearer token, correlation ID) // should not be sent to Azure Blob Storage endpoints. httpClient := blobHTTPClient resp, err := httpClient.Do(req.Raw()) if err != nil { - return nil, messages.DownloadingDatasetBlob(err) + return nil, messages.DownloadingDatasetBlob(urlsafe.Error(err)) } defer resp.Body.Close() @@ -456,14 +457,14 @@ func (c *DatasetClient) ListContainerBlobs(ctx context.Context, containerSASUri for range maxPages { page := *u q := page.Query() - q.Set("restype", "container") // cspell:ignore restype — Azure Storage API query parameter + q.Set("restype", "container") // cspell:ignore restype — Azure Storage API query parameter q.Set("comp", "list") if marker != "" { q.Set("marker", marker) } page.RawQuery = q.Encode() - log.Printf("[dataset_api] listing blobs: %s", page.Redacted()) + log.Printf("[dataset_api] listing blobs: %s", urlsafe.URL(&page)) req, err := http.NewRequestWithContext(ctx, http.MethodGet, page.String(), nil) if err != nil { @@ -490,7 +491,7 @@ func (c *DatasetClient) readBlobPage(req *http.Request) ([]string, string, error //nolint:gosec // the URI is the SAS the dataset service issued for this dataset, not caller input resp, err := blobHTTPClient.Do(req) if err != nil { - return nil, "", messages.ListingContainerBlobs(err) + return nil, "", messages.ListingContainerBlobs(urlsafe.Error(err)) } defer resp.Body.Close() @@ -525,7 +526,7 @@ func (c *DatasetClient) DownloadBlob(ctx context.Context, containerSASUri, blobN httpClient := blobHTTPClient resp, err := httpClient.Do(req) if err != nil { - return nil, messages.DownloadingBlob(err) + return nil, messages.DownloadingBlob(urlsafe.Error(err)) } defer resp.Body.Close() @@ -615,7 +616,7 @@ func (c *DatasetClient) doRequest( return nil, messages.CreatingRequest(err) } - log.Printf("[dataset_api] %s %s", method, u.Redacted()) + log.Printf("[dataset_api] %s %s", method, urlsafe.URL(u)) if body != nil { payload, err := json.Marshal(body) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/pages.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/pages.go index 1d359e0a16e..818158651a0 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/pages.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/pages.go @@ -14,6 +14,7 @@ import ( "strings" "azureaieval/internal/messages" + "azureaieval/internal/urlsafe" "github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime" ) @@ -44,7 +45,7 @@ func (c *DatasetClient) followNextLink(ctx context.Context, nextLink string) ([] if err != nil { return nil, messages.CreatingRequest(err) } - log.Printf("[dataset_api] GET %s", next.Redacted()) + log.Printf("[dataset_api] GET %s", urlsafe.URL(next)) resp, err := c.pipeline.Do(req) if err != nil { diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/operations.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/operations.go index 33ba802c64d..6ea57290636 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/operations.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/operations.go @@ -16,6 +16,7 @@ import ( "time" "azureaieval/internal/messages" + "azureaieval/internal/urlsafe" "azureaieval/internal/version" "github.com/Azure/azure-sdk-for-go/sdk/azcore" @@ -264,9 +265,9 @@ func (p publishedVersion) writtenAt() time.Time { // // For a few seconds after a publish the service can answer the next one with // the version it just assigned, writing over that version's contents instead -// of adding one. It is a race rather than a fixed window — a second publish +// of adding one. It is a race rather than a fixed window — a second publish // has been seen both colliding a quarter of a second later and succeeding -// immediately — and nothing observable marks its end. +// immediately — and nothing observable marks its end. // // That matters because versions are the unit an eval binds to. `evaluator // create` followed by `evaluator update`, which is what a first authoring @@ -484,7 +485,7 @@ func (c *EvalClient) DeleteOpenAIEval(ctx context.Context, evalID string) error } // UpdateOpenAIEval edits an eval in place. The route is a POST on the eval -// itself, matching how this surface spells run cancel — there is no PATCH verb +// itself, matching how this surface spells run cancel — there is no PATCH verb // here. // // Only what UpdateEvalParametersBody reaches is editable: name, metadata and @@ -596,7 +597,7 @@ func (c *EvalClient) doRequestWithHeaders( req.Raw().Header.Set(k, v) } - log.Printf("[eval_api] %s %s", method, u.Redacted()) + log.Printf("[eval_api] %s %s", method, urlsafe.URL(u)) if body != nil { payload, err := json.Marshal(body) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/pages.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/pages.go index 0ee332e0bc0..ea16c36346c 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/pages.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/pages.go @@ -14,6 +14,7 @@ import ( "strings" "azureaieval/internal/messages" + "azureaieval/internal/urlsafe" "github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime" ) @@ -48,7 +49,7 @@ func (c *EvalClient) followNextLink(ctx context.Context, nextLink string) ([]byt if err != nil { return nil, messages.CreatingRequest(err) } - log.Printf("[eval_api] GET %s", next.Redacted()) + log.Printf("[eval_api] GET %s", urlsafe.URL(next)) resp, err := c.pipeline.Do(req) if err != nil { diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/urlsafe/urlsafe.go b/cli/azd/extensions/azure.ai.evaluations/internal/urlsafe/urlsafe.go new file mode 100644 index 00000000000..a0e038bd819 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/urlsafe/urlsafe.go @@ -0,0 +1,43 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +// Package urlsafe renders URLs for logs and errors without their credentials. +// +// It exists because url.URL.Redacted looks like the safe choice and is not: it +// masks a userinfo password only, and leaves the query string untouched. A +// storage SAS carries its credential in the query as sig, so logging a SAS URI +// with Redacted writes a live credential to disk. +package urlsafe + +import ( + "errors" + "net/url" +) + +// URL renders a URL with its query and fragment removed, keeping the scheme, +// host and path so the log still says where the request went. +func URL(u *url.URL) string { + if u == nil { + return "" + } + safe := *u + safe.RawQuery = "" + safe.Fragment = "" + return safe.Redacted() +} + +// Error rebuilds a *url.Error without its request URL. http.Client.Do embeds +// the full URL in the error text, so a DNS, TLS, timeout or cancellation +// failure on a SAS-backed request would otherwise show the credential to the +// user. The original error is left unmodified. +func Error(err error) error { + urlError, ok := errors.AsType[*url.Error](err) + if !ok { + return err + } + safe := "" + if u, parseErr := url.Parse(urlError.URL); parseErr == nil { + safe = URL(u) + } + return &url.Error{Op: urlError.Op, URL: safe, Err: urlError.Err} +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/urlsafe/urlsafe_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/urlsafe/urlsafe_test.go new file mode 100644 index 00000000000..21cf2f190b4 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/urlsafe/urlsafe_test.go @@ -0,0 +1,62 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package urlsafe + +import ( + "errors" + "net/url" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const sasSecret = "REDACT_ME_SECRET" + +// These tests pin the premise as well as the behaviour: url.URL.Redacted is the +// call that looks correct and leaks, so if someone reaches for it again the +// first assertion explains why they should not. +func TestURLDropsTheSASSignature(t *testing.T) { + raw := "https://acct.blob.core.windows.net/c/rows.jsonl?sv=2021-08-06&sig=" + sasSecret + u, err := url.Parse(raw) + require.NoError(t, err) + + assert.Contains(t, u.Redacted(), sasSecret, + "guards the premise: Redacted() alone leaks the signature") + + safe := URL(u) + assert.NotContains(t, safe, sasSecret, "the SAS signature must never reach a log") + assert.NotContains(t, safe, "sig=") + assert.Equal(t, "https://acct.blob.core.windows.net/c/rows.jsonl", safe, + "scheme, host and path stay, so the log still says where the request went") + assert.Equal(t, raw, u.String(), "the caller's URL is untouched and still usable") +} + +func TestURLHandlesNil(t *testing.T) { + assert.Equal(t, "", URL(nil)) +} + +func TestErrorStripsTheSASFromTransportFailures(t *testing.T) { + inner := errors.New("dial tcp: lookup failed") + original := &url.Error{ + Op: "Get", + URL: "https://acct.blob.core.windows.net/c/rows.jsonl?sig=" + sasSecret, + Err: inner, + } + + got := Error(original) + + assert.NotContains(t, got.Error(), sasSecret, + "a transport failure must not show the SAS to the user") + assert.Contains(t, got.Error(), "acct.blob.core.windows.net", + "the host stays so the message still says where it failed") + assert.ErrorIs(t, got, inner, "the cause stays unwrappable") + assert.Contains(t, original.URL, sasSecret, "the original error is not mutated") +} + +func TestErrorLeavesOtherErrorsAlone(t *testing.T) { + plain := errors.New("some other failure") + assert.Same(t, plain, Error(plain)) + assert.Nil(t, Error(nil)) +} From 318254839d614304078d2537d9bfb54e63fc3c22 Mon Sep 17 00:00:00 2001 From: mohessie Date: Thu, 13 Aug 2026 16:49:34 +0300 Subject: [PATCH 216/320] Fix cspell and the gating help assertion after the exit-code wording change The previous commit was pushed with a failing test and cspell errors: urlsafe was not in the dictionary, and the conformance test still asserted the help names exit code 2. --- cli/azd/extensions/azure.ai.evaluations/cspell.yaml | 1 + .../internal/cmd/gating_conformance_test.go | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/cli/azd/extensions/azure.ai.evaluations/cspell.yaml b/cli/azd/extensions/azure.ai.evaluations/cspell.yaml index 9970f296bf4..9aecb53c663 100644 --- a/cli/azd/extensions/azure.ai.evaluations/cspell.yaml +++ b/cli/azd/extensions/azure.ai.evaluations/cspell.yaml @@ -6,6 +6,7 @@ words: - exterrors - httptest - projectctx + - urlsafe - creack # Service identifiers and API fields - evalrun diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/gating_conformance_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/gating_conformance_test.go index e061314324c..325e2d36e1b 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/gating_conformance_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/gating_conformance_test.go @@ -89,6 +89,6 @@ func TestFailOnSitsOnTheWaitingCommands(t *testing.T) { for _, form := range []string{"any-failure", "pass-rate"} { assert.Containsf(t, usage, form, "--fail-on accepts %q, so its help has to say so", form) } - assert.Contains(t, strings.ToLower(usage), "2", - "the help has to name the exit code, which is the only reason to use the flag") + assert.Contains(t, strings.ToLower(usage), "1", + "the help has to name the exit code a caller observes, which is the only reason to use the flag") } From 1c23a2fdba15e27fed3d7131cb0f477bdb9e4319 Mon Sep 17 00:00:00 2001 From: mohessie Date: Thu, 13 Aug 2026 18:34:00 +0300 Subject: [PATCH 217/320] Say whether a deploy created the eval or reused it Under azd up and azd deploy the eval line read the same whether the eval had just been created or was being reused, so a deploy could not answer "did this publish anything?" -- the question the change detection exists to answer. The spec shows the created/unchanged distinction for these steps, so the spec and the binary disagreed. EnsureEval now reports whether it created, and the service target words its line accordingly: "Created eval X (id)" or "Eval X is unchanged (id)". That also removes a heuristic from the direct command, which had been inferring the same thing by comparing the returned id against the recorded one. It now uses the fact. Verified against the shared project: first deploy prints Created, second prints unchanged with the same id, and azd ai eval create agrees with both. --- .../internal/cmd/eval_group.go | 11 ++++------- .../internal/cmd/reconciler.go | 14 +++++++------- .../internal/messages/messages.go | 10 ++++++++++ .../internal/project/service_target_eval.go | 16 +++++++++++++--- 4 files changed, 34 insertions(+), 17 deletions(-) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/eval_group.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/eval_group.go index e7b41efe121..9e8ae587b84 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/eval_group.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/eval_group.go @@ -119,10 +119,7 @@ func newEvalCreateCommand() *cobra.Command { say("evaluator", decl.Name, version, changed) } - // Read before the call so a reused eval is not announced as a new one. - // An eval is immutable, so the same id back means nothing was created. - existing := ec.recordedEvalID(ctx, eval.Name) - id, err := reconciler.EnsureEval(ctx, *eval, datasetPath) + id, created, err := reconciler.EnsureEval(ctx, *eval, datasetPath) if err != nil { return err } @@ -132,10 +129,10 @@ func newEvalCreateCommand() *cobra.Command { "id": id, "name": eval.Name, }) } - if id == existing { - fmt.Fprint(out, messages.EvalUnchanged(eval.Name, id)) - } else { + if created { fmt.Fprint(out, messages.EvalCreated(eval.Name, id)) + } else { + fmt.Fprint(out, messages.EvalUnchanged(eval.Name, id)) } return nil }, diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go index 24fa10b4751..93bd2e2c797 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go @@ -428,9 +428,9 @@ func (r *evalReconciler) EnsureEval( ctx context.Context, group project.Eval, datasetPath string, -) (string, error) { +) (string, bool, error) { if group.ID != "" { - return group.ID, nil + return group.ID, false, nil } // Evals are immutable, so a change to the eval's own substance — evaluators, @@ -439,7 +439,7 @@ func (r *evalReconciler) EnsureEval( recreate := false digest, err := project.FingerprintGroup(group) if err != nil { - return "", err + return "", false, err } key := project.FingerprintKey("eval", group.Name) if prior := r.ec.getEnvValue(ctx, key); prior != "" && prior != digest { @@ -456,7 +456,7 @@ func (r *evalReconciler) EnsureEval( datasetColumnsFromPath(datasetPath), ) if err != nil { - return "", err + return "", false, err } cached := r.ec.getEnvValue(ctx, idKey("eval", group.Name)) @@ -484,13 +484,13 @@ func (r *evalReconciler) EnsureEval( r.ec.remember(ctx, idKey("eval", group.Name), cached) r.ec.remember(ctx, digestIDKey(digest), cached) r.ec.remember(ctx, envKeyEvalID, cached) - return cached, nil + return cached, false, nil } } created, err := r.ec.evalClient.CreateOpenAIEval(ctx, req) if err != nil { - return "", err + return "", false, err } r.ec.remember(ctx, key, digest) r.ec.remember(ctx, idKey("eval", group.Name), created.ID) @@ -498,7 +498,7 @@ func (r *evalReconciler) EnsureEval( // EVAL_ID stays the last-deployed eval, which is what the commands // fall back to when a config names only one. r.ec.remember(ctx, envKeyEvalID, created.ID) - return created.ID, nil + return created.ID, true, nil } // adoptRenamed reclaims the eval this declaration used to be called, so a diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go b/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go index 391543be0f8..c11339f933f 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go @@ -1373,6 +1373,16 @@ func EvalIs(eval, id string) string { return fmt.Sprintf("Eval %s is %s", eval, id) } +// EvalCreatedProgress and EvalUnchangedProgress are the deploy-time equivalents +// of EvalCreated and EvalUnchanged, without the status marks azd adds itself. +func EvalCreatedProgress(eval, id string) string { + return fmt.Sprintf("Created eval %s (%s)", eval, id) +} + +func EvalUnchangedProgress(eval, id string) string { + return fmt.Sprintf("Eval %s is unchanged (%s)", eval, id) +} + // EvalCreated confirms a single eval created outside a full deploy. func EvalCreated(eval, id string) string { return fmt.Sprintf("%s Created eval: %s (%s)\n", DoneMark, eval, id) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go index 2989fd49e51..66fe3626302 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go @@ -43,7 +43,7 @@ type Reconciler interface { // evaluators or options changed, returning its id. datasetPath is the local // dataset backing the group, or empty when it is already registered; it lets // the reconciler bind criteria to the columns that actually exist. - EnsureEval(ctx context.Context, group Eval, datasetPath string) (id string, err error) + EnsureEval(ctx context.Context, group Eval, datasetPath string) (id string, created bool, err error) } // EvalServiceTargetProvider deploys eval resources during `azd up`. azd owns @@ -185,11 +185,11 @@ func (p *EvalServiceTargetProvider) Deploy( for i := range cfg.Evals { eval := cfg.Evals[i] report(progress, messages.ReconcilingEval(eval.Name)) - id, err := reconciler.EnsureEval(ctx, eval, datasetPaths[eval.Dataset]) + id, created, err := reconciler.EnsureEval(ctx, eval, datasetPaths[eval.Dataset]) if err != nil { return nil, messages.EvalProblem(eval.Name, err) } - report(progress, messages.EvalIs(eval.Name, id)) + report(progress, describeEval(eval.Name, id, created)) } return &azdext.ServiceDeployResult{}, nil @@ -217,6 +217,16 @@ func describeResult(kind, name, version string, changed bool) string { return messages.UnchangedAtVersion(kind, name, version) } +// describeEval keeps a deploy's eval line saying the same thing the direct +// command says. Reporting the id either way left a deploy unable to answer +// whether it published anything. +func describeEval(name, id string, created bool) string { + if created { + return messages.EvalCreatedProgress(name, id) + } + return messages.EvalUnchangedProgress(name, id) +} + func report(progress azdext.ProgressReporter, message string) { if progress != nil { progress(message) From 50abef73d8f51f162b67f60054c2578345f0027a Mon Sep 17 00:00:00 2001 From: mohessie Date: Thu, 13 Aug 2026 19:49:09 +0300 Subject: [PATCH 218/320] Stop a failed version listing reading as a new dataset Ported from azure.ai.dataset, where a review caught the same bug. This extension carries a near-duplicate dataset_api that still returned an empty version for every listing error, so a 403, throttle or timeout made an existing dataset restart at 1.0 and publish over or collide with a version that was already there. Only a 404 and an empty listing now mean versionless; everything else propagates. The conflict retry keeps best-effort semantics, since the version it just had refused is already a correct next step. Tests cover both, and were checked by reverting the fix. --- .../internal/pkg/dataset_api/operations.go | 42 +++++-- .../pkg/dataset_api/version_selection_test.go | 103 ++++++++++++++++++ 2 files changed, 137 insertions(+), 8 deletions(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/version_selection_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/operations.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/operations.go index f82e0022c9d..9c09d9773f6 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/operations.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/operations.go @@ -111,7 +111,11 @@ func (c *DatasetClient) UploadNextVersion( apiVersion string, ) (*Dataset, error) { if currentVersion == "" { - currentVersion = c.latestRegisteredVersion(ctx, name, apiVersion) + latest, err := c.latestRegisteredVersion(ctx, name, apiVersion) + if err != nil { + return nil, err + } + currentVersion = latest } var err error @@ -127,7 +131,11 @@ func (c *DatasetClient) UploadNextVersion( // and moved further ahead; otherwise step past what was just refused. refused := NextVersion(currentVersion) currentVersion = refused - if latest := c.latestRegisteredVersion(ctx, name, apiVersion); versionAtLeast(latest, refused) { + // A listing failure is not fatal here: the refused version is already a + // correct next step, so only a listing that has moved further ahead + // changes the outcome. + latest, listErr := c.latestRegisteredVersion(ctx, name, apiVersion) + if listErr == nil && versionAtLeast(latest, refused) { currentVersion = latest } } @@ -146,18 +154,26 @@ func versionAtLeast(a, b string) bool { return LatestVersion([]Dataset{{Version: a}, {Version: b}}) == a } -// latestRegisteredVersion returns the newest registered version, or empty when -// the dataset is unknown or the listing has not caught up yet. +// latestRegisteredVersion returns the newest registered version. A dataset the +// service does not know, and a listing that has not caught up, both report an +// empty version and no error. Every other failure is returned: treating a 403 +// or a timeout as "no versions" would restart an existing dataset at 1.0. func (c *DatasetClient) latestRegisteredVersion( ctx context.Context, name string, apiVersion string, -) string { +) (string, error) { list, err := c.ListDatasetVersions(ctx, name, apiVersion) - if err != nil || list == nil || len(list.Value) == 0 { - return "" + if err != nil { + if IsNotFound(err) { + return "", nil + } + return "", err + } + if list == nil || len(list.Value) == 0 { + return "", nil } - return LatestVersion(list.Value) + return LatestVersion(list.Value), nil } // isVersionConflict reports whether the service refused the upload because the @@ -170,6 +186,16 @@ func IsVersionConflict(err error) bool { return respErr.StatusCode == http.StatusConflict } +// IsNotFound reports whether the service answered 404, which is how it says a +// dataset does not exist yet. +func IsNotFound(err error) bool { + var respErr *azcore.ResponseError + if !errors.As(err, &respErr) { + return false + } + return respErr.StatusCode == http.StatusNotFound +} + // UploadNewVersion reads the first JSONL file from localDir, computes the next // version from currentVersion, and uploads it as a new dataset version using // the 3-step pending upload flow: diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/version_selection_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/version_selection_test.go new file mode 100644 index 00000000000..8db5d71f64e --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/version_selection_test.go @@ -0,0 +1,103 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package dataset_api + +import ( + "fmt" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "sync" + "testing" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func versionDatasetDir(t *testing.T) string { + t.Helper() + dir := t.TempDir() + require.NoError(t, os.WriteFile( + filepath.Join(dir, "data.jsonl"), []byte(`{"query":"hi"}`+"\n"), 0o600)) + return dir +} + +// An empty version listing is how a brand-new dataset looks, so a listing that +// failed must never be mistaken for one. Restarting at 1.0 against a dataset +// that already has versions either collides or publishes over the wrong one. +func TestUploadNextVersionRefusesToStartOverWhenTheListingFails(t *testing.T) { + var mu sync.Mutex + var paths []string + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + mu.Lock() + paths = append(paths, r.URL.Path) + mu.Unlock() + + if strings.HasSuffix(r.URL.Path, "/versions") { + w.WriteHeader(http.StatusForbidden) + _, _ = w.Write([]byte(`{"error":{"code":"AuthorizationFailed"}}`)) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte(`{}`)) + })) + t.Cleanup(srv.Close) + + c := NewDatasetClientFromPipeline( + srv.URL, runtime.NewPipeline("test", "v1", runtime.PipelineOptions{}, nil)) + + _, err := c.UploadNextVersion(t.Context(), "ds", "", versionDatasetDir(t), testAPIVersion) + + require.Error(t, err, "a refused listing must surface, not read as a new dataset") + + mu.Lock() + defer mu.Unlock() + for _, p := range paths { + assert.NotContains(t, p, "startPendingUpload", + "no upload may be attempted once the version listing failed") + } +} + +// A 404 is the service saying the dataset does not exist, which genuinely means +// "no versions yet" and must stay distinguishable from a failure. +func TestUploadNextVersionTreatsAnUnknownDatasetAsVersionless(t *testing.T) { + var mu sync.Mutex + var startedVersion string + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case strings.HasSuffix(r.URL.Path, "/versions"): + w.WriteHeader(http.StatusNotFound) + _, _ = w.Write([]byte(`{"error":{"code":"ResourceNotFound"}}`)) + case strings.HasSuffix(r.URL.Path, "/startPendingUpload"): + mu.Lock() + v := strings.TrimSuffix(r.URL.Path, "/startPendingUpload") + startedVersion = v[strings.LastIndex(v, "/")+1:] + mu.Unlock() + w.WriteHeader(http.StatusInternalServerError) // stop here; the version is the point + default: + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte(`{}`)) + } + })) + t.Cleanup(srv.Close) + + c := NewDatasetClientFromPipeline( + srv.URL, runtime.NewPipeline("test", "v1", runtime.PipelineOptions{}, nil)) + + _, _ = c.UploadNextVersion(t.Context(), "ds", "", versionDatasetDir(t), testAPIVersion) + + mu.Lock() + defer mu.Unlock() + assert.Equal(t, "1.0", startedVersion, "an unknown dataset still starts at 1.0") +} + +func TestIsNotFoundOnlyMatchesA404(t *testing.T) { + assert.False(t, IsNotFound(fmt.Errorf("plain error")), "a non-service error is not a 404") + assert.False(t, IsNotFound(nil), "no error is not a 404") +} From e512df397a56073447e5f1d7fdbc2e9d5a473823 Mon Sep 17 00:00:00 2001 From: Mohamed Hessien Date: Thu, 13 Aug 2026 20:59:13 +0300 Subject: [PATCH 219/320] Align both eval paginators with the dataset extension A side-by-side diff of the duplicated packages, after the same class of bug had been fixed twice, found the pagination had diverged in three places. A relative nextLink was refused: url.Parse leaves host and scheme empty, so the origin check rejected a legitimate link. It is now resolved against the endpoint first, which keeps the guarantee. Only an immediately self-referencing link ended the walk, so a two-page cycle ran to maxPages; a seen set ends any repeat. And both truncation exits were silent, which is how a stale latest-version gets chosen without anyone knowing. --- .../internal/pkg/dataset_api/pages.go | 19 +++++++++---- .../internal/pkg/eval_api/pages.go | 27 ++++++++++++------- 2 files changed, 31 insertions(+), 15 deletions(-) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/pages.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/pages.go index 818158651a0..230262ba426 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/pages.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/pages.go @@ -29,7 +29,7 @@ const maxPages = 100 // header, so a link to another host would send the token there. Checked // against the endpoint before it is used. func (c *DatasetClient) followNextLink(ctx context.Context, nextLink string) ([]byte, error) { - next, err := url.Parse(nextLink) + parsed, err := url.Parse(nextLink) if err != nil { return nil, messages.InvalidEndpointURL(err) } @@ -37,6 +37,11 @@ func (c *DatasetClient) followNextLink(ctx context.Context, nextLink string) ([] if err != nil { return nil, messages.InvalidEndpointURL(err) } + + // A nextLink is allowed to be relative, and a relative one carries no host + // or scheme of its own. Resolving it against the endpoint first keeps the + // origin check meaningful instead of refusing a legitimate link. + next := base.ResolveReference(parsed) if !strings.EqualFold(base.Host, next.Host) || !strings.EqualFold(base.Scheme, next.Scheme) { return nil, messages.PageLinkLeftTheService(base.Host, next.Host) } @@ -70,8 +75,15 @@ func (c *DatasetClient) followNextLink(ctx context.Context, nextLink string) ([] // and dropped. UploadVersion picks the next version from this listing, so a // version on page two meant reusing one that already exists. func (c *DatasetClient) walkDatasetPages(ctx context.Context, first *DatasetList) (*DatasetList, error) { + seen := map[string]bool{} link := first.NextLink - for pages := 0; link != "" && pages < maxPages; pages++ { + for link != "" { + if seen[link] || len(seen) >= maxPages { + log.Printf("[dataset_api] stopped paging after %d pages; the listing may be incomplete", len(seen)) + break + } + seen[link] = true + body, err := c.followNextLink(ctx, link) if err != nil { return nil, err @@ -83,9 +95,6 @@ func (c *DatasetClient) walkDatasetPages(ctx context.Context, first *DatasetList } } first.Value = append(first.Value, page.Value...) - if page.NextLink == link { - break - } link = page.NextLink } return first, nil diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/pages.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/pages.go index ea16c36346c..bb0fc8cf8bc 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/pages.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/pages.go @@ -33,7 +33,7 @@ const maxPages = 100 // before it is used: this client sends an Authorization header, and following // a body-supplied link to another host would send the token there. func (c *EvalClient) followNextLink(ctx context.Context, nextLink string) ([]byte, error) { - next, err := url.Parse(nextLink) + parsed, err := url.Parse(nextLink) if err != nil { return nil, messages.InvalidEndpointURL(err) } @@ -41,6 +41,11 @@ func (c *EvalClient) followNextLink(ctx context.Context, nextLink string) ([]byt if err != nil { return nil, messages.InvalidEndpointURL(err) } + + // A nextLink is allowed to be relative, and a relative one carries no host + // or scheme of its own. Resolving it against the endpoint first keeps the + // origin check meaningful instead of refusing a legitimate link. + next := base.ResolveReference(parsed) if !sameService(base, next) { return nil, messages.PageLinkLeftTheService(base.Host, next.Host) } @@ -87,8 +92,17 @@ func walkNextLinks[T any]( nextLinkOf func(*T) string, merge func(into, page *T), ) (*T, error) { + seen := map[string]bool{} link := nextLinkOf(first) - for pages := 0; link != "" && pages < maxPages; pages++ { + for link != "" { + // A repeated link, not just a self-referencing one, ends the walk: a + // two-page cycle would otherwise run to maxPages for no benefit. + if seen[link] || len(seen) >= maxPages { + log.Printf("[eval_api] stopped paging after %d pages; the listing may be incomplete", len(seen)) + break + } + seen[link] = true + body, err := c.followNextLink(ctx, link) if err != nil { return nil, err @@ -100,14 +114,7 @@ func walkNextLinks[T any]( } } merge(first, &page) - - nextLink := nextLinkOf(&page) - if nextLink == link { - // A link that points at the page it came from is the one shape that - // would otherwise spin until maxPages for no benefit. - break - } - link = nextLink + link = nextLinkOf(&page) } return first, nil } From a305b931b74d9fb5166f2146ece8df4f7c3dcae1 Mon Sep 17 00:00:00 2001 From: mohessie Date: Thu, 13 Aug 2026 21:05:56 +0300 Subject: [PATCH 220/320] Surface a failed drift check, and say what the output means Four follow-ups from the review and bug bash rounds. checkDatasetDrift tolerated a failed listing. That guard exists to catch a version someone else published, so a 403 or a timeout silently skipping it defeats the point; a listing we could not read is not evidence there was none. latestDatasetVersion now returns an error, and only a 404 or an empty listing still mean "nothing registered". This reverses a documented decision. The old test was named ToleratesAFailedListing and its comment argued an unreachable project should not fail the deploy. That trade is now the other way: fail closed. The test and its comment say so rather than being left stale. init reported "Created evals/azure.eval.yaml" over a file that already existed and was only appended to, which reads like it was overwritten. It now says "Updated" and marks the config line "(eval added)". run output list showed pass/fail per sample and no scores, so a bare pass and a perfect one looked identical and the only way to tell was the portal. It now carries a SCORE column with the sample's mean. --- .../internal/cmd/evalref_drift_test.go | 16 ++++++++---- .../azure.ai.evaluations/internal/cmd/init.go | 8 ++++-- .../internal/cmd/reconciler.go | 26 ++++++++++++++----- .../internal/cmd/run_output.go | 16 +++++++++++- .../internal/messages/messages.go | 18 +++++++++++++ 5 files changed, 69 insertions(+), 15 deletions(-) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evalref_drift_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evalref_drift_test.go index c21a40f1aa3..eb23ac81f81 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evalref_drift_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evalref_drift_test.go @@ -71,7 +71,9 @@ func TestCheckDatasetDriftAcceptsAMatch(t *testing.T) { func TestCheckDatasetDriftIgnoresAnEmptyListing(t *testing.T) { r := reconcilerListingVersions(t) require.NoError(t, r.checkDatasetDrift(context.Background(), "golden", "2.0")) - assert.Empty(t, r.latestDatasetVersion(context.Background(), "golden")) + latest, err := r.latestDatasetVersion(context.Background(), "golden") + require.NoError(t, err) + assert.Empty(t, latest) } // Only a newer version is someone else's work. An older one means this repo is @@ -82,8 +84,10 @@ func TestCheckDatasetDriftIgnoresAnOlderVersion(t *testing.T) { "a project behind this repo is not drift") } -// An unreachable project must not fail the deploy on drift it could not check. -func TestLatestDatasetVersionToleratesAFailedListing(t *testing.T) { +// A listing that failed is not evidence there was no newer version. The drift +// check exists to catch someone else's publish, so it fails closed: tolerating +// the error let a 403 or a timeout skip the guard silently. +func TestLatestDatasetVersionSurfacesAFailedListing(t *testing.T) { srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusInternalServerError) })) @@ -95,8 +99,10 @@ func TestLatestDatasetVersionToleratesAFailedListing(t *testing.T) { datasetClient: dataset_api.NewDatasetClientFromPipeline(srv.URL, pipeline), }} - assert.Empty(t, r.latestDatasetVersion(context.Background(), "golden")) - require.NoError(t, r.checkDatasetDrift(context.Background(), "golden", "2.0")) + _, err := r.latestDatasetVersion(context.Background(), "golden") + require.Error(t, err, "a listing we could not read is not proof there was no newer version") + require.Error(t, r.checkDatasetDrift(context.Background(), "golden", "2.0"), + "the drift guard fails closed rather than skipping silently") } // writeEvalYAML puts a configuration in a temp dir and returns the dir. diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go index 5e1e5905c24..abd26e8ec5a 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go @@ -101,6 +101,10 @@ func newInitCommand() *cobra.Command { } configPath := project.ResolveEvalConfigPath(path) + // Captured before the write: init merges into an existing config, so + // reporting it as created would claim a file it only added to. + _, configExistedErr := os.Stat(configPath) + configExisted := configExistedErr == nil cfg, err := project.OpenEvalConfig(path) if err != nil { return err @@ -223,8 +227,8 @@ func newInitCommand() *cobra.Command { fmt.Fprint(out, messages.JudgeModelDeployment(judgeModel)) } - fmt.Fprint(out, messages.CreatedHeading()) - fmt.Fprint(out, messages.CreatedConfigLine(filepath.ToSlash(configPath))) + fmt.Fprint(out, messages.ScaffoldHeading(configExisted)) + fmt.Fprint(out, messages.ScaffoldConfigLine(filepath.ToSlash(configPath), configExisted)) switch rootWiring { case wiringAdded: fmt.Fprint(out, messages.AddedServiceLine(rootConfigName, serviceName)) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go index 93bd2e2c797..1718e0ba4ff 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go @@ -195,7 +195,12 @@ func (r *evalReconciler) checkDatasetDrift( ctx context.Context, name, recorded string, ) error { - latest := r.latestDatasetVersion(ctx, name) + latest, err := r.latestDatasetVersion(ctx, name) + if err != nil { + // The whole point of this check is to catch a version published behind + // our back. A listing we could not read is not evidence there was none. + return err + } if latest == "" || latest == recorded { return nil } @@ -205,14 +210,21 @@ func (r *evalReconciler) checkDatasetDrift( return messages.DatasetDrifted(name, latest, recorded) } -// latestDatasetVersion reports the newest registered version, or empty when the -// dataset is unknown or the listing has not caught up. -func (r *evalReconciler) latestDatasetVersion(ctx context.Context, name string) string { +// latestDatasetVersion reports the newest registered version. A dataset the +// service does not know, and a listing that has not caught up, both report an +// empty version and no error; anything else is returned. +func (r *evalReconciler) latestDatasetVersion(ctx context.Context, name string) (string, error) { list, err := r.ec.datasetClient.ListDatasetVersions(ctx, name, ProjectEndpointAPIVersion) - if err != nil || list == nil || len(list.Value) == 0 { - return "" + if err != nil { + if dataset_api.IsNotFound(err) { + return "", nil + } + return "", err + } + if list == nil || len(list.Value) == 0 { + return "", nil } - return dataset_api.LatestVersion(list.Value) + return dataset_api.LatestVersion(list.Value), nil } // EnsureEvaluator publishes a new version when the local definition differs diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output.go index 7f498a3164d..a78025e9c0a 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output.go @@ -441,6 +441,19 @@ func formatScore(score eval_api.LenientFloat) string { return strconv.FormatFloat(float64(score), 'f', 2, 64) } +// meanScoreOf averages a sample's scores so the list can tell a bare pass from +// a strong one. Pass/fail alone sent anyone asking "how well?" to the portal. +func meanScoreOf(results []eval_api.OutputResult) string { + if len(results) == 0 { + return "-" + } + total := 0.0 + for _, r := range results { + total += float64(r.Score) + } + return strconv.FormatFloat(total/float64(len(results)), 'f', 2, 64) +} + func renderResults( w io.Writer, run *eval_api.OpenAIEvalRun, @@ -508,6 +521,7 @@ func renderResults( rows = append(rows, []string{ it.ID, strconv.Itoa(i + 1), + meanScoreOf(it.Results), truncate(verdicts, 40), truncate(reason, 44), }) @@ -515,7 +529,7 @@ func renderResults( // Only the first failure's reason fits a cell; `run output show` has // the rest. if err := emitTable(w, - []string{"ITEM", "SAMPLE", "FAILED EVALUATORS", "REASON"}, + []string{"ITEM", "SAMPLE", "SCORE", "FAILED EVALUATORS", "REASON"}, rows); err != nil { return err } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go b/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go index c11339f933f..18f5f81ef8c 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go @@ -1620,11 +1620,29 @@ func CreatedHeading() string { return "\nCreated\n" } +// ScaffoldHeading opens the list of what a scaffold wrote. `init` appends to an +// existing configuration rather than replacing it, and a reader who sees +// "Created" over a file they already had reasonably fears it was overwritten. +func ScaffoldHeading(existed bool) string { + if existed { + return "\nUpdated\n" + } + return CreatedHeading() +} + // CreatedConfigLine names the configuration a scaffold wrote. func CreatedConfigLine(configPath string) string { return fmt.Sprintf(" %-33s evaluation configuration\n", configPath) } +// ScaffoldConfigLine names the configuration a scaffold wrote or added to. +func ScaffoldConfigLine(configPath string, existed bool) string { + if existed { + return fmt.Sprintf(" %-33s evaluation configuration (eval added)\n", configPath) + } + return CreatedConfigLine(configPath) +} + // AddedServiceLine reports the eval service being added to the root config. func AddedServiceLine(rootConfig, service string) string { return fmt.Sprintf(" %-33s added service '%s'\n", rootConfig, service) From cd4d5670dae0827a7d23f2ba39b019b5b2bde467 Mon Sep 17 00:00:00 2001 From: Mohamed Hessien Date: Thu, 13 Aug 2026 21:23:03 +0300 Subject: [PATCH 221/320] Pin the eval paginator behaviour the dataset extension already pins The pagination fixes went in unguarded, which is the same asymmetry that let the two copies drift in the first place. Three tests, each checked by reverting its fix: a relative nextLink is followed rather than refused; a link resolving to another host is refused and that host is never contacted; and a two-page cycle ends the walk. Without the cycle fix the last one runs long enough to blow a 60s test timeout, so it was a hang, not a slow path. --- .../pkg/dataset_api/paging_edge_test.go | 88 +++++++++++++++++++ 1 file changed, 88 insertions(+) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/paging_edge_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/paging_edge_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/paging_edge_test.go new file mode 100644 index 00000000000..390af92b3ff --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/paging_edge_test.go @@ -0,0 +1,88 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package dataset_api + +import ( + "fmt" + "net/http" + "net/http/httptest" + "sync/atomic" + "testing" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func pagingEdgeClient(t *testing.T, h http.HandlerFunc) (*DatasetClient, *httptest.Server) { + t.Helper() + srv := httptest.NewServer(h) + t.Cleanup(srv.Close) + return NewDatasetClientFromPipeline( + srv.URL, runtime.NewPipeline("test", "v1", runtime.PipelineOptions{}, nil)), srv +} + +// A nextLink is allowed to be relative, and a relative one has no host or +// scheme of its own. Comparing it to the endpoint before resolving refused a +// legitimate link and turned a working listing into a hard failure. +func TestListDatasetsFollowsARelativeNextLink(t *testing.T) { + c, _ := pagingEdgeClient(t, func(w http.ResponseWriter, r *http.Request) { + if r.URL.Query().Get("page") == "" { + fmt.Fprint(w, `{"value":[{"name":"one"}],"nextLink":"/datasets?page=2"}`) + return + } + fmt.Fprint(w, `{"value":[{"name":"two"}]}`) + }) + + list, err := c.ListDatasets(t.Context(), testAPIVersion) + require.NoError(t, err, "a relative nextLink must be followed, not refused") + require.NotNil(t, list) + require.Len(t, list.Value, 2) + assert.Equal(t, "two", list.Value[1].Name) +} + +// A relative link still has to stay on the endpoint. Resolving must not become +// a way to reach another host by writing a protocol-relative link. +func TestListDatasetsRefusesAProtocolRelativeLinkToAnotherHost(t *testing.T) { + var elsewhereHits int32 + elsewhere := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + atomic.AddInt32(&elsewhereHits, 1) + fmt.Fprint(w, `{"value":[{"name":"leaked"}]}`) + })) + t.Cleanup(elsewhere.Close) + + // "//host/path" resolves to the same scheme on a different host. + c, _ := pagingEdgeClient(t, func(w http.ResponseWriter, r *http.Request) { + fmt.Fprintf(w, `{"value":[{"name":"one"}],"nextLink":"//%s/datasets"}`, + elsewhere.Listener.Addr().String()) + }) + + _, err := c.ListDatasets(t.Context(), testAPIVersion) + + require.Error(t, err, "a link resolving to another host must be refused") + assert.Zero(t, atomic.LoadInt32(&elsewhereHits), "the other host must never be contacted") +} + +// A cycle longer than one hop used to run to maxPages, because only a link +// pointing at the page it came from ended the walk. +func TestListDatasetsStopsOnATwoPageCycle(t *testing.T) { + var hits int32 + var base string + c, srv := pagingEdgeClient(t, func(w http.ResponseWriter, r *http.Request) { + atomic.AddInt32(&hits, 1) + // a -> b -> a, which never repeats the immediately previous link. + if r.URL.Query().Get("page") == "b" { + fmt.Fprintf(w, `{"value":[{"name":"b"}],"nextLink":%q}`, base+"/datasets?page=a") + return + } + fmt.Fprintf(w, `{"value":[{"name":"a"}],"nextLink":%q}`, base+"/datasets?page=b") + }) + base = srv.URL + + list, err := c.ListDatasets(t.Context(), testAPIVersion) + require.NoError(t, err, "a cycle ends the walk rather than failing the command") + require.NotNil(t, list) + assert.LessOrEqual(t, atomic.LoadInt32(&hits), int32(4), + "a two-page cycle must stop quickly, not run to maxPages") +} From 686ce5d26a52b101d008cd75a251ced4b104c94a Mon Sep 17 00:00:00 2001 From: Mohamed Hessien Date: Thu, 13 Aug 2026 22:12:59 +0300 Subject: [PATCH 222/320] Pin the eval_api paginator too This is the paginator that lists evals, runs and evaluators, so it is the more exercised of the two and it had no cover for the three fixes. Same three cases as dataset_api, each checked by removing the guard it tests: a relative nextLink is followed; a protocol-relative link resolving to another host is refused and that host is never contacted, so resolving cannot become a bypass; and a two-page cycle ends the walk rather than running to maxPages, which without the guard blows a 45s timeout. --- .../internal/pkg/eval_api/paging_edge_test.go | 93 +++++++++++++++++++ 1 file changed, 93 insertions(+) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/paging_edge_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/paging_edge_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/paging_edge_test.go new file mode 100644 index 00000000000..a47503bfd4c --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/paging_edge_test.go @@ -0,0 +1,93 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package eval_api + +import ( + "fmt" + "net/http" + "net/http/httptest" + "sync/atomic" + "testing" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// clientAndServer is clientServing plus the server, for tests that need to +// build an absolute link back to it. +func clientAndServer(t *testing.T, handler http.HandlerFunc) (*EvalClient, *httptest.Server) { + t.Helper() + srv := httptest.NewServer(handler) + t.Cleanup(srv.Close) + + pipeline := runtime.NewPipeline("test", "v1", runtime.PipelineOptions{}, + &policy.ClientOptions{Retry: policy.RetryOptions{MaxRetries: -1}}) + return NewEvalClientFromPipeline(srv.URL, pipeline), srv +} + +// A nextLink is allowed to be relative, and a relative one has no host or +// scheme of its own. Comparing it to the endpoint before resolving refused a +// legitimate link, which turned a working listing into a hard failure. +func TestListEvaluatorVersionsFollowsARelativeNextLink(t *testing.T) { + c, _ := clientAndServer(t, func(w http.ResponseWriter, r *http.Request) { + if r.URL.Query().Get("page") == "" { + fmt.Fprint(w, `{"value":[{"name":"one"}],"nextLink":"/evaluators/e/versions?page=2"}`) + return + } + fmt.Fprint(w, `{"value":[{"name":"two"}]}`) + }) + + list, err := c.ListEvaluatorVersions(t.Context(), "e", "v1") + require.NoError(t, err, "a relative nextLink must be followed, not refused") + require.NotNil(t, list) + require.Len(t, list.Value, 2, "the second page has to be merged in") +} + +// Resolving relative links must not become a way to reach another host: a +// protocol-relative link keeps the scheme and swaps the host, and this client +// sends an Authorization header. +func TestListEvaluatorVersionsRefusesALinkResolvingToAnotherHost(t *testing.T) { + var elsewhereHits int32 + elsewhere := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + atomic.AddInt32(&elsewhereHits, 1) + fmt.Fprint(w, `{"value":[{"name":"leaked"}]}`) + })) + t.Cleanup(elsewhere.Close) + + c, _ := clientAndServer(t, func(w http.ResponseWriter, r *http.Request) { + fmt.Fprintf(w, `{"value":[{"name":"one"}],"nextLink":"//%s/evaluators"}`, + elsewhere.Listener.Addr().String()) + }) + + _, err := c.ListEvaluatorVersions(t.Context(), "e", "v1") + + require.Error(t, err, "a link resolving to another host must be refused") + assert.Zero(t, atomic.LoadInt32(&elsewhereHits), + "the token must never be sent to the other host") +} + +// A cycle longer than one hop used to run to maxPages, because only a link +// pointing at the page it came from ended the walk. +func TestListEvaluatorVersionsStopsOnATwoPageCycle(t *testing.T) { + var hits int32 + var base string + c, srv := clientAndServer(t, func(w http.ResponseWriter, r *http.Request) { + atomic.AddInt32(&hits, 1) + // a -> b -> a, so no link ever repeats the one immediately before it. + if r.URL.Query().Get("page") == "b" { + fmt.Fprintf(w, `{"value":[{"name":"b"}],"nextLink":%q}`, base+"/evaluators?page=a") + return + } + fmt.Fprintf(w, `{"value":[{"name":"a"}],"nextLink":%q}`, base+"/evaluators?page=b") + }) + base = srv.URL + + list, err := c.ListEvaluatorVersions(t.Context(), "e", "v1") + require.NoError(t, err, "a cycle ends the walk rather than failing the command") + require.NotNil(t, list) + assert.LessOrEqual(t, atomic.LoadInt32(&hits), int32(4), + "a two-page cycle must stop quickly, not run to maxPages") +} From 8f4773b8381160e5ce70c3d4fc24830a44446e7b Mon Sep 17 00:00:00 2001 From: mohessie Date: Sun, 16 Aug 2026 16:53:22 +0300 Subject: [PATCH 223/320] Refuse an update only when the service confirms the name is unknown Same fix as the dataset extension, applied to this copy of the gate. The dataset path reads an eventually consistent version listing and cannot prove absence, so it now passes whether the service actually said 404; the evaluator path reads a point endpoint and returns early on any non-404, so reaching the gate already means the answer is authoritative and it passes true. Fifth time a change has had to be made twice across the duplicated dataset_api surface. --- .../azure.ai.evaluations/internal/cmd/dataset.go | 15 ++++++++++----- .../internal/cmd/dataset_version_probe_test.go | 12 ++++++++++++ .../internal/cmd/evaluator.go | 15 ++++++++++++--- 3 files changed, 34 insertions(+), 8 deletions(-) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset.go index 39544914bcb..58895a04c5b 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset.go @@ -90,15 +90,15 @@ func newDatasetWriteCommand(verb, short string) *cobra.Command { } defer ec.Close() - existing, err := ec.datasetClient.ListDatasetVersions( + existing, listErr := ec.datasetClient.ListDatasetVersions( ctx, name, ProjectEndpointAPIVersion, ) - exists := err == nil && existing != nil && len(existing.Value) > 0 + exists := listErr == nil && existing != nil && len(existing.Value) > 0 if !exists { // The version listing lags a publish, so a `create` followed by // an `update` was told the dataset it had just made does not - // exist. A direct read settles it: point reads go consistent - // immediately. + // exist. A direct read usually settles it, catching up sooner + // than the listing does. for _, v := range firstDatasetVersions { if _, err := ec.datasetClient.GetDataset( ctx, name, v, ProjectEndpointAPIVersion, @@ -108,7 +108,12 @@ func newDatasetWriteCommand(verb, short string) *cobra.Command { } } } - if err := checkAssetExistence(verb, "dataset", name, exists); err != nil { + // Only an outright 404 proves the name is unknown. An empty 200 does + // not: an unknown dataset and a listing that has not caught up are + // indistinguishable. + if err := checkAssetExistence( + verb, "dataset", name, exists, dataset_api.IsNotFound(listErr), + ); err != nil { return err } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset_version_probe_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset_version_probe_test.go index 79908be2f60..fa238574328 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset_version_probe_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset_version_probe_test.go @@ -22,3 +22,15 @@ func TestFirstDatasetVersionsCoverWhatThisCLIPublishes(t *testing.T) { assert.Contains(t, firstDatasetVersions, "1", "a generation job, the SDK or the portal can register a plain 1") } + +// The probe still cannot prove absence -- it only ever proves existence, and a +// dataset whose early versions were deleted has none of them left to find. So +// an absence the service never confirmed has to publish rather than refuse, +// otherwise the caller is sent to `create`, which fails in turn once the +// listing catches up and reports the name already taken. +func TestCheckAssetExistenceLetsAnUnprovableAbsenceThrough(t *testing.T) { + assert.NoError(t, checkAssetExistence("update", "dataset", "x", false, false)) + + err := checkAssetExistence("update", "dataset", "x", false, true) + assert.Error(t, err, "a 404 is the service saying the name is unknown, which still refuses") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator.go index 39b3330aea5..dc8679e7ec5 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator.go @@ -96,7 +96,9 @@ func newEvaluatorWriteCommand(verb, short string) *cobra.Command { if readErr != nil && !eval_api.IsNotFound(readErr) { return messages.CheckingEvaluatorExists(name, readErr) } - if err := checkAssetExistence(verb, "evaluator", name, readErr == nil); err != nil { + // A non-404 already returned above, so reaching here means the read + // either found the evaluator or the service said it is unknown. + if err := checkAssetExistence(verb, "evaluator", name, readErr == nil, true); err != nil { return err } @@ -128,11 +130,18 @@ func newEvaluatorWriteCommand(verb, short string) *cobra.Command { } // checkAssetExistence enforces the one difference between create and update. -func checkAssetExistence(verb, kind, name string, exists bool) error { +// +// absenceCertain separates "the service says this name is unknown" from "nothing +// came back", and only the former refuses an update. A caller reading an +// eventually consistent version listing cannot prove absence, so an update +// issued moments after a create would otherwise be refused for a dataset that +// plainly exists -- and sent to `create`, which fails in turn once the listing +// catches up. Callers that read a point endpoint can prove it and pass true. +func checkAssetExistence(verb, kind, name string, exists, absenceCertain bool) error { switch { case verb == "create" && exists: return messages.AssetAlreadyExists(kind, name) - case verb == "update" && !exists: + case verb == "update" && !exists && absenceCertain: return messages.AssetDoesNotExist(kind, name) } return nil From 0206aaaf49e76cd784944d920c7dae138677f7fa Mon Sep 17 00:00:00 2001 From: mohessie Date: Sun, 16 Aug 2026 17:58:13 +0300 Subject: [PATCH 224/320] Let LatestVersion fall back to the last entry as documented VersionOrder returns -1 for anything it cannot order, so a sentinel of -2.0 let the first unorderable version become the running best and the documented fallback to the last entry never ran. The dataset extension was fixed for this; this copy was not, and neither was its test -- the sixth time a change has had to be made twice across the duplicated dataset_api surface. Found by a review agent comparing the two copies. --- .../internal/pkg/dataset_api/list.go | 5 ++++- .../internal/pkg/dataset_api/version_test.go | 13 +++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/list.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/list.go index 28c4e6e459c..805902736a4 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/list.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/list.go @@ -98,7 +98,10 @@ func VersionGreater(a, b string) bool { // last entry when none of the versions can be ordered. func LatestVersion(datasets []Dataset) string { best := "" - bestOrder := -2.0 + // VersionOrder returns -1 for anything it cannot order, so the sentinel has + // to be -1 rather than lower: below it, the first unorderable version becomes + // the running best and the fallback below never runs. + bestOrder := -1.0 for _, d := range datasets { if o := VersionOrder(d.Version); o > bestOrder { bestOrder, best = o, d.Version diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/version_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/version_test.go index 052a63ae504..f4817100c51 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/version_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/version_test.go @@ -66,3 +66,16 @@ func TestLatestVersionOrdersNumerically(t *testing.T) { t.Fatal("LatestVersion(nil) should be empty") } } + +// LatestVersion documents a fallback to the last entry when nothing can be +// ordered. That fallback only runs if an unorderable version never becomes the +// running best, which a sentinel below -1 quietly prevented. +func TestLatestVersionFallsBackToTheLastEntryWhenNoneAreOrderable(t *testing.T) { + got := LatestVersion([]Dataset{{Version: "alpha"}, {Version: "beta"}, {Version: "gamma"}}) + require.Equal(t, "gamma", got, "with nothing orderable the service's last entry wins") +} + +func TestLatestVersionPrefersAnOrderableVersionOverAnUnorderableOne(t *testing.T) { + require.Equal(t, "2.0", LatestVersion([]Dataset{{Version: "alpha"}, {Version: "2.0"}})) + require.Equal(t, "2.0", LatestVersion([]Dataset{{Version: "2.0"}, {Version: "alpha"}})) +} From 98c114443064c62f164585c15d3b7a8e26cb7f34 Mon Sep 17 00:00:00 2001 From: mohessie Date: Sun, 16 Aug 2026 18:06:35 +0300 Subject: [PATCH 225/320] Match the extension's case-insensitive .jsonl scan in the API layer Same one-word divergence in this copy of dataset_api. --- .../azure.ai.evaluations/internal/pkg/dataset_api/models.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/models.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/models.go index 3cfbd1dba4a..2a38dbc9185 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/models.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/models.go @@ -219,7 +219,7 @@ func ReadFirstJSONLFile(path string) (string, error) { if e.IsDir() { continue } - if filepath.Ext(e.Name()) == ".jsonl" { + if strings.EqualFold(filepath.Ext(e.Name()), ".jsonl") { data, err := os.ReadFile(filepath.Join(dir, e.Name())) //nolint:gosec // local artifact path if err != nil { return "", messages.ReadingPath(e.Name(), err) From e00359baae01f2d1cae2d9198293af5e4a1287a6 Mon Sep 17 00:00:00 2001 From: mohessie Date: Sun, 16 Aug 2026 18:15:35 +0300 Subject: [PATCH 226/320] Add unorderable to the extension's cspell word list Used in the LatestVersion comment explaining why the sentinel has to be -1. --- cli/azd/extensions/azure.ai.evaluations/cspell.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/cli/azd/extensions/azure.ai.evaluations/cspell.yaml b/cli/azd/extensions/azure.ai.evaluations/cspell.yaml index 9aecb53c663..a328b6bc634 100644 --- a/cli/azd/extensions/azure.ai.evaluations/cspell.yaml +++ b/cli/azd/extensions/azure.ai.evaluations/cspell.yaml @@ -36,6 +36,7 @@ words: - undeployed - undoable - unbuildable + - unorderable - unpassed - unscored - Unparseable From 8ef1c9a9ec905650b72eaf8b9815e78d396bfb5b Mon Sep 17 00:00:00 2001 From: mohessie Date: Sun, 16 Aug 2026 18:42:38 +0300 Subject: [PATCH 227/320] Validate JSONL rows on every upload path, not just the reconciler The reconciler checked rows before publishing, but dataset create and dataset update call UploadNextVersion directly and never went through it. Upload does not parse rows either, so a malformed line registered a version that looked healthy and only failed in the run that scored it, at a line number in a file nobody was looking at. The check now lives in jsonlContent, which every upload shares, matching the dataset extension where it already did. Ninth divergence found between the two copies of dataset_api, this one in the other direction. Reported by a review agent diffing them. --- .../pkg/dataset_api/jsonl_read_test.go | 26 +++++++++++++++ .../internal/pkg/dataset_api/models.go | 32 +++++++++++++++++++ 2 files changed, 58 insertions(+) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/jsonl_read_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/jsonl_read_test.go index 5e1e80a48b0..0b8e11661ad 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/jsonl_read_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/jsonl_read_test.go @@ -65,3 +65,29 @@ func TestReadFirstJSONLFile_RefusesAnEmptyFile(t *testing.T) { require.Error(t, err) assert.Contains(t, err.Error(), "no rows") } + +// The reconciler validates rows before it publishes, but `dataset create` and +// `dataset update` do not go through it. Upload does not parse rows either, so +// without this a malformed line registers a version that looks healthy and only +// fails in the run that reads it. +func TestReadFirstJSONLFileRefusesAMalformedRow(t *testing.T) { + dir := t.TempDir() + named := filepath.Join(dir, "rows.jsonl") + body := "{\"query\":\"ok\"}\n{not json}\n" + require.NoError(t, os.WriteFile(named, []byte(body), 0o600)) + + _, err := ReadFirstJSONLFile(named) + + require.Error(t, err) + assert.Contains(t, err.Error(), "2", "the error has to name the line that is wrong") +} + +func TestReadFirstJSONLFileRefusesAnEmptyRow(t *testing.T) { + dir := t.TempDir() + named := filepath.Join(dir, "rows.jsonl") + require.NoError(t, os.WriteFile(named, []byte("{\"query\":\"ok\"}\n{}\n"), 0o600)) + + _, err := ReadFirstJSONLFile(named) + + require.Error(t, err) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/models.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/models.go index 2a38dbc9185..947477266d7 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/models.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/models.go @@ -4,7 +4,9 @@ package dataset_api import ( + "bufio" "bytes" + "encoding/json" "math" "os" "path/filepath" @@ -242,5 +244,35 @@ func jsonlContent(name string, data []byte) (string, error) { if strings.TrimSpace(string(data)) == "" { return "", messages.DatasetFileHasNoRows(name) } + if err := validateJSONLRows(name, data); err != nil { + return "", err + } return string(data), nil } + +// validateJSONLRows refuses a file the service would happily store. +// +// Upload does not parse the rows, so one malformed line registers a version +// that looks healthy and only fails in the run that reads it. The reconciler +// checks this too, but `dataset create` and `dataset update` do not go through +// it, so the check belongs on the path every upload shares. +func validateJSONLRows(name string, data []byte) error { + scanner := bufio.NewScanner(bytes.NewReader(data)) + // A row carrying a whole conversation runs well past the 64KB default. + scanner.Buffer(make([]byte, 0, 64*1024), 8*1024*1024) + + for line := 1; scanner.Scan(); line++ { + text := strings.TrimSpace(scanner.Text()) + if text == "" { + continue + } + var row map[string]any + if err := json.Unmarshal([]byte(text), &row); err != nil { + return messages.JSONLRowInvalid(name, line, err) + } + if len(row) == 0 { + return messages.JSONLRowEmpty(name, line) + } + } + return scanner.Err() +} From 7d1a6deedc9d4f91b69a7f0884e358b5d81a6034 Mon Sep 17 00:00:00 2001 From: mohessie Date: Sun, 16 Aug 2026 21:12:31 +0300 Subject: [PATCH 228/320] Render an eval's Created time instead of printing a float The service sends epoch seconds for this field and RFC3339 elsewhere, so the model types it as any. fmt.Sprint on the float64 that lands there printed Created 1.78690163e+09, which says nothing to a reader. timestampString already normalizes both shapes and every other call site uses it. Found by a bug bash agent against the published feed. --- .../azure.ai.evaluations/internal/cmd/eval_group.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/eval_group.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/eval_group.go index 9e8ae587b84..80d8e342ad0 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/eval_group.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/eval_group.go @@ -227,7 +227,10 @@ func newEvalShowCommand() *cobra.Command { return emitDetail(cmd.OutOrStdout(), []field{ {"Id", group.ID}, {"Name", group.Name}, - {"Created", fmt.Sprint(group.CreatedAt)}, + // CreatedAt is `any` because the service sends epoch seconds here + // and RFC3339 elsewhere; fmt.Sprint on the former prints a float + // in scientific notation. + {"Created", timestampString(group.CreatedAt)}, {"Created By", group.CreatedBy}, }) }, From e08d9d59fa0046ea800253e71f129651264b45ec Mon Sep 17 00:00:00 2001 From: mohessie Date: Sun, 16 Aug 2026 21:20:59 +0300 Subject: [PATCH 229/320] Stop config validation stranding commands that only need to look a name up resolveEvalRef ran the full config validation on the way to turning --eval into an id, so two evals differing only in substance refused every run command -- including read-only ones that had already been told which eval they meant, like un list --eval . The only way out was to hand-edit evals/azure.eval.yaml, which the error did not mention. That clash is about what happens once deployed, where the environment records an id against each eval's substance. A lookup by name is unambiguous whatever the substance, so it now runs the checks a lookup depends on and leaves the rest to create and deploy, which still refuse. Also emits a bare array from un output list -o json, as every other list does. Wrapping the rows beside the run made this the one listing a script could not iterate, and it failed silently: the loop walked the two keys instead. The run is what un show answers. Both found by a bug bash agent against the published feed. --- .../internal/cmd/evalref.go | 2 +- .../internal/cmd/run_output.go | 9 ++++--- .../internal/project/eval_config.go | 20 +++++++++++++++ .../internal/project/eval_config_test.go | 25 +++++++++++++++++++ 4 files changed, 52 insertions(+), 4 deletions(-) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evalref.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evalref.go index 3ba974d96b1..4aefaf35a6b 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evalref.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evalref.go @@ -49,7 +49,7 @@ func (ec *evalContext) resolveEvalRef( } if cfg != nil { - if err := cfg.Validate(); err != nil { + if err := cfg.ValidateForLookup(); err != nil { return evalRef{}, err } eval, err := cfg.Eval(nameOrID) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output.go index a78025e9c0a..259ffff1b5e 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output.go @@ -85,17 +85,20 @@ func newRunOutputListCommand() *cobra.Command { rows = kept } - payload := map[string]any{"run": run, "output_items": rows} + // A bare array, as every other list emits. Wrapping the rows beside + // the run made `-o json` the one listing a script could not iterate, + // and it failed silently: the loop walked the two keys instead. The + // run itself is what `run show` answers. if outFile != "" { f, err := os.Create(outFile) if err != nil { return messages.Creating(outFile, err) } defer f.Close() - return emitJSON(f, payload) + return emitJSON(f, rows) } if isJSON(cmd) { - return emitJSON(cmd.OutOrStdout(), payload) + return emitJSON(cmd.OutOrStdout(), rows) } return renderResults(cmd.OutOrStdout(), run, rows, failedOnly) }, diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go index e00cf4e12dc..fb9abc2aee3 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go @@ -207,6 +207,22 @@ func (c *EvalConfig) LocalDatasets() []DatasetDecl { // Validate checks the invariants the provider relies on before it calls the // service, so failures surface as config errors rather than opaque 4xx. func (c *EvalConfig) Validate() error { + return c.validate(true) +} + +// ValidateForLookup checks only what resolving a declaration by name depends +// on. +// +// Two evals that differ only in substance still resolve unambiguously by name, +// and that clash only matters to something about to deploy. Enforcing it on the +// way to a lookup stranded commands that had already been told which eval they +// meant -- `run list --eval ` refused to list anything, and the way out +// was to hand-edit the config, which the error did not say. +func (c *EvalConfig) ValidateForLookup() error { + return c.validate(false) +} + +func (c *EvalConfig) validate(deploying bool) error { if err := c.validateCatalogs(); err != nil { return err } @@ -229,6 +245,10 @@ func (c *EvalConfig) Validate() error { return err } + if !deploying { + continue + } + // Two evals that differ only by name are indistinguishable once // deployed: the environment records an id against each eval's substance // so a renamed declaration can find what it already deployed, and a diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_test.go index b83440373ea..15a33ae3d96 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_test.go @@ -320,6 +320,31 @@ func TestValidate_Rejects(t *testing.T) { } } +// Two evals that differ only in substance still resolve unambiguously by name, +// and the clash only matters to something about to deploy. Enforcing it on the +// way to a lookup stranded `run list --eval `, which had already been +// told which eval it meant, behind an error whose only escape was hand-editing +// the config. +func TestValidateForLookupAllowsWhatOnlyDeployingCannotTellApart(t *testing.T) { + body := "evals:\n - name: a\n evaluators:\n - evaluator: builtin.relevance\n" + + " - name: b\n evaluators:\n - evaluator: builtin.relevance\n" + cfg := loadFromString(t, body) + + require.NoError(t, cfg.ValidateForLookup()) + require.Error(t, cfg.Validate(), "deploying still cannot tell the two apart") +} + +// Lookup still depends on names being unique, so that check stays. +func TestValidateForLookupStillRefusesADuplicateName(t *testing.T) { + body := "evals:\n - name: a\n evaluators:\n - evaluator: builtin.relevance\n" + + " - name: a\n evaluators:\n - evaluator: builtin.coherence\n" + + err := loadFromString(t, body).ValidateForLookup() + + require.Error(t, err) + require.Contains(t, err.Error(), "duplicate") +} + // outputDir accepts a directory or an explicit file path. func TestArtifactPath(t *testing.T) { cases := []struct { From 0f80154f685988d62fe07a39e13e197d4149f7d5 Mon Sep 17 00:00:00 2001 From: mohessie Date: Sun, 16 Aug 2026 22:06:01 +0300 Subject: [PATCH 230/320] Say how to declare an eval when the catalog has none generate writes the dataset and evaluator it made into the catalog but declares no eval, so a create run straight afterwards hit t least one eval is required -- true, and no help at all about which command supplies one. It now names init and says what generate did and did not add. --- .../azure.ai.evaluations/internal/messages/messages.go | 8 +++++++- .../internal/project/eval_config_test.go | 8 +++++--- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go b/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go index 18f5f81ef8c..1d48fa717ce 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go @@ -1705,8 +1705,14 @@ func EvalNotDeclared(eval string, names []string) error { } // AtLeastOneEvalRequired reports a configuration that declares no eval. +// +// `generate` writes the dataset and evaluator it made into the catalog but +// declares no eval, so a `create` run straight afterwards lands here with +// nothing to act on. func AtLeastOneEvalRequired() error { - return errors.New("at least one eval is required") + return errors.New( + "no eval is declared; `azd ai eval init` declares one. " + + "`generate` only adds the dataset and evaluator it made") } // EvalNameRequired reports an eval entry with no name. diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_test.go index 15a33ae3d96..96898445f47 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_test.go @@ -250,9 +250,11 @@ func TestValidate_Rejects(t *testing.T) { wantErr: "cannot be set with `source`", }, { - name: "no evals", - body: "datasets:\n - name: d\n", - wantErr: "at least one eval is required", + name: "no evals", + body: "datasets:\n - name: d\n", + // A catalog with no eval is what `generate` leaves behind, so the + // error has to name the command that declares one. + wantErr: "azd ai eval init", }, { name: "duplicate eval", From bbac3a0d9207a60e953ea821df41eb998916381b Mon Sep 17 00:00:00 2001 From: mohessie Date: Sun, 16 Aug 2026 22:10:36 +0300 Subject: [PATCH 231/320] Confirm a pinned dataset version still exists before reusing it When the local file was unchanged and the declaration pinned a version, reconciliation returned that version without asking the service anything. Deleting the pinned version therefore left create reporting it unchanged while ersions list showed nothing and the eval pointed at a version that was gone. A pin settles which version to use, not whether it is still there, so the pinned branch now reads it the same way the already-registered branch above does. Only a confirmed 404 refuses; any other read failure leaves the pin alone rather than turning a transient error into a failed deploy. Not covered by a unit test: the branch is only reachable once getEnvValue returns a recorded fingerprint, which needs a live azd client. Found by a bug bash agent against the published feed. --- .../azure.ai.evaluations/internal/cmd/reconciler.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go index 1718e0ba4ff..12076b32e8c 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go @@ -102,6 +102,16 @@ func (r *evalReconciler) EnsureDataset( // it settles the question and the check does not apply. if version := r.ec.getEnvValue(ctx, versionKey("dataset", decl.Name)); version != "" { if decl.Version != "" { + // A pin settles which version to use, not whether it is still + // there. Skipping the service entirely let a deleted version + // report as unchanged while the eval pointed at nothing. Only a + // confirmed 404 refuses: anything else leaves the pin alone + // rather than failing a deploy on a transient read. + if _, err := r.ec.datasetClient.GetDataset( + ctx, decl.Name, decl.Version, ProjectEndpointAPIVersion, + ); err != nil && dataset_api.IsNotFound(err) { + return "", false, messages.DatasetVersionNotFoundWithHint(decl.Name, decl.Version) + } return decl.Version, false, nil } if err := r.checkDatasetDrift(ctx, decl.Name, version); err != nil { From 9ee2cbfb69b50fbf95d2eba55d8a7dd749d4903c Mon Sep 17 00:00:00 2001 From: mohessie Date: Sun, 16 Aug 2026 23:03:10 +0300 Subject: [PATCH 232/320] Make the reconciler's recorded-state branches reachable from a test These branches only run once a previous deploy recorded a fingerprint, and getEnvValue answers nothing without an azd client, so none of them had ever executed under test -- 38 call sites depend on it. The agents extension already solves this by serving the environment over gRPC the way azd does, so this borrows that: a small in-memory EnvironmentServiceServer, a real grpc.Server on an ephemeral port, and the actual client pointed at it. That covers the pinned-version check three ways: a version the service no longer has is refused, one that is still there is reused unchanged, and a read that fails for any other reason leaves the pin alone rather than breaking a deploy. The harness found a divergence on its first run: this extension's version-not-found hint still pointed at dataset list, which lists datasets rather than versions. The dataset extension's copy was corrected earlier today and this one was missed -- the tenth divergence between the two. --- .../internal/cmd/reconciler_env_test.go | 170 ++++++++++++++++++ .../internal/cmd/run_output.go | 4 +- .../internal/cmd/run_output_write_test.go | 4 +- .../internal/messages/messages.go | 2 +- .../tests/cli/run_output_test.go | 2 +- 5 files changed, 177 insertions(+), 5 deletions(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler_env_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler_env_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler_env_test.go new file mode 100644 index 00000000000..5f053cc9226 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler_env_test.go @@ -0,0 +1,170 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "context" + "encoding/json" + "net" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + + "azureaieval/internal/pkg/dataset_api" + "azureaieval/internal/project" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime" + "github.com/azure/azure-dev/cli/azd/pkg/azdext" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "google.golang.org/grpc" +) + +// testEnvServer is an azd environment held in memory, so reconciliation paths +// that only run once something was recorded at the last deploy are reachable +// from a test. Without it, getEnvValue answers "" for everything and those +// branches never execute. +type testEnvServer struct { + azdext.UnimplementedEnvironmentServiceServer + values map[string]string +} + +func (s *testEnvServer) GetValue( + _ context.Context, req *azdext.GetEnvRequest, +) (*azdext.KeyValueResponse, error) { + return &azdext.KeyValueResponse{Value: s.values[req.Key]}, nil +} + +func (s *testEnvServer) SetValue( + _ context.Context, req *azdext.SetEnvRequest, +) (*azdext.EmptyResponse, error) { + if s.values == nil { + s.values = map[string]string{} + } + s.values[req.Key] = req.Value + return &azdext.EmptyResponse{}, nil +} + +// newTestAzdClient serves the environment over gRPC the way azd itself does, +// rather than faking the accessor, so the client code under test is the real one. +func newTestAzdClient(t *testing.T, env *testEnvServer) *azdext.AzdClient { + t.Helper() + + server := grpc.NewServer() + azdext.RegisterEnvironmentServiceServer(server, env) + + listener, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + go func() { _ = server.Serve(listener) }() + t.Cleanup(func() { + server.Stop() + _ = listener.Close() + }) + + client, err := azdext.NewAzdClient(azdext.WithAddress(listener.Addr().String())) + require.NoError(t, err) + t.Cleanup(func() { client.Close() }) + + return client +} + +// pinnedDatasetReconciler builds a reconciler whose environment already holds +// what a previous deploy recorded for a dataset, and whose service answers a +// version read with the given status. +func pinnedDatasetReconciler( + t *testing.T, name, version string, versionStatus int, +) (*evalReconciler, string) { + t.Helper() + + dir := t.TempDir() + localPath := filepath.Join(dir, name+".jsonl") + require.NoError(t, os.WriteFile(localPath, []byte("{\"query\":\"hi\"}\n"), 0o600)) + + digest, err := project.Fingerprint(localPath) + require.NoError(t, err) + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if !strings.Contains(r.URL.Path, "/versions/") { + w.WriteHeader(http.StatusNotFound) + return + } + if versionStatus != http.StatusOK { + w.WriteHeader(versionStatus) + return + } + w.Header().Set("Content-Type", "application/json") + require.NoError(t, json.NewEncoder(w).Encode(map[string]any{ + "name": name, "version": version, + })) + })) + t.Cleanup(srv.Close) + + env := &testEnvServer{values: map[string]string{ + project.FingerprintKey("dataset", name): digest, + versionKey("dataset", name): version, + }} + + pipeline := runtime.NewPipeline("test", "v1", runtime.PipelineOptions{}, + &policy.ClientOptions{Retry: policy.RetryOptions{MaxRetries: -1}}) + + return &evalReconciler{ec: &evalContext{ + azdClient: newTestAzdClient(t, env), + envName: "test", + datasetClient: dataset_api.NewDatasetClientFromPipeline(srv.URL, pipeline), + }}, localPath +} + +// A pin settles which version to use, not whether it is still there. Reusing it +// unread let a deleted version report as unchanged while the eval pointed at +// nothing, which is what `create` did straight after `dataset delete`. +func TestEnsureDatasetRefusesAPinnedVersionTheServiceNoLongerHas(t *testing.T) { + r, localPath := pinnedDatasetReconciler(t, "golden", "1.0", http.StatusNotFound) + + _, _, err := r.EnsureDataset( + context.Background(), + project.DatasetDecl{Name: "golden", Version: "1.0"}, + localPath, + ) + + require.Error(t, err) + assert.Contains(t, err.Error(), "1.0", "the version that is missing") + assert.Contains(t, err.Error(), "versions list", "and the command that shows what is there") +} + +// The ordinary case: the pin is still registered, so reconciliation reuses it +// and reports no change. +func TestEnsureDatasetReusesAPinnedVersionThatStillExists(t *testing.T) { + r, localPath := pinnedDatasetReconciler(t, "golden", "1.0", http.StatusOK) + + version, changed, err := r.EnsureDataset( + context.Background(), + project.DatasetDecl{Name: "golden", Version: "1.0"}, + localPath, + ) + + require.NoError(t, err) + assert.Equal(t, "1.0", version) + assert.False(t, changed, "an unchanged file at a pinned version publishes nothing") +} + +// A read that failed is not a read that came back empty. Failing the deploy on +// a 403 or a timeout would turn a transient service problem into a broken +// pipeline for a pin that is very probably fine. +func TestEnsureDatasetKeepsAPinnedVersionWhenTheReadFails(t *testing.T) { + r, localPath := pinnedDatasetReconciler(t, "golden", "1.0", http.StatusForbidden) + + version, changed, err := r.EnsureDataset( + context.Background(), + project.DatasetDecl{Name: "golden", Version: "1.0"}, + localPath, + ) + + require.NoError(t, err) + assert.Equal(t, "1.0", version) + assert.False(t, changed) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output.go index 259ffff1b5e..139d38f328e 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output.go @@ -564,7 +564,9 @@ func writeResultsCSV(w io.Writer, run *eval_api.OpenAIEvalRun) error { cw := csv.NewWriter(w) defer cw.Flush() - if err := cw.Write([]string{"run_id", "status", "criterion", "passed", "failed"}); err != nil { + // Named as the service names it, and as the jsonl export already did, so a + // pipeline reading both formats needs one spelling rather than two. + if err := cw.Write([]string{"run_id", "status", "testing_criteria", "passed", "failed"}); err != nil { return err } if len(run.PerTestingCriteria) == 0 { diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output_write_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output_write_test.go index b347dca58c1..b3e0b8d5aa3 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output_write_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output_write_test.go @@ -38,7 +38,7 @@ func TestWriteResultsCSV(t *testing.T) { rows, err := csv.NewReader(&buf).ReadAll() require.NoError(t, err) - assert.Equal(t, []string{"run_id", "status", "criterion", "passed", "failed"}, rows[0]) + assert.Equal(t, []string{"run_id", "status", "testing_criteria", "passed", "failed"}, rows[0]) assert.Equal(t, []string{"evalrun_abc", "completed", "task_adherence", "8", "2"}, rows[1]) assert.Equal(t, []string{"evalrun_abc", "completed", "coherence", "10", "0"}, rows[2]) assert.Len(t, rows, 3, "one header and one row per criterion") @@ -57,7 +57,7 @@ func TestWriteResultsCSV_RunWithNoCriteria(t *testing.T) { require.NoError(t, err) require.Len(t, rows, 2) - assert.Equal(t, []string{"run_id", "status", "criterion", "passed", "failed"}, rows[0]) + assert.Equal(t, []string{"run_id", "status", "testing_criteria", "passed", "failed"}, rows[0]) assert.Equal(t, []string{"evalrun_empty", "failed", "", "", ""}, rows[1]) } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go b/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go index 1d48fa717ce..89d844b4d89 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go @@ -958,7 +958,7 @@ func DatasetNotFound(dataset string) error { func DatasetVersionNotFoundWithHint(dataset, version string) error { return fmt.Errorf( "no dataset %q at version %q in this project; "+ - "`azd ai eval dataset list` shows the ones there are", dataset, version) + "`azd ai eval dataset versions list %s` shows the ones there are", dataset, version, dataset) } // DatasetVersionNotFound reports a dataset version there is nothing to delete at. diff --git a/cli/azd/extensions/azure.ai.evaluations/tests/cli/run_output_test.go b/cli/azd/extensions/azure.ai.evaluations/tests/cli/run_output_test.go index 2ba5ab5264b..ab8c07fc96b 100644 --- a/cli/azd/extensions/azure.ai.evaluations/tests/cli/run_output_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/tests/cli/run_output_test.go @@ -192,7 +192,7 @@ func TestCLIResultsExport(t *testing.T) { require.NoError(t, err, "--format csv must emit parseable CSV:\n%s", r.Stdout) require.Len(t, rows, 2, "a header and one row per criterion") require.Equal(t, - []string{"run_id", "status", "criterion", "passed", "failed"}, rows[0]) + []string{"run_id", "status", "testing_criteria", "passed", "failed"}, rows[0]) require.Equal(t, f.FirstRunID, rows[1][0]) require.Equal(t, "completed", rows[1][1]) require.Equal(t, f.EvaluatorName, rows[1][2]) From ca6fd771a40297f55edfe3e7b983268c2c9620e8 Mon Sep 17 00:00:00 2001 From: mohessie Date: Sun, 16 Aug 2026 23:12:21 +0300 Subject: [PATCH 233/320] Offer --version on dataset update only, as the dataset extension does The shared create/update builder registered it on both, but the upload derives the next version from it, so create --version 4.0 published 5.0 rather than a first version -- and left the existence probe, which looks for the versions a first publish can carry, unable to find what it had just written. Fixed in the dataset extension this morning and missed here: the eleventh divergence between the duplicated packages, found by checking the rest of that morning's single-copy fixes after a test caught the tenth. Verified in a clean worktree at HEAD, since the working tree currently carries an unrelated in-progress refactor: create no longer lists --version, update still does. --- .../azure.ai.evaluations/internal/cmd/dataset.go | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset.go index 58895a04c5b..f5e8fb7c457 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset.go @@ -144,8 +144,14 @@ func newDatasetWriteCommand(verb, short string) *cobra.Command { cmd.Flags().StringVar(&fromFile, "from-file", "", "Path to a .jsonl file, or a directory containing one.") - cmd.Flags().StringVar(&version, "version", "", - "Current version to increment from. Omit to increment from the latest registered version.") + // Only on update. create publishes a first version, and the upload derives + // the next version from whatever this holds, so `create --version 4.0` + // would publish 5.0 -- and leave the existence probe, which looks for the + // versions a first publish can carry, unable to find what it wrote. + if verb == "update" { + cmd.Flags().StringVar(&version, "version", "", + "Current version to increment from. Omit to increment from the latest registered version.") + } cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") return cmd } From ce3a73b3d1b71919a3143eaa2d2a446f99dd102b Mon Sep 17 00:00:00 2001 From: mohessie Date: Sun, 16 Aug 2026 23:15:32 +0300 Subject: [PATCH 234/320] Assert, not require, inside test HTTP handlers testify's require calls t.FailNow, which the testing package documents as safe only on the goroutine running the test. Called from an httptest handler it aborts that goroutine mid-response, so the client reads a truncated reply and the failure lands on whichever test happens to be running -- reported with no assertion detail at all, which is what made it look like a flake in the reconciler tests. assert routes through t.Errorf, which is safe from any goroutine. --- .../internal/cmd/evalref_drift_test.go | 4 +++- .../internal/cmd/mutable_metadata_test.go | 8 +++++--- .../internal/cmd/reconciler_env_test.go | 4 +++- .../internal/pkg/dataset_api/download_wire_test.go | 5 ++++- 4 files changed, 15 insertions(+), 6 deletions(-) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evalref_drift_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evalref_drift_test.go index eb23ac81f81..3e90e2acb83 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evalref_drift_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evalref_drift_test.go @@ -32,7 +32,9 @@ func reconcilerListingVersions(t *testing.T, versions ...string) *evalReconciler values = append(values, map[string]any{"name": "golden", "version": v}) } w.Header().Set("Content-Type", "application/json") - require.NoError(t, json.NewEncoder(w).Encode(map[string]any{"value": values})) + // assert, not require: this runs on the server's goroutine, and FailNow + // there aborts mid-response and fails whichever test is running instead. + assert.NoError(t, json.NewEncoder(w).Encode(map[string]any{"value": values})) })) t.Cleanup(srv.Close) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/mutable_metadata_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/mutable_metadata_test.go index 9a0b4142b2c..c2448ececee 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/mutable_metadata_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/mutable_metadata_test.go @@ -37,15 +37,17 @@ func reconcilerHoldingEval( seen := &recordedUpdate{} srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // assert, not require: this runs on the server's goroutine, and FailNow + // there aborts mid-response and fails whichever test is running instead. if r.Method == http.MethodPost { raw, err := io.ReadAll(r.Body) - require.NoError(t, err) + assert.NoError(t, err) var body eval_api.UpdateOpenAIEvalRequest - require.NoError(t, json.Unmarshal(raw, &body)) + assert.NoError(t, json.Unmarshal(raw, &body)) seen.body = &body } w.Header().Set("Content-Type", "application/json") - require.NoError(t, json.NewEncoder(w).Encode(held)) + assert.NoError(t, json.NewEncoder(w).Encode(held)) })) t.Cleanup(srv.Close) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler_env_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler_env_test.go index 5f053cc9226..a29598c0db4 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler_env_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler_env_test.go @@ -98,7 +98,9 @@ func pinnedDatasetReconciler( return } w.Header().Set("Content-Type", "application/json") - require.NoError(t, json.NewEncoder(w).Encode(map[string]any{ + // assert, not require: this runs on the server's goroutine, and FailNow + // there aborts mid-response and fails whichever test is running instead. + assert.NoError(t, json.NewEncoder(w).Encode(map[string]any{ "name": name, "version": version, })) })) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/download_wire_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/download_wire_test.go index 6e1b80b60bb..822ad65c627 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/download_wire_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/download_wire_test.go @@ -64,7 +64,10 @@ func (s *storageServer) start(t *testing.T) (*DatasetClient, *httptest.Server) { switch { case strings.HasSuffix(r.URL.Path, "/credentials"): w.Header().Set("Content-Type", "application/json") - require.NoError(t, json.NewEncoder(w).Encode(map[string]any{ + // assert, not require: this runs on the server's goroutine, and + // FailNow there aborts mid-response and fails whichever test is + // running instead. + assert.NoError(t, json.NewEncoder(w).Encode(map[string]any{ "blobReferenceForConsumption": map[string]any{ "credential": map[string]any{"sasUri": srv.URL + s.uriPath + "?sig=secret"}, }, From 04ad4b6a7594860e0de01da783a6bd20d9655172 Mon Sep 17 00:00:00 2001 From: mohessie Date: Sun, 16 Aug 2026 23:17:04 +0300 Subject: [PATCH 235/320] Give this copy of dataset_api the same paging as the other The two copies implemented one concept twice under different names: followPages/doRequestGetURL/sameOrigin here, walkDatasetPages/followNextLink there. Four of the nine divergences found between the copies were in paging or redaction, which is what happens when the same idea has two homes and a fix lands in one. This copy now carries the dataset extension's version verbatim: the defensive copy of the caller's slice, the sameOrigin helper, and InvalidNextLink for a link that will not parse. Paging is in pages.go in both, and both log through internal/urlsafe. The two files are byte-identical apart from the module path, and the package function surface now matches exactly -- 42 and 42, down from seven differences. eval_api keeps its own paginator and PageLinkLeftTheService; only the duplicated dataset_api is converged here. --- .../internal/messages/messages.go | 11 ++ .../internal/pkg/dataset_api/list.go | 4 +- .../internal/pkg/dataset_api/operations.go | 2 +- .../internal/pkg/dataset_api/pages.go | 114 ++++++++++-------- .../internal/pkg/dataset_api/pages_test.go | 5 +- 5 files changed, 82 insertions(+), 54 deletions(-) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go b/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go index 89d844b4d89..92cb1820be8 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go @@ -1950,6 +1950,17 @@ func ConfigLockUnavailable(evalDir string, err error) error { "processes: %w", filepath.ToSlash(evalDir), err) } +// InvalidNextLink reports a pagination link the service sent that will not parse. +func InvalidNextLink(link string, err error) error { + return fmt.Errorf("invalid nextLink %q: %w", link, err) +} + +// NextLinkOffOrigin reports a pagination link pointing somewhere other than the +// project endpoint. Following it would send the caller's token to that host. +func NextLinkOffOrigin(origin string) error { + return fmt.Errorf("refusing to follow nextLink to %s: it is not the project endpoint", origin) +} + // PageLinkLeftTheService reports a paging link pointing somewhere else. // // The link arrives in a response body and this client sends an Authorization diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/list.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/list.go index 805902736a4..0396241c53c 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/list.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/list.go @@ -25,7 +25,7 @@ func (c *DatasetClient) ListDatasets(ctx context.Context, apiVersion string) (*D if err != nil { return nil, err } - return c.walkDatasetPages(ctx, first) + return c.followPages(ctx, first) } // ListDatasetVersions returns every version of a single dataset. @@ -39,7 +39,7 @@ func (c *DatasetClient) ListDatasetVersions( if err != nil { return nil, err } - return c.walkDatasetPages(ctx, first) + return c.followPages(ctx, first) } // DeleteDatasetVersion removes a single dataset version. diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/operations.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/operations.go index 9c09d9773f6..12691edd528 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/operations.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/operations.go @@ -480,7 +480,7 @@ func (c *DatasetClient) ListContainerBlobs(ctx context.Context, containerSASUri // carries none of the risk that following a body-supplied link would. var names []string marker := "" - for range maxPages { + for range maxListPages { page := *u q := page.Query() q.Set("restype", "container") // cspell:ignore restype — Azure Storage API query parameter diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/pages.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/pages.go index 230262ba426..adaefffd1ea 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/pages.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/pages.go @@ -19,38 +19,83 @@ import ( "github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime" ) -// maxPages bounds a walk the service controls, so a nextLink that points at -// itself cannot hold the command open indefinitely. -const maxPages = 100 +// maxListPages bounds page following so a service that keeps handing back a +// nextLink cannot spin forever. +const maxListPages = 100 -// followNextLink fetches one service-supplied page URL. -// -// The URL arrives in a response body and this client sends an Authorization -// header, so a link to another host would send the token there. Checked -// against the endpoint before it is used. -func (c *DatasetClient) followNextLink(ctx context.Context, nextLink string) ([]byte, error) { - parsed, err := url.Parse(nextLink) +// followPages walks nextLink until the service stops sending one, returning a +// single list holding every page. Without this, a project with more than one +// page lists incompletely and a latest-version check can decide from a stale +// first page. +func (c *DatasetClient) followPages(ctx context.Context, first *DatasetList) (*DatasetList, error) { + if first == nil { + return nil, nil + } + + // Copied rather than aliased: appending to first.Value could write into the + // caller's backing array when it has spare capacity. + out := &DatasetList{Value: append([]Dataset(nil), first.Value...)} + seen := map[string]bool{} + for next := first.NextLink; next != ""; { + if seen[next] || len(seen) >= maxListPages { + // A repeated or endless link is the service misbehaving, not a reason + // to fail the command, but the list is short and nobody would know. + log.Printf("[dataset_api] stopped paging after %d pages; the listing may be incomplete", len(seen)) + break + } + seen[next] = true + + body, err := c.doRequestGetURL(ctx, next) + if err != nil { + return nil, err + } + var page DatasetList + // A page that answers 200 with no body ends the walk; unmarshaling it + // would throw away every page already collected. + if len(body) > 0 { + if err := json.Unmarshal(body, &page); err != nil { + return nil, messages.ParsingResponse(err) + } + } + out.Value = append(out.Value, page.Value...) + next = page.NextLink + } + return out, nil +} + +// sameOrigin reports whether two URLs share a scheme and host. +func sameOrigin(a, b *url.URL) bool { + return strings.EqualFold(a.Scheme, b.Scheme) && strings.EqualFold(a.Host, b.Host) +} + +// doRequestGetURL issues a GET against an absolute URL the service supplied, +// such as a nextLink. The URL is refused unless it shares the endpoint's +// origin: the pipeline attaches the caller's token, so a link pointing +// elsewhere would hand that token to another host. +func (c *DatasetClient) doRequestGetURL(ctx context.Context, rawURL string) ([]byte, error) { + parsed, err := url.Parse(rawURL) if err != nil { - return nil, messages.InvalidEndpointURL(err) + return nil, messages.InvalidNextLink(rawURL, err) } base, err := url.Parse(c.endpoint) if err != nil { return nil, messages.InvalidEndpointURL(err) } - // A nextLink is allowed to be relative, and a relative one carries no host - // or scheme of its own. Resolving it against the endpoint first keeps the - // origin check meaningful instead of refusing a legitimate link. - next := base.ResolveReference(parsed) - if !strings.EqualFold(base.Host, next.Host) || !strings.EqualFold(base.Scheme, next.Scheme) { - return nil, messages.PageLinkLeftTheService(base.Host, next.Host) + // A nextLink is allowed to be relative. Resolving it against the endpoint + // first keeps the origin check meaningful instead of rejecting a legitimate + // relative link for having no scheme or host of its own. + u := base.ResolveReference(parsed) + if !sameOrigin(u, base) { + return nil, messages.NextLinkOffOrigin(u.Scheme + "://" + u.Host) } - req, err := runtime.NewRequest(ctx, http.MethodGet, next.String()) + req, err := runtime.NewRequest(ctx, http.MethodGet, u.String()) if err != nil { return nil, messages.CreatingRequest(err) } - log.Printf("[dataset_api] GET %s", urlsafe.URL(next)) + + log.Printf("[dataset_api] GET %s", urlsafe.URL(u)) resp, err := c.pipeline.Do(req) if err != nil { @@ -68,34 +113,3 @@ func (c *DatasetClient) followNextLink(ctx context.Context, nextLink string) ([] } return respBody, nil } - -// walkDatasetPages gathers every page of a dataset listing. -// -// The listing answered with one page and a nextLink, and the link was decoded -// and dropped. UploadVersion picks the next version from this listing, so a -// version on page two meant reusing one that already exists. -func (c *DatasetClient) walkDatasetPages(ctx context.Context, first *DatasetList) (*DatasetList, error) { - seen := map[string]bool{} - link := first.NextLink - for link != "" { - if seen[link] || len(seen) >= maxPages { - log.Printf("[dataset_api] stopped paging after %d pages; the listing may be incomplete", len(seen)) - break - } - seen[link] = true - - body, err := c.followNextLink(ctx, link) - if err != nil { - return nil, err - } - var page DatasetList - if len(body) > 0 { - if err := json.Unmarshal(body, &page); err != nil { - return nil, messages.ParsingResponse(err) - } - } - first.Value = append(first.Value, page.Value...) - link = page.NextLink - } - return first, nil -} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/pages_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/pages_test.go index 469de2d7efe..cdf8949db04 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/pages_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/pages_test.go @@ -67,7 +67,10 @@ func TestNextLinkToAnotherHostIsRefused(t *testing.T) { _, err := c.ListDatasets(context.Background(), "v1") require.Error(t, err) - assert.Contains(t, err.Error(), "not followed") + assert.Contains(t, err.Error(), "refusing to follow", + "the refusal has to say the link was not followed") + assert.Contains(t, err.Error(), elsewhere.URL, + "and name the host it refused, so the reader can see what was in the body") } // A link pointing at the page it came from is the one shape that would From c12e25667c78f0f94a66dc6b2803fe6f33e8283c Mon Sep 17 00:00:00 2001 From: mohessie Date: Sun, 16 Aug 2026 23:18:35 +0300 Subject: [PATCH 236/320] Say where the per-evaluator breakdown lives The table carries one rate per run, because a column per evaluator stops being readable as soon as two runs score different evaluators, and an absent evaluator renders as a gap that reads like a zero. The service does return per_testing_criteria_results on the list route and the runs go out whole, so a script can already read it -- nothing said so. Help text and a test now name the field. The test pins the name and that it survives marshalling; it does not catch someone narrowing the emitted type, which is how this would actually be lost, and that needs the command harness the reconciler tests now have. A run-to-run comparison, which is the question people actually ask, is a next-milestone item and sidesteps the varying evaluator set by comparing exactly two runs. --- .../internal/cmd/run_list_test.go | 27 +++++++++++++++++++ .../internal/cmd/run_ops.go | 11 +++++++- 2 files changed, 37 insertions(+), 1 deletion(-) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_list_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_list_test.go index 7e2aaf9f157..bec0c241092 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_list_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_list_test.go @@ -4,11 +4,13 @@ package cmd import ( + "bytes" "testing" "azureaieval/internal/pkg/eval_api" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) // Scenario 3 answers "did my change help?" by reading two rows of `run list`, @@ -56,3 +58,28 @@ func TestRunListTimestampsAreRFC3339UTC(t *testing.T) { assert.Equal(t, "2026-08-01T09:15:22Z", timestampString("2026-08-01T09:15:22Z")) assert.Empty(t, timestampString(nil)) } + +// The table shows one rate per run because a column per evaluator stops being +// readable as soon as two runs score different evaluators. That makes `-o json` +// the only place a per-evaluator breakdown can be read, and the service does +// return it on the list route, so the runs go out unprojected. +// +// This pins the field name and that it survives marshalling. It does not catch +// someone replacing the emitted type with a projection, which is the way this +// would actually be lost -- that needs the command harness the reconciler tests +// now have. +func TestRunListJSONCarriesThePerEvaluatorBreakdown(t *testing.T) { + var buf bytes.Buffer + runs := []eval_api.OpenAIEvalRun{{ + ID: "evalrun_1", + PerTestingCriteria: []eval_api.EvalRunCriteriaResult{ + {TestingCriteria: "task_adherence", Passed: 14, Failed: 1}, + }, + }} + + require.NoError(t, emitJSONList(&buf, runs)) + + assert.Contains(t, buf.String(), `"per_testing_criteria_results"`, + "the only place a script can read a per-evaluator result") + assert.Contains(t, buf.String(), "task_adherence") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go index d3323edc661..8edaecb0b48 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go @@ -39,7 +39,12 @@ func newRunListCommand() *cobra.Command { cmd := &cobra.Command{ Use: "list", Short: "List runs for an eval.", - Args: cobra.NoArgs, + Long: "List runs for an eval.\n\n" + + "The table carries one pass rate per run. A per-evaluator breakdown " + + "cannot fit a column each and stay readable when runs score different " + + "evaluators, so `-o json` carries it instead, under " + + "`per_testing_criteria_results` on every run.", + Args: cobra.NoArgs, RunE: func(cmd *cobra.Command, args []string) error { ctx := cmd.Context() ec, err := newEvalContext(ctx, endpointFlg) @@ -61,6 +66,10 @@ func newRunListCommand() *cobra.Command { return messages.ListingRuns(evalID, err) } if isJSON(cmd) { + // Emitted whole, unlike `run start`, which hands back a small + // handoff. The table cannot show a per-evaluator breakdown, so + // this is the only place a script can read one; narrowing these + // to the table's columns would drop it silently. var runs []eval_api.OpenAIEvalRun if list != nil { runs = list.Data From 7e7fc29b90a4360d78e575c64409e6a9d02fe4c6 Mon Sep 17 00:00:00 2001 From: mohessie Date: Sun, 16 Aug 2026 23:34:46 +0300 Subject: [PATCH 237/320] Match errors on type rather than the legacy As pattern cli/azd/AGENTS.md requires errors.AsType[*azcore.ResponseError] over the legacy errors.As target pattern, which also avoids the mutable out-parameter. Copilot raised this on the dataset extension's PR and I converted that copy this morning; this one kept the old form. Twelfth divergence between the duplicated packages, found by diffing the two copies function by function after convergence rather than waiting for a reviewer. Verified in a clean worktree at HEAD: the working tree currently has an in-progress edit removing messages that committed code still calls, so a build there fails for unrelated reasons. --- .../internal/pkg/dataset_api/operations.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/operations.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/operations.go index 12691edd528..9540e8edc1d 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/operations.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/operations.go @@ -179,8 +179,8 @@ func (c *DatasetClient) latestRegisteredVersion( // isVersionConflict reports whether the service refused the upload because the // target version already exists. func IsVersionConflict(err error) bool { - var respErr *azcore.ResponseError - if !errors.As(err, &respErr) { + respErr, ok := errors.AsType[*azcore.ResponseError](err) + if !ok { return false } return respErr.StatusCode == http.StatusConflict @@ -189,8 +189,8 @@ func IsVersionConflict(err error) bool { // IsNotFound reports whether the service answered 404, which is how it says a // dataset does not exist yet. func IsNotFound(err error) bool { - var respErr *azcore.ResponseError - if !errors.As(err, &respErr) { + respErr, ok := errors.AsType[*azcore.ResponseError](err) + if !ok { return false } return respErr.StatusCode == http.StatusNotFound From 22b41a3eb1ea262e9cf2fd0a6f582aa3a416645b Mon Sep 17 00:00:00 2001 From: mohessie Date: Mon, 17 Aug 2026 00:10:41 +0300 Subject: [PATCH 238/320] Refuse to report success when a gate never got a verdict A run that outlived the two-hour wait returned nil, on the reasoning that this is the same contract as --no-wait. It is not: --no-wait with --fail-on is refused up front precisely because a gate with no result is silently dropped. Reached by running long instead, the same drop happened and the command exited 0, so a pipeline that asked to be gated read it as a pass. It now refuses when a gate was set, naming the run, the wait it outlived, and how to reattach with the gate. Without a gate nothing changes: the wait stopping is not a failure. Two tests. One pins the message. The other reads the source of the branch, which is ugly but proportionate here -- driving it needs a run outliving a const budget and a signed-in client, and the failure mode is a gate that passes silently, so a regression looks exactly like success. Same reasoning as the linker-path check in internal/version. Also converts the four FailNow-based assertions in eval_api's paging test handlers to goroutine-safe ones, including a t.Fatalf the report missed. Found by a targeted review of the surface the earlier passes skipped. --- .../internal/cmd/gating_budget_test.go | 58 +++++++++++++++++++ .../azure.ai.evaluations/internal/cmd/run.go | 6 ++ .../internal/messages/messages.go | 14 +++++ .../pkg/eval_api/output_items_paging_test.go | 14 +++-- 4 files changed, 87 insertions(+), 5 deletions(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/gating_budget_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/gating_budget_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/gating_budget_test.go new file mode 100644 index 00000000000..8b137b96a0b --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/gating_budget_test.go @@ -0,0 +1,58 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "os" + "strings" + "testing" + "time" + + "azureaieval/internal/messages" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// A run that outlives the wait leaves --fail-on with nothing to judge. Exiting +// 0 there tells a pipeline the gate passed, which is the silent drop --no-wait +// is refused for, reached by running long instead. The message has to say the +// gate did not run, not merely that the wait stopped. +func TestGateOutlivedTheWaitSaysTheGateNeverRan(t *testing.T) { + err := messages.GateOutlivedTheWait("run_abc", 2*time.Hour) + + require.Error(t, err) + assert.Contains(t, err.Error(), "run_abc") + assert.Contains(t, err.Error(), "2h0m0s", "the message has to say how long it waited") + assert.Contains(t, err.Error(), "--fail-on", + "a reader has to know which flag went unanswered") + assert.Contains(t, err.Error(), "run show", + "and how to get the verdict they asked for") +} + +// The check above only proves the message is right, not that anything calls it. +// Driving the branch itself needs a run that outlives a two-hour const budget +// and a signed-in client, so this reads the source instead. +// +// Worth the ugliness here: the failure mode is a gate that passes silently, so +// a regression looks exactly like success and no other test would notice. Same +// reasoning as the linker-path check in internal/version. +func TestWaitBudgetBranchStillConsultsTheGate(t *testing.T) { + body, err := os.ReadFile("run.go") + require.NoError(t, err) + + src := string(body) + start := strings.Index(src, "errors.Is(err, errWaitBudgetSpent)") + require.NotEqual(t, -1, start, "the wait-budget branch has moved or gone") + + branch := src[start:] + if end := strings.Index(branch, "\n\t\t\tif err != nil {"); end != -1 { + branch = branch[:end] + } + + assert.Contains(t, branch, "threshold.set", + "the branch has to ask whether a gate was set before reporting success") + assert.Contains(t, branch, "GateOutlivedTheWait", + "and refuse rather than exit 0 when one was") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go index 375a22dc883..8952c34b32b 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go @@ -219,6 +219,12 @@ func buildRunCommand(use, short string) *cobra.Command { final, err := ec.pollRun(ctx, evalID, run.ID, out, isJSON(cmd)) if errors.Is(err, errWaitBudgetSpent) { + // A gate asked for a verdict that never arrived. Exiting 0 here + // would tell a pipeline the gate passed, which is the silent + // drop --no-wait is refused for, reached by running long. + if threshold.set { + return messages.GateOutlivedTheWait(run.ID, waitBudget) + } // The run did not fail, the wait ran out. Same contract as // --no-wait: exit 0 and say how to pick it back up. if isJSON(cmd) { diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go b/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go index 92cb1820be8..098d9af89c9 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go @@ -131,6 +131,20 @@ func GateNeedsTheWait() error { " --wait --fail-on `") } +// GateOutlivedTheWait reports a gate that never got a verdict because the run +// outlived the wait. +// +// Without a gate this is not a failure and exits 0, which is why the run was +// reported and the reattach line printed. With one, exiting 0 tells a pipeline +// the gate passed when nothing was ever judged -- the same silent drop +// GateNeedsTheWait refuses up front, arrived at by running long instead. +func GateOutlivedTheWait(runID string, budget time.Duration) error { + return fmt.Errorf( + "run %s outlived the %s wait, so --fail-on never got a result to judge. "+ + "The run is still going: reattach with `azd ai eval run show %s "+ + "--wait --fail-on `", runID, budget, runID) +} + // DatasetHasUnregisteredEdits reports local rows no deployed version holds. func DatasetHasUnregisteredEdits(dataset, deployCmd string) error { return fmt.Errorf( diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/output_items_paging_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/output_items_paging_test.go index 1503c252539..bb6a6516254 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/output_items_paging_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/output_items_paging_test.go @@ -28,7 +28,7 @@ func TestListOutputItemsFollowsTheCursor(t *testing.T) { switch n { case 1: - require.Empty(t, after, "the first page is not asked for by cursor") + assert.Empty(t, after, "the first page is not asked for by cursor") writeJSON(t, w, map[string]any{ "data": []map[string]any{ {"id": "a", "status": "completed"}, @@ -38,13 +38,14 @@ func TestListOutputItemsFollowsTheCursor(t *testing.T) { "last_id": "b", }) case 2: - require.Equal(t, "b", after, "the next page is asked for from the last id") + assert.Equal(t, "b", after, "the next page is asked for from the last id") writeJSON(t, w, map[string]any{ "data": []map[string]any{{"id": "c", "status": "completed"}}, "has_more": false, }) default: - t.Fatalf("asked for a page after the service said there were none") + t.Errorf("asked for a page after the service said there were none") + http.Error(w, "unexpected page request", http.StatusInternalServerError) } }) @@ -104,7 +105,7 @@ func TestListOutputItemsStopsOnAnEmptyPage(t *testing.T) { client := newRecordingClient(t, func(w http.ResponseWriter, r *http.Request) { requests.Add(1) - require.Less(t, requests.Load(), int32(5), "the client is looping") + assert.Less(t, requests.Load(), int32(5), "the client is looping") writeJSON(t, w, map[string]any{ "data": []map[string]any{}, "has_more": true, @@ -119,10 +120,13 @@ func TestListOutputItemsStopsOnAnEmptyPage(t *testing.T) { assert.EqualValues(t, 1, requests.Load()) } +// writeJSON is called from the server's goroutine, so it asserts rather than +// requires: FailNow off the test's own goroutine aborts mid-response and the +// failure lands on whichever test happens to be running. func writeJSON(t *testing.T, w http.ResponseWriter, body any) { t.Helper() w.Header().Set("Content-Type", "application/json") - require.NoError(t, json.NewEncoder(w).Encode(body)) + assert.NoError(t, json.NewEncoder(w).Encode(body)) } func ids(items []OutputItem) []string { From 5c09c85049afd7056b5f47ebe6f6d8d9a6e1ebb1 Mon Sep 17 00:00:00 2001 From: mohessie Date: Mon, 17 Aug 2026 01:04:55 +0300 Subject: [PATCH 239/320] Redact the SAS when a request cannot even be built Redaction covered what http.Client.Do returns and stopped there. Two earlier paths were left: url.Parse of the container SAS at the top of UploadBlob, DownloadBlob and ListContainerBlobs, and the request construction that follows. Both hand back an error naming the URL, so a SAS the parser refuses reached the user's screen and the debug log with its signature intact. Seven sites across the two copies now wrap with urlsafe.Error, which rebuilds a *url.Error without the URL. A test drives all four entry points with a URL the parser refuses that still carries a sig, and asserts neither the signature nor the parameter name survives. Worth noting how it was found: reading the code caught the construction sites, and the test caught the parse sites -- three of its four cases failed on first run against the fix I had just written. --- .../construction_redaction_test.go | 64 +++++++++++++++++++ .../internal/pkg/dataset_api/operations.go | 14 ++-- 2 files changed, 71 insertions(+), 7 deletions(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/construction_redaction_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/construction_redaction_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/construction_redaction_test.go new file mode 100644 index 00000000000..f4059a61779 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/construction_redaction_test.go @@ -0,0 +1,64 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package dataset_api + +import ( + "context" + "strings" + "testing" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// sasSecret is the signature a storage SAS carries in its query string. +const constructionSASSecret = "SIGNATUREVALUETHATMUSTNOTAPPEAR" + +// A URL the parser refuses, still carrying a SAS. Building a request from it +// fails before any transport error can happen, which is the path the redaction +// work missed: Do's failures were wrapped and NewRequest's were not. +func malformedSASURL() string { + return "https://acct.blob.core.windows.net/c/d.jsonl\x7f?sig=" + constructionSASSecret +} + +func constructionClient(t *testing.T) *DatasetClient { + t.Helper() + pipeline := runtime.NewPipeline("test", "v1", runtime.PipelineOptions{}, + &policy.ClientOptions{Retry: policy.RetryOptions{MaxRetries: -1}}) + return NewDatasetClientFromPipeline("https://example.invalid", pipeline) +} + +// The premise: the raw error names the URL, so returning it unwrapped hands the +// signature to whoever reads the message or the debug log. +func TestRequestConstructionErrorsDoNotCarryTheSAS(t *testing.T) { + raw := malformedSASURL() + require.Contains(t, raw, constructionSASSecret, "the fixture has to carry a secret to leak") + + client := constructionClient(t) + ctx := context.Background() + + cases := []struct { + name string + call func() error + }{ + {"DownloadDataset", func() error { _, err := client.DownloadDataset(ctx, raw); return err }}, + {"DownloadBlob", func() error { _, err := client.DownloadBlob(ctx, raw, "d.jsonl"); return err }}, + {"UploadBlob", func() error { return client.UploadBlob(ctx, raw, "d.jsonl", []byte("{}")) }}, + {"ListContainerBlobs", func() error { _, err := client.ListContainerBlobs(ctx, raw); return err }}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + err := tc.call() + + require.Error(t, err, "a URL the parser refuses has to fail") + assert.NotContains(t, err.Error(), constructionSASSecret, + "the signature reached the caller through %s", tc.name) + assert.NotContains(t, strings.ToLower(err.Error()), "sig=", + "even the parameter name should not survive") + }) + } +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/operations.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/operations.go index 9540e8edc1d..5fbd71d1c01 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/operations.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/operations.go @@ -287,7 +287,7 @@ var blobHTTPClient = &http.Client{Timeout: 10 * time.Minute} func (c *DatasetClient) UploadBlob(ctx context.Context, containerSASUri, blobName string, data []byte) error { u, err := url.Parse(containerSASUri) if err != nil { - return messages.InvalidContainerURI(err) + return messages.InvalidContainerURI(urlsafe.Error(err)) } // Append blob name to the container path. @@ -295,7 +295,7 @@ func (c *DatasetClient) UploadBlob(ctx context.Context, containerSASUri, blobNam req, err := http.NewRequestWithContext(ctx, http.MethodPut, u.String(), bytes.NewReader(data)) if err != nil { - return messages.CreatingUploadRequest(err) + return messages.CreatingUploadRequest(urlsafe.Error(err)) } req.Header.Set("x-ms-blob-type", "BlockBlob") req.Header.Set("Content-Type", "application/octet-stream") @@ -439,7 +439,7 @@ func pickDatasetBlob(names []string) string { func (c *DatasetClient) DownloadDataset(ctx context.Context, downloadURL string) ([]byte, error) { req, err := runtime.NewRequest(ctx, http.MethodGet, downloadURL) if err != nil { - return nil, messages.CreatingDownloadRequest(err) + return nil, messages.CreatingDownloadRequest(urlsafe.Error(err)) } // Use a plain HTTP client for blob downloads — the SAS token in the URL provides @@ -472,7 +472,7 @@ func (c *DatasetClient) ListContainerBlobs(ctx context.Context, containerSASUri // Parse the container URI and append list query parameters. u, err := url.Parse(containerSASUri) if err != nil { - return nil, messages.InvalidContainerURI(err) + return nil, messages.InvalidContainerURI(urlsafe.Error(err)) } // The Blob service answers one page and a NextMarker. Only the marker value @@ -494,7 +494,7 @@ func (c *DatasetClient) ListContainerBlobs(ctx context.Context, containerSASUri req, err := http.NewRequestWithContext(ctx, http.MethodGet, page.String(), nil) if err != nil { - return nil, messages.CreatingListRequest(err) + return nil, messages.CreatingListRequest(urlsafe.Error(err)) } pageNames, next, err := c.readBlobPage(req) @@ -538,7 +538,7 @@ func (c *DatasetClient) readBlobPage(req *http.Request) ([]string, string, error func (c *DatasetClient) DownloadBlob(ctx context.Context, containerSASUri, blobName string) ([]byte, error) { u, err := url.Parse(containerSASUri) if err != nil { - return nil, messages.InvalidContainerURI(err) + return nil, messages.InvalidContainerURI(urlsafe.Error(err)) } // Append blob name to the container path. @@ -546,7 +546,7 @@ func (c *DatasetClient) DownloadBlob(ctx context.Context, containerSASUri, blobN req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil) if err != nil { - return nil, messages.CreatingBlobDownloadRequest(err) + return nil, messages.CreatingBlobDownloadRequest(urlsafe.Error(err)) } httpClient := blobHTTPClient From 75edbb23b2673b5ba512a0a9eaa8e6545ca0502b Mon Sep 17 00:00:00 2001 From: mohessie Date: Mon, 17 Aug 2026 01:14:45 +0300 Subject: [PATCH 240/320] Pin what any-failure counts as a row that did not pass The gate reads unpassed as total minus passed, so a row that errored or was skipped breaches -- which is the point of a gate. Nothing tested it: replacing that with the Failed count alone left every test green, and a run whose rows all errored would have reported success to a pipeline. The pass-rate half of the same concern was already covered, so the gap was one gate, not the idea. Found by mutating the two guards rather than reading the test names, which read as though both were covered. Also worth recording: the first attempt at that mutation silently failed to match and reported the tests as toothless. A mutation you do not verify landed is worse than none, because it accuses the tests instead of the code. --- .../internal/cmd/gating_conformance_test.go | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/gating_conformance_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/gating_conformance_test.go index 325e2d36e1b..9637402f061 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/gating_conformance_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/gating_conformance_test.go @@ -65,6 +65,24 @@ func TestPassRateCountsErroredAndSkippedAgainstTheThreshold(t *testing.T) { assert.Empty(t, g.breach(&eval_api.EvalRunResultCounts{Total: 3, Passed: 3})) } +// any-failure is the same concern as the rate above asked as a yes or no, and +// it was the untested half: the gate counts everything that is not a pass, so +// replacing that with the Failed count alone let a run whose rows errored +// report success, and no test noticed. +func TestAnyFailureCountsErroredAndSkippedAsUnpassed(t *testing.T) { + g, err := parseGate("any-failure") + require.NoError(t, err) + + assert.NotEmpty(t, g.breach(&eval_api.EvalRunResultCounts{Total: 3, Passed: 2, Errored: 1}), + "a row that errored did not pass") + assert.NotEmpty(t, g.breach(&eval_api.EvalRunResultCounts{Total: 3, Passed: 2, Skipped: 1}), + "a row that was skipped did not pass either") + assert.NotEmpty(t, g.breach(&eval_api.EvalRunResultCounts{Total: 3, Passed: 2, Failed: 1})) + + assert.Empty(t, g.breach(&eval_api.EvalRunResultCounts{Total: 3, Passed: 3}), + "every row passed, so there is nothing to report") +} + // A run that scored nothing breaches every threshold rather than dividing by // zero. "No rows passed" is the honest reading of an empty result. func TestEmptyRunBreachesEveryThreshold(t *testing.T) { From 7037105ba18708147c322bf6c4566b8210f0b795 Mon Sep 17 00:00:00 2001 From: mohessie Date: Mon, 17 Aug 2026 01:20:09 +0300 Subject: [PATCH 241/320] Blame the paging link, not the endpoint, when the service sends a bad one A nextLink that would not parse was reported as an invalid endpoint URL, which is the one thing it is not: the link comes from the service, the endpoint is the caller's and was fine. It sent the reader to check configuration that was never wrong. The dataset extension's paginator already separates the two. This is the only thing worth changing in this paginator. It is a third implementation on purpose -- a different client, bearer auth rather than SAS, and generic over the response type -- and the two extensions are separate Go modules, so sharing one would mean copying rather than importing. Its defences are already sound: it resolves relative links before checking the origin, compares scheme as well as host so a link cannot downgrade to http, remembers every link so a multi-hop cycle ends the walk, and bounds the walk at 100 pages. --- .../internal/pkg/eval_api/pages.go | 2 +- .../internal/pkg/eval_api/paging_edge_test.go | 17 +++++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/pages.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/pages.go index bb0fc8cf8bc..8b3e4200804 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/pages.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/pages.go @@ -35,7 +35,7 @@ const maxPages = 100 func (c *EvalClient) followNextLink(ctx context.Context, nextLink string) ([]byte, error) { parsed, err := url.Parse(nextLink) if err != nil { - return nil, messages.InvalidEndpointURL(err) + return nil, messages.InvalidNextLink(nextLink, err) } base, err := url.Parse(c.endpoint) if err != nil { diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/paging_edge_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/paging_edge_test.go index a47503bfd4c..f1ad0912687 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/paging_edge_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/paging_edge_test.go @@ -69,6 +69,23 @@ func TestListEvaluatorVersionsRefusesALinkResolvingToAnotherHost(t *testing.T) { "the token must never be sent to the other host") } +// A nextLink comes from the service, so a link that will not parse is the +// service's fault, not the caller's. Reporting it as an invalid endpoint sent +// the reader to check configuration that was never wrong. +func TestListEvaluatorVersionsBlamesTheLinkNotTheEndpoint(t *testing.T) { + c, _ := clientAndServer(t, func(w http.ResponseWriter, r *http.Request) { + fmt.Fprint(w, `{"value":[{"name":"one"}],"nextLink":"https://host\u007f/x"}`) + }) + + _, err := c.ListEvaluatorVersions(t.Context(), "e", "v1") + + require.Error(t, err, "a nextLink the parser refuses has to fail") + assert.Contains(t, err.Error(), "nextLink", + "the reader has to know the service sent a bad link") + assert.NotContains(t, err.Error(), "invalid endpoint URL", + "their endpoint is fine and sending them to it wastes the investigation") +} + // A cycle longer than one hop used to run to maxPages, because only a link // pointing at the page it came from ended the walk. func TestListEvaluatorVersionsStopsOnATwoPageCycle(t *testing.T) { From 4eaf72a46914f5d02c1cdc464ab67686f2b8972b Mon Sep 17 00:00:00 2001 From: mohessie Date: Mon, 17 Aug 2026 01:59:11 +0300 Subject: [PATCH 242/320] Document the endpoint cascade as implemented The README listed three levels and the resolver has four: it omitted the AZURE_AI_PROJECT_ENDPOINT fallback at both environment levels, and the machine-wide global config that azure.ai.agents writes, which outranks the host environment variable. That last one is the surprising part and now says so: a project context left behind by zd ai agent on an unrelated project silently wins over the variable exported in the current shell. Validate() keeps the endpoint inside services.ai.azure.com, so this cannot redirect a token to an arbitrary host, but it can send it to the wrong Foundry project without saying so. --- cli/azd/extensions/azure.ai.evaluations/README.md | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/cli/azd/extensions/azure.ai.evaluations/README.md b/cli/azd/extensions/azure.ai.evaluations/README.md index 86398f2ef46..148b7fbb711 100644 --- a/cli/azd/extensions/azure.ai.evaluations/README.md +++ b/cli/azd/extensions/azure.ai.evaluations/README.md @@ -112,8 +112,16 @@ anything. The project endpoint is resolved in this order: 1. `--project-endpoint` -2. `FOUNDRY_PROJECT_ENDPOINT` in the active azd environment -3. the host environment variable of the same name +2. `FOUNDRY_PROJECT_ENDPOINT` in the active azd environment, then + `AZURE_AI_PROJECT_ENDPOINT` there +3. `extensions.ai-agents.project.context.endpoint` in azd's global config, + which `azure.ai.agents` writes and this extension only reads +4. `FOUNDRY_PROJECT_ENDPOINT` in the host environment, then + `AZURE_AI_PROJECT_ENDPOINT` + +Level 3 is worth knowing about: it is machine-wide rather than per-project, so +a project context left behind by `azd ai agent` somewhere else takes precedence +over the variable exported in this shell. `--debug` prints which level answered. ## Local development From 29dbc77f081e1e14886265444e8882ef60c4cef2 Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 18 Aug 2026 00:46:55 +0300 Subject: [PATCH 243/320] Fix the defects the bug bash and review rounds found Bug bash items - `init` no longer stops mid-sentence at "to read one from": the message names both places a deployment is read from, and a test asserts the rendered text. - The judge model now falls back to AZURE_AI_MODEL_DEPLOYMENT_NAME, which is the only place it survives when azd binds to an existing Foundry project. - The `Next:` command `init` prints carries --target and --generation-model, so it runs as printed instead of failing twice first. - `--evaluator` splits on commas like its sibling flag, and a reference that cannot name an evaluator is refused by `init` rather than by `create` two commands later. - `job`'s --dataset/--evaluator say they are required, since the same two names are optional filters on `generate` one command over. - `create` and `run start` ask which eval when several are declared, and still refuse under --no-prompt, where there is nobody to ask. - The debug log is written outside the working directory, which is the user's repository and where a routine `git add -A` committed one. Review findings - The endpoint cascade no longer treats every daemon error as fatal. azd answers "no default environment" with a plain error, which grpc encodes as Unknown, so a project with no environment selected could not reach the global config or the host variable. - A read that failed is no longer read as an eval that was deleted: falling through on a 503 created a second eval and forked the run history for good. - Cursor pagination is bounded. A cursor that repeats, or one that advances forever, held the command open and grew the result until the process died. - `pass-rate=NaN` is refused. It parsed, cleared both range checks, and then lost every comparison, so a pipeline that asked to be gated never was. - A run that scored nothing breaches an any-failure gate, as the comment always claimed and the code did not. - "Nothing selected" is no longer read as the first choice, which could have started a billed run against an eval nobody picked. - `run output export` reports a failed close instead of exiting 0 over a file that stops mid-row. - The retry inside `generate` records its own job id rather than the abandoned one, and SplitEvaluators reads the reference rather than the criterion label. - Repaired nine double-encoded characters in comments. --- .../internal/cmd/context.go | 2 +- .../internal/cmd/dataset.go | 54 +++--- .../internal/cmd/dataset_presence_test.go | 157 ++++++++++++++++++ .../internal/cmd/debug.go | 18 +- .../internal/cmd/eval_choice.go | 74 +++++++++ .../internal/cmd/eval_choice_test.go | 75 +++++++++ .../internal/cmd/eval_group.go | 2 +- .../internal/cmd/evaluator_flag_test.go | 69 ++++++++ .../internal/cmd/gating.go | 16 +- .../internal/cmd/gating_silent_test.go | 74 +++++++++ .../internal/cmd/generate.go | 6 + .../internal/cmd/generate_composite.go | 6 + .../azure.ai.evaluations/internal/cmd/init.go | 45 ++++- .../internal/cmd/init_evaluators.go | 7 +- .../internal/cmd/init_model.go | 61 ++++++- .../internal/cmd/init_target.go | 7 +- .../internal/cmd/init_test.go | 33 +++- .../azure.ai.evaluations/internal/cmd/job.go | 9 +- .../internal/cmd/output.go | 11 ++ .../internal/cmd/reconciler.go | 39 ++++- .../azure.ai.evaluations/internal/cmd/run.go | 3 +- .../internal/cmd/run_output.go | 65 +++++--- .../foundry/projectctx/hosted_absence_test.go | 54 ++++++ .../internal/foundry/projectctx/resolver.go | 41 ++++- .../internal/messages/agent_warnings_test.go | 4 +- .../messages/evaluator_missing_test.go | 4 +- .../internal/messages/messages.go | 27 ++- .../internal/messages/paths_test.go | 64 +++++-- .../internal/messages/rendered_text_test.go | 26 +++ .../internal/pkg/dataset_api/operations.go | 10 +- .../pkg/eval_api/cursor_bound_test.go | 116 +++++++++++++ .../internal/pkg/eval_api/evaluators.go | 14 +- .../internal/pkg/eval_api/generation.go | 5 +- .../internal/pkg/eval_api/operations.go | 20 ++- .../internal/pkg/eval_api/portal_urls_test.go | 16 +- 35 files changed, 1119 insertions(+), 115 deletions(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset_presence_test.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/eval_choice.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/eval_choice_test.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator_flag_test.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/gating_silent_test.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/foundry/projectctx/hosted_absence_test.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/messages/rendered_text_test.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/cursor_bound_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/context.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/context.go index fe39ee1c640..3c0b7060aa0 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/context.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/context.go @@ -172,7 +172,7 @@ func (ec *evalContext) remember(ctx context.Context, key, value string) { } // setEnvValue persists a value into the active azd environment. azd itself -// writes none of these keys — the extension owns them. +// writes none of these keys -- the extension owns them. func (ec *evalContext) setEnvValue(ctx context.Context, key, value string) error { if ec.envName == "" { envResp, err := ec.azdClient.Environment().GetCurrent(ctx, &azdext.EmptyRequest{}) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset.go index f5e8fb7c457..8df34f24b8b 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset.go @@ -4,6 +4,7 @@ package cmd import ( + "context" "errors" "fmt" "os" @@ -54,6 +55,35 @@ func newDatasetUpdateCommand() *cobra.Command { return newDatasetWriteCommand("update", "Publish a new version of a dataset.") } +// datasetPresence answers whether the dataset is already registered, and +// whether a "no" can be trusted. +// +// The version listing lags a publish, so a create followed straight by an +// update was told the dataset it had just made does not exist. A point read of +// the versions a first publish can carry usually settles that, catching up +// sooner than the listing. +// +// Absence is only certain when the listing itself answered 404. An empty 200 +// does not prove it: an unknown dataset and a listing that has not caught up +// are indistinguishable. +func datasetPresence( + ctx context.Context, + client *dataset_api.DatasetClient, + name string, +) (exists, absenceCertain bool) { + existing, listErr := client.ListDatasetVersions(ctx, name, ProjectEndpointAPIVersion) + if listErr == nil && existing != nil && len(existing.Value) > 0 { + return true, false + } + + for _, v := range firstDatasetVersions { + if _, err := client.GetDataset(ctx, name, v, ProjectEndpointAPIVersion); err == nil { + return true, false + } + } + return false, dataset_api.IsNotFound(listErr) +} + // newDatasetWriteCommand builds create and update. Both run the same upload, // and the existence check is the only thing that separates them: a version is // brought into being by startPendingUpload, which neither knows nor cares @@ -90,29 +120,9 @@ func newDatasetWriteCommand(verb, short string) *cobra.Command { } defer ec.Close() - existing, listErr := ec.datasetClient.ListDatasetVersions( - ctx, name, ProjectEndpointAPIVersion, - ) - exists := listErr == nil && existing != nil && len(existing.Value) > 0 - if !exists { - // The version listing lags a publish, so a `create` followed by - // an `update` was told the dataset it had just made does not - // exist. A direct read usually settles it, catching up sooner - // than the listing does. - for _, v := range firstDatasetVersions { - if _, err := ec.datasetClient.GetDataset( - ctx, name, v, ProjectEndpointAPIVersion, - ); err == nil { - exists = true - break - } - } - } - // Only an outright 404 proves the name is unknown. An empty 200 does - // not: an unknown dataset and a listing that has not caught up are - // indistinguishable. + exists, absenceCertain := datasetPresence(ctx, ec.datasetClient, name) if err := checkAssetExistence( - verb, "dataset", name, exists, dataset_api.IsNotFound(listErr), + verb, "dataset", name, exists, absenceCertain, ); err != nil { return err } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset_presence_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset_presence_test.go new file mode 100644 index 00000000000..5cc190d45db --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset_presence_test.go @@ -0,0 +1,157 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "net/http" + "net/http/httptest" + "strings" + "sync" + "testing" + + "azureaieval/internal/pkg/dataset_api" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// presenceServer stands up a fake project endpoint and records every path the +// presence probe asks for, so a test can assert what was tried as well as what +// was concluded. +type presenceServer struct { + mu sync.Mutex + paths []string +} + +// requested returns the paths seen so far, in order. +func (s *presenceServer) requested() []string { + s.mu.Lock() + defer s.mu.Unlock() + return append([]string(nil), s.paths...) +} + +// newPresenceClient wires a DatasetClient to a server that answers the version +// listing with listStatus/listBody, and answers a point read of a dataset +// version with whatever found reports for that version. +func newPresenceClient( + t *testing.T, + listStatus int, + listBody string, + found map[string]bool, +) (*dataset_api.DatasetClient, *presenceServer) { + t.Helper() + + rec := &presenceServer{} + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + rec.mu.Lock() + rec.paths = append(rec.paths, r.URL.Path) + rec.mu.Unlock() + + // Assertions inside a handler run on the server's goroutine, where a + // Fatalf would leave the client hanging on a response never written. + assert.Equal(t, http.MethodGet, r.Method) + + switch { + case strings.HasSuffix(r.URL.Path, "/versions"): + w.WriteHeader(listStatus) + _, _ = w.Write([]byte(listBody)) + default: + version := r.URL.Path[strings.LastIndex(r.URL.Path, "/")+1:] + if found[version] { + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte(`{"name":"ds","version":"` + version + `"}`)) + return + } + http.Error(w, `{"error":{"code":"NotFound"}}`, http.StatusNotFound) + } + })) + t.Cleanup(srv.Close) + + client := dataset_api.NewDatasetClientFromPipeline( + srv.URL, runtime.NewPipeline("test", "v1", runtime.PipelineOptions{}, nil)) + return client, rec +} + +// TestPresenceTrustsANonEmptyVersionListing is the ordinary case: the listing +// answered, so no point read is needed. +func TestPresenceTrustsANonEmptyVersionListing(t *testing.T) { + client, rec := newPresenceClient(t, + http.StatusOK, `{"value":[{"name":"ds","version":"1.0"}]}`, nil) + + exists, absenceCertain := datasetPresence(t.Context(), client, "ds") + + require.True(t, exists) + require.False(t, absenceCertain) + require.Equal(t, []string{"/datasets/ds/versions"}, rec.requested(), + "a listing that answered should settle it without a point read") +} + +// TestPresenceProbesPastAListingThatHasNotCaughtUp covers the bug the probe +// exists for: a create publishes 1.0, the listing still reports nothing, and +// the update that follows must not be told the dataset is missing. +func TestPresenceProbesPastAListingThatHasNotCaughtUp(t *testing.T) { + client, rec := newPresenceClient(t, + http.StatusOK, `{"value":[]}`, map[string]bool{"1.0": true}) + + exists, absenceCertain := datasetPresence(t.Context(), client, "ds") + + require.True(t, exists, "the point read found the version the listing had not") + require.False(t, absenceCertain) + require.Equal(t, []string{"/datasets/ds/versions", "/datasets/ds/versions/1.0"}, + rec.requested()) + require.NoError(t, checkAssetExistence("update", "dataset", "ds", exists, absenceCertain)) + require.Error(t, checkAssetExistence("create", "dataset", "ds", exists, absenceCertain), + "create must still refuse a name the probe found") +} + +// TestPresenceProbesTheVersionSomethingElseRegistered covers a dataset created +// by the portal, the SDK or a generation job, which numbers its first version +// "1" rather than the "1.0" this CLI publishes. +func TestPresenceProbesTheVersionSomethingElseRegistered(t *testing.T) { + client, rec := newPresenceClient(t, + http.StatusOK, `{"value":[]}`, map[string]bool{"1": true}) + + exists, absenceCertain := datasetPresence(t.Context(), client, "ds") + + require.True(t, exists) + require.False(t, absenceCertain) + require.Equal(t, + []string{"/datasets/ds/versions", "/datasets/ds/versions/1.0", "/datasets/ds/versions/1"}, + rec.requested(), + "both first-publish versions should be probed before giving up") +} + +// TestPresenceWillNotCallAnEmptyListingProofOfAbsence is the guard that keeps +// `update` working against a service whose listing lags. An empty 200 is not a +// 404, so the gate must let the update through rather than refuse it. +func TestPresenceWillNotCallAnEmptyListingProofOfAbsence(t *testing.T) { + client, _ := newPresenceClient(t, http.StatusOK, `{"value":[]}`, nil) + + exists, absenceCertain := datasetPresence(t.Context(), client, "ds") + + require.False(t, exists) + require.False(t, absenceCertain, + "an empty listing does not distinguish an unknown dataset from a stale one") + require.NoError(t, checkAssetExistence("update", "dataset", "ds", exists, absenceCertain), + "update must proceed when absence is unproven") +} + +// TestPresenceTreatsA404ListingAsProofOfAbsence is the other half: a service +// that actually said "no such dataset" should stop an update before it uploads. +func TestPresenceTreatsA404ListingAsProofOfAbsence(t *testing.T) { + client, _ := newPresenceClient(t, + http.StatusNotFound, `{"error":{"code":"NotFound"}}`, nil) + + exists, absenceCertain := datasetPresence(t.Context(), client, "ds") + + require.False(t, exists) + require.True(t, absenceCertain) + + err := checkAssetExistence("update", "dataset", "ds", exists, absenceCertain) + require.Error(t, err) + require.Contains(t, err.Error(), `dataset "ds" does not exist`) + require.NoError(t, checkAssetExistence("create", "dataset", "ds", exists, absenceCertain), + "create is exactly what a proven-absent name should allow") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/debug.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/debug.go index 48fc0802aaf..197e3feadad 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/debug.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/debug.go @@ -8,6 +8,7 @@ import ( "io" "log" "os" + "path/filepath" "strconv" "time" @@ -28,7 +29,19 @@ func setupDebugLogging(flags *pflag.FlagSet) func() { return func() {} } - logFileName := fmt.Sprintf("azd-ai-eval-%s.log", time.Now().Format("2006-01-02")) + // Written outside the working directory: that is the user's repository, the + // scaffolded .gitignore does not cover this name, and a routine `git add -A` + // committed one. The private subdirectory matters because the temp directory + // is shared on Linux -- at a predictable path another user could leave a + // symlink and have the HTTP trace appended to a file of their choosing. + logDir := filepath.Join(os.TempDir(), "azd-ai-eval") + if err := os.MkdirAll(logDir, 0o700); err != nil { + logDir = os.TempDir() + } + logFileName := filepath.Join( + logDir, + fmt.Sprintf("azd-ai-eval-%s.log", time.Now().Format("2006-01-02")), + ) //nolint:gosec // the name is generated locally from the date, not user input logFile, err := os.OpenFile(logFileName, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0600) @@ -41,6 +54,9 @@ func setupDebugLogging(flags *pflag.FlagSet) func() { } else { w = logFile closeFile = func() { logFile.Close() } //nolint:gosec // best-effort cleanup + // A log nobody can find is not a log. Debugging was asked for + // explicitly, so naming the file costs nothing. + fmt.Fprintf(os.Stderr, "Debug log: %s\n", filepath.ToSlash(logFileName)) } log.SetOutput(w) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/eval_choice.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/eval_choice.go new file mode 100644 index 00000000000..d05445f56f2 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/eval_choice.go @@ -0,0 +1,74 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "azureaieval/internal/messages" + "azureaieval/internal/project" + + "github.com/azure/azure-dev/cli/azd/pkg/azdext" + "github.com/spf13/cobra" +) + +// chooseEvalIn is chooseEval for the run commands, which hold a directory +// rather than a loaded configuration. A configuration that will not open is +// left to the command that opens it properly, so the error stays the same one. +func chooseEvalIn(cmd *cobra.Command, evalDir, named string) string { + if named != "" || noPrompt(cmd) { + return named + } + cfg, err := project.OpenEvalConfig(evalDir) + if err != nil { + return named + } + return chooseEval(cmd, cfg, named) +} + +// chooseEval settles which eval a command means when the caller named none. +// +// Refusing is right under --no-prompt, where there is nobody to ask. Standing +// at a terminal it is not: the command holds the whole candidate list, and the +// documented scenarios declare a second eval, so every bare `run start` after +// that would fail permanently. +// +// Returning the name empty leaves the existing error to the caller, which is +// what happens whenever the prompt cannot run. +func chooseEval(cmd *cobra.Command, cfg *project.EvalConfig, named string) string { + if named != "" || cfg == nil || len(cfg.Evals) < 2 || noPrompt(cmd) { + return named + } + + azdClient, err := azdext.NewAzdClient() + if err != nil { + return named + } + defer azdClient.Close() + + names := cfg.EvalNames() + choices := make([]*azdext.SelectChoice, 0, len(names)) + for i := range names { + choices = append(choices, &azdext.SelectChoice{Label: names[i], Value: names[i]}) + } + + resp, err := azdClient.Prompt().Select(commandContext(cmd), &azdext.SelectRequest{ + Options: &azdext.SelectOptions{ + Message: messages.SelectEvalPrompt(), + Choices: choices, + }, + }) + if err != nil { + return named + } + // Value is optional on the wire, so an unset one arrives as 0 from + // GetValue -- indistinguishable from the first choice. Reading it as a + // selection would start a billed run against an eval nobody picked. + if resp == nil || resp.Value == nil { + return named + } + index := int(resp.GetValue()) + if index < 0 || index >= len(names) { + return named + } + return names[index] +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/eval_choice_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/eval_choice_test.go new file mode 100644 index 00000000000..d17840b398c --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/eval_choice_test.go @@ -0,0 +1,75 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "testing" + + "azureaieval/internal/project" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// configWith builds a configuration declaring the named evals. +func configWith(names ...string) *project.EvalConfig { + cfg := &project.EvalConfig{} + for _, n := range names { + cfg.Evals = append(cfg.Evals, project.Eval{Name: n}) + } + return cfg +} + +// chooseEval only asks when asking can settle something. Every case below +// resolves without a prompt, so none of them reaches the azd client. +func TestChooseEvalOnlyAsksWhenThereIsAChoice(t *testing.T) { + t.Run("a name given is never second-guessed", func(t *testing.T) { + got := chooseEval(newEvalCreateCommand(), configWith("a", "b"), "b") + assert.Equal(t, "b", got) + }) + + t.Run("one declared eval needs no question", func(t *testing.T) { + got := chooseEval(newEvalCreateCommand(), configWith("only"), "") + assert.Empty(t, got, "the caller resolves the single eval, so nothing is chosen here") + }) + + t.Run("no configuration is left to the caller", func(t *testing.T) { + assert.Empty(t, chooseEval(newEvalCreateCommand(), nil, "")) + }) + + t.Run("--no-prompt keeps the error", func(t *testing.T) { + cmd := newEvalCreateCommand() + cmd.Flags().Bool("no-prompt", true, "") + + got := chooseEval(cmd, configWith("a", "b"), "") + + assert.Empty(t, got, + "there is nobody to ask, so the command must still refuse rather than guess") + }) +} + +// Returning the name unchanged is what leaves the existing error in place, and +// that error is the one users praised: it counts the evals and names them all. +func TestSeveralEvalsErrorStillNamesEveryCandidate(t *testing.T) { + cfg := configWith("obs-trace-eval", "obs-eval") + cmd := newEvalCreateCommand() + // Without this the picker reaches azdext.NewAzdClient and attempts a real + // RPC, which passes only because resolving an empty address fails fast. + // This test is about the message, not about network behaviour. + cmd.Flags().Bool("no-prompt", true, "") + + _, err := cfg.Eval(chooseEval(cmd, cfg, "")) + + require.Error(t, err) + assert.Contains(t, err.Error(), "obs-trace-eval") + assert.Contains(t, err.Error(), "obs-eval") + assert.Contains(t, err.Error(), "--eval") +} + +// A directory with no configuration must not turn into a prompt, and must not +// swallow the error the command that opens it properly will raise. +func TestChooseEvalInLeavesAnAbsentConfigAlone(t *testing.T) { + assert.Empty(t, chooseEvalIn(newEvalCreateCommand(), t.TempDir(), "")) + assert.Equal(t, "named", chooseEvalIn(newEvalCreateCommand(), t.TempDir(), "named")) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/eval_group.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/eval_group.go index 80d8e342ad0..12b6bffcf14 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/eval_group.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/eval_group.go @@ -51,7 +51,7 @@ func newEvalCreateCommand() *cobra.Command { return err } - eval, err := cfg.Eval(firstArg(args)) + eval, err := cfg.Eval(chooseEval(cmd, cfg, firstArg(args))) if err != nil { return err } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator_flag_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator_flag_test.go new file mode 100644 index 00000000000..9dd8c5059a7 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator_flag_test.go @@ -0,0 +1,69 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "testing" + + "github.com/spf13/pflag" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// `--evaluator a,b` used to be accepted whole, writing one evaluator literally +// named "a,b" into the config. init exited 0 and the failure surfaced two +// commands later at create, naming a value passed to a different command. +func TestEvaluatorFlagSplitsOnCommas(t *testing.T) { + cmd := newInitCommand() + require.NoError(t, cmd.Flags().Parse([]string{ + "--evaluator", "builtin.task_adherence,builtin.relevance", + })) + + got, err := cmd.Flags().GetStringSlice("evaluator") + require.NoError(t, err) + assert.Equal(t, []string{"builtin.task_adherence", "builtin.relevance"}, got, + "a comma separates references; it is never part of an evaluator name") +} + +// The sibling repeatable flag already split on commas, and two flags documented +// the same way behaving differently is visible only to someone who knows pflag. +func TestRepeatableFlagsAgreeOnCommas(t *testing.T) { + typeOf := func(cmd *pflag.FlagSet, name string) string { + f := cmd.Lookup(name) + require.NotNilf(t, f, "%s is not registered", name) + return f.Value.Type() + } + + assert.Equal(t, "stringSlice", typeOf(newInitCommand().Flags(), "evaluator")) + assert.Equal(t, typeOf(newGenerateCommand().Flags(), "from"), + typeOf(newInitCommand().Flags(), "evaluator"), + "both are repeatable reference lists, so they must split alike") +} + +// Repeating the flag still works, because a comma list is an addition rather +// than a replacement. +func TestEvaluatorFlagStillRepeats(t *testing.T) { + cmd := newInitCommand() + require.NoError(t, cmd.Flags().Parse([]string{ + "--evaluator", "builtin.task_adherence", "--evaluator", "builtin.relevance", + })) + + got, err := cmd.Flags().GetStringSlice("evaluator") + require.NoError(t, err) + assert.Equal(t, []string{"builtin.task_adherence", "builtin.relevance"}, got) +} + +// A stray comma leaves an empty reference, which would otherwise be written to +// the config and looked up as "". +func TestEvaluatorRefsRejectWhatCannotNameAnEvaluator(t *testing.T) { + require.NoError(t, validateEvaluatorRefs([]string{"builtin.relevance", "my-rubric"})) + + err := validateEvaluatorRefs([]string{"builtin.relevance", ""}) + require.Error(t, err) + assert.Contains(t, err.Error(), "--evaluator") + + err = validateEvaluatorRefs([]string{"two words"}) + require.Error(t, err) + assert.Contains(t, err.Error(), "two words") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/gating.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/gating.go index b7232fbbc0b..2978db2e50c 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/gating.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/gating.go @@ -5,6 +5,7 @@ package cmd import ( "fmt" + "math" "os" "strconv" "strings" @@ -54,6 +55,12 @@ func parseGate(spec string) (gate, error) { if err != nil { return gate{}, messages.FailOnRateNotNumber(rate) } + // NaN parses, then passes both range checks, and then loses every + // comparison it is put in -- so a pipeline that asked to be gated would + // never be, and nothing would say so. + if math.IsNaN(value) { + return gate{}, messages.FailOnRateNotNumber(rate) + } if value < 0 || value > 1 { return gate{}, messages.FailOnRateOutOfRange(value) } @@ -77,6 +84,12 @@ func (g gate) breach(counts *eval_api.EvalRunResultCounts) string { if counts == nil { return messages.GateNoResultCounts() } + // Checked before any-failure as well as before the rate: a run that graded + // nothing has not passed, and reading zero unpassed rows as success let an + // empty run clear the gate that exists to catch exactly that. + if counts.Total == 0 { + return messages.GateNoRowsScored() + } if g.anyFailure { unpassed := counts.Total - counts.Passed if unpassed > 0 { @@ -84,9 +97,6 @@ func (g gate) breach(counts *eval_api.EvalRunResultCounts) string { } return "" } - if counts.Total == 0 { - return messages.GateNoRowsScored() - } actual := float64(counts.Passed) / float64(counts.Total) if actual < g.passRate { return messages.GatePassRateBelow(actual, g.passRate) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/gating_silent_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/gating_silent_test.go new file mode 100644 index 00000000000..930f205da14 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/gating_silent_test.go @@ -0,0 +1,74 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "testing" + + "azureaieval/internal/pkg/eval_api" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// A gate is the whole reason a pipeline runs this command, so the ways it can +// silently stop gating matter more than the ways it can fire. + +// NaN parses, clears both range checks, and then loses every comparison it is +// put in. A pipeline written this way believes it is gated and is not. +func TestFailOnRejectsAThresholdThatCanNeverFire(t *testing.T) { + for _, spec := range []string{"pass-rate=NaN", "pass-rate=nan", "pass-rate=-nan"} { + _, err := parseGate(spec) + require.Errorf(t, err, "%s would disable the gate while looking like one", spec) + } + + // The range check already covers infinities; this pins that it still does. + for _, spec := range []string{"pass-rate=Inf", "pass-rate=-Inf", "pass-rate=1.5", "pass-rate=-0.1"} { + _, err := parseGate(spec) + require.Errorf(t, err, "%s is not a pass rate", spec) + } + + g, err := parseGate("pass-rate=0.8") + require.NoError(t, err) + assert.True(t, g.set) + assert.InDelta(t, 0.8, g.passRate, 1e-9) +} + +// A run that graded nothing has not passed. The pass-rate gate always said so; +// any-failure computed Total-Passed, which is zero for an empty run, and let it +// through -- the one shape a gate exists to catch. +func TestEveryGateBreachesOnARunThatScoredNothing(t *testing.T) { + empty := &eval_api.EvalRunResultCounts{Total: 0} + + anyFailure, err := parseGate("any-failure") + require.NoError(t, err) + assert.NotEmpty(t, anyFailure.breach(empty), + "an empty run must not clear an any-failure gate") + + rate, err := parseGate("pass-rate=0.8") + require.NoError(t, err) + assert.NotEmpty(t, rate.breach(empty)) + + // A gate that was never asked for stays silent whatever the counts. + assert.Empty(t, gate{}.breach(empty)) +} + +// The ordinary cases still behave, so the empty-run guard did not swallow them. +func TestGatesStillJudgeRunsThatScoredSomething(t *testing.T) { + anyFailure, err := parseGate("any-failure") + require.NoError(t, err) + + assert.Empty(t, anyFailure.breach(&eval_api.EvalRunResultCounts{Total: 3, Passed: 3}), + "every row passed, so there is nothing to report") + assert.NotEmpty(t, anyFailure.breach(&eval_api.EvalRunResultCounts{Total: 3, Passed: 2}), + "one row did not pass") + + rate, err := parseGate("pass-rate=0.8") + require.NoError(t, err) + assert.Empty(t, rate.breach(&eval_api.EvalRunResultCounts{Total: 10, Passed: 9})) + assert.NotEmpty(t, rate.breach(&eval_api.EvalRunResultCounts{Total: 10, Passed: 7})) + + // Counts the service never sent are not a pass. + assert.NotEmpty(t, rate.breach(nil)) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go index ec25868683c..3f0c45a4263 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go @@ -352,6 +352,12 @@ func (ec *evalContext) generateDataset( if err != nil { return nil, messages.SubmittingDataJob(err) } + // The retry is a second billed job, so the id the caller reports + // has to move with it. Leaving it on the abandoned first job points + // every resume and every `job show` at the wrong one. + if jobID != nil { + *jobID = job.ID + } completed, err = ec.pollGeneration(ctx, job.ID, DataGenerationAPIVersion, ec.evalClient.GetDataGenerationJob) } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_composite.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_composite.go index 3fb8b4f45c3..717b1a784ee 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_composite.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_composite.go @@ -271,6 +271,12 @@ func nameIsAPathComponent(name string) bool { if name == "" || name == "." || name == ".." { return false } + // A leading dash writes a file whose name reads as a flag to whatever the + // caller pipes the path into, which is the filesystem's objection rather + // than the service's. + if strings.HasPrefix(name, "-") { + return false + } if strings.ContainsAny(name, `/\:`) || filepath.IsAbs(name) { return false } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go index abd26e8ec5a..722b05df970 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go @@ -67,6 +67,14 @@ func newInitCommand() *cobra.Command { if maxTraces < 0 { return messages.MaxTracesMustBePositive() } + // Checked with the other flag-only rules, before anything is asked + // or read: a reference that cannot name an evaluator is otherwise + // written by a command that exits 0, and only fails two commands + // later. Answering two prompts first to be told a flag was wrong is + // the same defect one step removed. + if err := validateEvaluatorRefs(evaluators); err != nil { + return err + } if source == "" { source = initSourceDataset } @@ -259,9 +267,9 @@ func newInitCommand() *cobra.Command { cmd.Flags().IntVar(&maxTraces, "max-traces", project.DefaultScaffoldMaxTraces, "Cap on traces read by a --source traces eval. Delete max_traces from the "+ "file to take the service default instead.") - cmd.Flags().StringArrayVar(&evaluators, "evaluator", nil, - "Evaluator reference, repeatable. Use builtin. for a built-in. "+ - "Passing this replaces the defaults, so it also opts out of rubric generation.") + cmd.Flags().StringSliceVar(&evaluators, "evaluator", nil, + "Evaluator reference, repeatable and comma-separated. Use builtin. for a "+ + "built-in. Passing this replaces the defaults, so it also opts out of rubric generation.") cmd.Flags().StringVar(&judgeModel, "judge-model", "", "Model deployment the graders judge with. Detected from the project when omitted.") cmd.Flags().StringVar(&path, "path", project.DefaultEvalDir, @@ -299,6 +307,8 @@ type scaffold struct { eval *project.Eval datasetName string rubricName string + target string + judgeModel string generateDataset bool generateRubric bool } @@ -312,7 +322,7 @@ type scaffold struct { // rubric generation. func planScaffold(in scaffoldInput) scaffold { cfg := in.cfg - out := scaffold{rubricName: in.rubricName} + out := scaffold{rubricName: in.rubricName, target: in.target, judgeModel: in.judgeModel} eval := project.Eval{ Name: in.evalName, @@ -454,18 +464,20 @@ func (s scaffold) evaluatorNames() []string { // A caller who supplied both a dataset and their evaluators has nothing left to // generate, and pointing them at a generation command would submit a billed job // for an artifact they already have. +// +// Every generate step carries --target and --generation-model, which `generate` +// requires and does not detect. Omitting them printed a next step that failed +// twice before it ran, each failure naming one more flag. func (s scaffold) nextSteps(deployCmd string) []string { var steps []string switch { case s.generateDataset && s.generateRubric: // One command produces both, which is the whole point of the composite. - steps = append(steps, "azd ai eval generate") + steps = append(steps, s.generateCommand("")) case s.generateDataset: - steps = append(steps, - "azd ai eval generate --dataset --dataset-name "+s.datasetName) + steps = append(steps, s.generateCommand("--dataset --dataset-name "+s.datasetName)) case s.generateRubric: - steps = append(steps, - "azd ai eval generate --evaluator --evaluator-name "+s.rubricName) + steps = append(steps, s.generateCommand("--evaluator --evaluator-name "+s.rubricName)) } if len(steps) == 0 { steps = append(steps, deployCmd, "azd ai eval run start") @@ -473,6 +485,21 @@ func (s scaffold) nextSteps(deployCmd string) []string { return steps } +// generateCommand builds a `generate` invocation that runs as printed. +func (s scaffold) generateCommand(what string) string { + cmd := "azd ai eval generate" + if what != "" { + cmd += " " + what + } + if s.target != "" { + cmd += " --target " + s.target + } + if s.judgeModel != "" { + cmd += " --generation-model " + s.judgeModel + } + return cmd +} + // relativeToConfig rewrites a path given relative to the working directory so // it resolves from the directory holding the eval config. func relativeToConfig(path, evalDir string) string { diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_evaluators.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_evaluators.go index 1aee19273a2..088189ee6eb 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_evaluators.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_evaluators.go @@ -103,7 +103,7 @@ func promptEvaluators(cmd *cobra.Command, choices, preselected []string) ([]stri }) } - resp, err := azdClient.Prompt().MultiSelect(cmd.Context(), &azdext.MultiSelectRequest{ + resp, err := azdClient.Prompt().MultiSelect(commandContext(cmd), &azdext.MultiSelectRequest{ Options: &azdext.MultiSelectOptions{ Message: messages.SelectEvaluatorsPrompt(), Choices: opts, @@ -115,6 +115,11 @@ func promptEvaluators(cmd *cobra.Command, choices, preselected []string) ([]stri chosen := make([]string, 0, len(resp.GetValues())) for _, v := range resp.GetValues() { + // A blank choice would otherwise be written to the config as an + // evaluator named "", and looked up as one two commands later. + if v.GetValue() == "" { + continue + } chosen = append(chosen, v.GetValue()) } if len(chosen) == 0 { diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_model.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_model.go index 2060e5ce9d5..a93084d5945 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_model.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_model.go @@ -4,7 +4,9 @@ package cmd import ( + "context" "sort" + "strings" "azureaieval/internal/messages" @@ -12,6 +14,10 @@ import ( "github.com/spf13/cobra" ) +// judgeModelEnvKey is the key `azd ai agent init` writes the bound deployment +// to, and the only place it appears when the project was created beforehand. +const judgeModelEnvKey = "AZURE_AI_MODEL_DEPLOYMENT_NAME" + // modelDeployments names every model deployment the project declares, sorted so // a prompt and an error list the same way twice. // @@ -73,6 +79,13 @@ func resolveJudgeModel(cmd *cobra.Command, proj *azdext.ProjectConfig) (string, deployments := modelDeployments(proj) switch len(deployments) { case 0: + // Binding to an existing Foundry project writes `deployments: []` into + // azure.yaml, so the deployment `azd ai agent init` chose survives only + // in the azd environment. Reading it there is the difference between a + // configured project working and erroring. + if model := modelDeploymentFromAzdEnv(commandContext(cmd)); model != "" { + return model, nil + } return "", messages.JudgeModelRequired() case 1: return deployments[0], nil @@ -84,6 +97,47 @@ func resolveJudgeModel(cmd *cobra.Command, proj *azdext.ProjectConfig) (string, return promptJudgeModel(cmd, deployments) } +// modelDeploymentFromAzdEnv reads the deployment `azd ai agent init` recorded +// in the active azd environment. Absence is ordinary: `init` runs outside an +// azd project too, and the caller falls back to naming --judge-model. +func modelDeploymentFromAzdEnv(ctx context.Context) string { + azdClient, err := azdext.NewAzdClient() + if err != nil { + return "" + } + defer azdClient.Close() + + envResp, err := azdClient.Environment().GetCurrent(ctx, &azdext.EmptyRequest{}) + if err != nil || envResp.GetEnvironment() == nil { + return "" + } + val, err := azdClient.Environment().GetValue(ctx, &azdext.GetEnvRequest{ + EnvName: envResp.Environment.Name, + Key: judgeModelEnvKey, + }) + if err != nil { + return "" + } + return val.GetValue() +} + +// validateEvaluatorRefs rejects references that cannot name an evaluator. +// +// This needs no service call, so it keeps `init`'s promise to make none. The +// alternative is a config that scaffolds cleanly and fails at `create`, naming +// a value the user passed to a different command. +func validateEvaluatorRefs(refs []string) error { + for _, ref := range refs { + if strings.TrimSpace(ref) == "" { + return messages.EvaluatorRefEmpty() + } + if strings.ContainsAny(ref, " \t") { + return messages.EvaluatorRefMalformed(ref) + } + } + return nil +} + // promptJudgeModel asks which of the project's deployments to judge with. func promptJudgeModel(cmd *cobra.Command, deployments []string) (string, error) { azdClient, err := azdext.NewAzdClient() @@ -99,7 +153,7 @@ func promptJudgeModel(cmd *cobra.Command, deployments []string) (string, error) }) } - resp, err := azdClient.Prompt().Select(cmd.Context(), &azdext.SelectRequest{ + resp, err := azdClient.Prompt().Select(commandContext(cmd), &azdext.SelectRequest{ Options: &azdext.SelectOptions{ Message: messages.SelectJudgeModelPrompt(), Choices: choices, @@ -108,6 +162,11 @@ func promptJudgeModel(cmd *cobra.Command, deployments []string) (string, error) if err != nil { return "", messages.SelectingJudgeModel(err) } + // Value is optional on the wire, so an unset one arrives as 0 from + // GetValue and would read as the first deployment rather than as no answer. + if resp == nil || resp.Value == nil { + return "", messages.AmbiguousJudgeModel(deployments) + } index := int(resp.GetValue()) if index < 0 || index >= len(deployments) { return "", messages.AmbiguousJudgeModel(deployments) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_target.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_target.go index 9a44bae691c..9b03ec1d8aa 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_target.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_target.go @@ -63,7 +63,7 @@ func promptAgentTarget(cmd *cobra.Command, agents []string) (string, error) { choices = append(choices, &azdext.SelectChoice{Label: agents[i], Value: agents[i]}) } - resp, err := azdClient.Prompt().Select(cmd.Context(), &azdext.SelectRequest{ + resp, err := azdClient.Prompt().Select(commandContext(cmd), &azdext.SelectRequest{ Options: &azdext.SelectOptions{ Message: messages.SelectAgentPrompt(), Choices: choices, @@ -72,6 +72,11 @@ func promptAgentTarget(cmd *cobra.Command, agents []string) (string, error) { if err != nil { return "", messages.SelectingAgent(err) } + // Value is optional on the wire, so an unset one arrives as 0 from + // GetValue and would read as the first agent rather than as no answer. + if resp == nil || resp.Value == nil { + return "", messages.AmbiguousAgentTarget(agents) + } index := int(resp.GetValue()) if index < 0 || index >= len(agents) { return "", messages.AmbiguousAgentTarget(agents) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_test.go index 3774614af71..7eca43c1e76 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_test.go @@ -210,7 +210,9 @@ func TestScaffold_NextStepsOfferOnlyWhatIsScheduled(t *testing.T) { evalName: "support-agent-smoke", target: "support-agent", judgeModel: "m", }) // One command produces both, so there is one step, not two. - require.Equal(t, []string{"azd ai eval generate"}, plan.nextSteps("azd ai eval create")) + require.Equal(t, + []string{"azd ai eval generate --target support-agent --generation-model m"}, + plan.nextSteps("azd ai eval create")) }) t.Run("dataset supplied", func(t *testing.T) { @@ -218,7 +220,8 @@ func TestScaffold_NextStepsOfferOnlyWhatIsScheduled(t *testing.T) { evalName: "smoke", target: "support-agent", dataset: "prod-golden", judgeModel: "m", }) require.Equal(t, - []string{"azd ai eval generate --evaluator --evaluator-name support-agent-quality"}, + []string{"azd ai eval generate --evaluator --evaluator-name support-agent-quality " + + "--target support-agent --generation-model m"}, plan.nextSteps("azd ai eval create")) }) @@ -302,6 +305,32 @@ func TestScaffold_NextStepsNameCommandsThatExist(t *testing.T) { } } +// The Next: line is what a new user runs immediately after init, and it used to +// fail twice before it worked: `generate` requires --target and +// --generation-model, detects neither, and reports them one per invocation. +// Both values were on screen when init printed the hint. +func TestScaffold_NextStepsCarryWhatGenerateRequires(t *testing.T) { + for _, in := range []scaffoldInput{ + {evalName: "smoke", target: "support-agent", judgeModel: "gpt-4o-mini"}, + {evalName: "smoke", target: "support-agent", dataset: "prod-golden", judgeModel: "gpt-4o-mini"}, + } { + plan, _ := scaffoldFor(t, in) + seen := 0 + for _, step := range plan.nextSteps("azd ai eval create") { + if !strings.HasPrefix(step, "azd ai eval generate") { + continue + } + seen++ + require.Containsf(t, step, "--target support-agent", + "%q omits the target init had just detected", step) + require.Containsf(t, step, "--generation-model gpt-4o-mini", + "%q omits the model init had just resolved", step) + } + require.NotZerof(t, seen, + "no generate step was produced, so this asserted nothing: %+v", in) + } +} + // Built-ins are referenced but never declared, so the scaffold must not give // one a catalog entry to publish. func TestScaffold_BuiltinEvaluatorsGetNoCatalogEntry(t *testing.T) { diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/job.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/job.go index 0a71da7dafd..62c31a95b22 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/job.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/job.go @@ -87,8 +87,13 @@ type jobSelector struct { } func (s *jobSelector) bind(cmd *cobra.Command) { - cmd.Flags().BoolVar(&s.dataset, "dataset", false, "Act on dataset generation jobs.") - cmd.Flags().BoolVar(&s.evaluator, "evaluator", false, "Act on evaluator generation jobs.") + // "Required." leads, because the same two flag names are optional filters + // one command over on `generate`, and the help is the only thing that says + // which meaning applies here. + cmd.Flags().BoolVar(&s.dataset, "dataset", false, + "Required (or --evaluator). Act on dataset generation jobs.") + cmd.Flags().BoolVar(&s.evaluator, "evaluator", false, + "Required (or --dataset). Act on evaluator generation jobs.") cmd.MarkFlagsMutuallyExclusive("dataset", "evaluator") cmd.MarkFlagsOneRequired("dataset", "evaluator") } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/output.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/output.go index 36cabcf4867..db1bed539f5 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/output.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/output.go @@ -4,6 +4,7 @@ package cmd import ( + "context" "encoding/json" "errors" "fmt" @@ -63,6 +64,16 @@ func noPrompt(cmd *cobra.Command) bool { return err == nil && value } +// commandContext is cmd.Context() with cobra's pre-Execute nil made safe. +// gRPC dereferences the context it is handed, so a nil one is a panic rather +// than a failed call. +func commandContext(cmd *cobra.Command) context.Context { + if ctx := cmd.Context(); ctx != nil { + return ctx + } + return context.Background() +} + // emitJSON writes v as indented JSON. func emitJSON(w io.Writer, v any) error { enc := json.NewEncoder(w) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go index 12076b32e8c..77ce20628f9 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go @@ -9,6 +9,7 @@ import ( "encoding/json" "errors" "io/fs" + "log" "maps" "os" "reflect" @@ -487,12 +488,24 @@ func (r *evalReconciler) EnsureEval( // deployed under the name it had before. The environment records the id // against the digest as well, which is what recognizes a rename rather // than reading it as a delete plus an add. - if adopted := r.adoptRenamed(ctx, group, digest); adopted != "" { + adopted, err := r.adoptRenamed(ctx, group, digest) + if err != nil { + return "", false, err + } + if adopted != "" { cached = adopted } } if cached != "" && !recreate { - if remote, err := r.ec.evalClient.GetOpenAIEval(ctx, cached); err == nil { + remote, err := r.ec.evalClient.GetOpenAIEval(ctx, cached) + if err != nil && !eval_api.IsNotFound(err) { + // A read that failed is not an eval that is gone. Falling through + // on a 429, a 503 or an expired token would create a second eval + // and overwrite the recorded id, forking for good the run history + // this lookup exists to keep. + return "", false, err + } + if err == nil { // Reusing the eval is not the same as leaving it alone: name and // description are excluded from the digest because they must not // split a history, which makes this the only place an edit to @@ -532,17 +545,22 @@ func (r *evalReconciler) adoptRenamed( ctx context.Context, group project.Eval, digest string, -) string { +) (string, error) { id := r.ec.getEnvValue(ctx, digestIDKey(digest)) if id == "" { - return "" + return "", nil } remote, err := r.ec.evalClient.GetOpenAIEval(ctx, id) if err != nil { - return "" + if eval_api.IsNotFound(err) { + // The eval it used to be called is genuinely gone, so there is + // nothing to adopt and the caller creates one. + return "", nil + } + return "", err } r.pushMutable(ctx, id, group, remote) - return id + return id, nil } // pushMutable sends the half of a declaration the service treats as mutable. @@ -569,10 +587,15 @@ func (r *evalReconciler) pushMutable( if remote.Name == group.Name && maps.Equal(remote.Metadata, desired) { return } - _, _ = r.ec.evalClient.UpdateOpenAIEval(ctx, id, &eval_api.UpdateOpenAIEvalRequest{ + if _, err := r.ec.evalClient.UpdateOpenAIEval(ctx, id, &eval_api.UpdateOpenAIEvalRequest{ Name: group.Name, Metadata: desired, - }) + }); err != nil { + // Deliberately not fatal: a name or description that did not travel + // leaves the eval usable, and failing the deploy over it would be + // worse. It still has to be findable, so --debug can see it. + log.Printf("[reconcile] updating eval %s name/description: %v", id, err) + } } // withDescription applies the declaration's description to the metadata the diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go index 8952c34b32b..d1d678917b3 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go @@ -117,7 +117,8 @@ func buildRunCommand(use, short string) *cobra.Command { // One flag takes a name or an id. A declared name also brings the // declaration, which is what says where rows come from; a bare id // has none, so the pairing comes from the eval's previous run. - ref, err := ec.resolveEvalRef(ctx, ec.evalDir(ctx, evalPath), groupName) + evalDir := ec.evalDir(ctx, evalPath) + ref, err := ec.resolveEvalRef(ctx, evalDir, chooseEvalIn(cmd, evalDir, groupName)) if err != nil { return err } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output.go index 139d38f328e..fd98947745c 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output.go @@ -94,8 +94,16 @@ func newRunOutputListCommand() *cobra.Command { if err != nil { return messages.Creating(outFile, err) } - defer f.Close() - return emitJSON(f, rows) + if err := emitJSON(f, rows); err != nil { + _ = f.Close() + return err + } + // The last write is flushed by Close, so discarding its error + // reports success over a file that stops mid-row. + if err := f.Close(); err != nil { + return messages.Writing(outFile, err) + } + return nil } if isJSON(cmd) { return emitJSON(cmd.OutOrStdout(), rows) @@ -172,6 +180,25 @@ func newRunOutputShowCommand() *cobra.Command { return cmd } +// writeExport renders a run in the requested export format. +// +// Separate from the command so the file path can close explicitly and report +// the failure. A deferred Close cannot reach an unnamed return, so discarding +// it exits 0 over an export that stops mid-row. +func writeExport(w io.Writer, format string, run *eval_api.OpenAIEvalRun) error { + switch format { + case formatCSV: + return writeResultsCSV(w, run) + case formatJSON: + return emitJSON(w, run) + case formatJSONL: + return writeResultsJSONL(w, run) + default: + return messages.ExportFormatUnsupported( + format, formatCSV, formatJSON, formatJSONL) + } +} + func newRunOutputExportCommand() *cobra.Command { var ( format string @@ -214,27 +241,25 @@ func newRunOutputExportCommand() *cobra.Command { return err } - var w io.Writer = cmd.OutOrStdout() - if outFile != "" { - f, err := os.Create(outFile) - if err != nil { - return messages.Creating(outFile, err) - } - defer f.Close() - w = f + if outFile == "" { + return writeExport(cmd.OutOrStdout(), format, run) } - switch format { - case formatCSV: - return writeResultsCSV(w, run) - case formatJSON: - return emitJSON(w, run) - case formatJSONL: - return writeResultsJSONL(w, run) - default: - return messages.ExportFormatUnsupported( - format, formatCSV, formatJSON, formatJSONL) + f, createErr := os.Create(outFile) + if createErr != nil { + return messages.Creating(outFile, createErr) + } + writeErr := writeExport(f, format, run) + // The last write is flushed by Close, so discarding its error + // reports success over a file that stops mid-row. + closeErr := f.Close() + if writeErr != nil { + return writeErr + } + if closeErr != nil { + return messages.Writing(outFile, closeErr) } + return nil }, } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/foundry/projectctx/hosted_absence_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/foundry/projectctx/hosted_absence_test.go new file mode 100644 index 00000000000..096b4e33655 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/foundry/projectctx/hosted_absence_test.go @@ -0,0 +1,54 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package projectctx + +import ( + "errors" + "fmt" + "testing" + + "github.com/stretchr/testify/assert" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +// azd answers "no default environment" and "no such key" with plain Go errors. +// Its interceptor only rewrites errors carrying a suggestion or an auth +// failure, so everything else reaches the client as Unknown. Reading Unknown as +// a failure would stop a project with no environment selected from ever +// reaching the global config or the host variable. +func TestUnansweredHostedSourcesLetTheCascadeCarryOn(t *testing.T) { + for name, err := range map[string]error{ + "no daemon at all": status.Error(codes.Unavailable, "connection refused"), + "nothing under that key": status.Error(codes.NotFound, "key not found"), + "no default environment": status.Error(codes.Unknown, "default environment not found"), + "a plain error, unwrapped": errors.New("default environment not found"), + "wrapped in context": fmt.Errorf("reading the environment: %w", + status.Error(codes.Unknown, "default environment not found")), + } { + t.Run(name, func(t *testing.T) { + assert.True(t, hostedSourceAbsent(err), + "this is absence, so levels 3 and 4 still have to be consulted") + }) + } +} + +// A daemon that refused is not a daemon with nothing to say. Falling through +// here would resolve to a lower-priority endpoint belonging to another project, +// and nothing would have said so. +func TestARefusedReadIsReportedRatherThanSkipped(t *testing.T) { + for name, err := range map[string]error{ + "expired login": status.Error(codes.Unauthenticated, "reauthentication required"), + "not allowed": status.Error(codes.PermissionDenied, "forbidden"), + "the user hit ctrl-c": status.Error(codes.Canceled, "context canceled"), + "the read timed out": status.Error(codes.DeadlineExceeded, "deadline exceeded"), + "wrapped expiry": fmt.Errorf("reading the environment: %w", status.Error(codes.Unauthenticated, "expired")), + "nested twice over": fmt.Errorf("outer: %w", fmt.Errorf("inner: %w", status.Error(codes.PermissionDenied, "no"))), + } { + t.Run(name, func(t *testing.T) { + assert.False(t, hostedSourceAbsent(err), + "a refusal has to surface, not resolve to a different project") + }) + } +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/foundry/projectctx/resolver.go b/cli/azd/extensions/azure.ai.evaluations/internal/foundry/projectctx/resolver.go index 25fccf2b134..0e131334836 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/foundry/projectctx/resolver.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/foundry/projectctx/resolver.go @@ -34,15 +34,27 @@ func readAzdHostedSources(ctx context.Context) (AzdHostedSources, error) { } defer azdClient.Close() - if envResp, err := azdClient.Environment().GetCurrent( - ctx, &azdext.EmptyRequest{}, - ); err == nil && envResp.GetEnvironment() != nil { + envResp, envErr := azdClient.Environment().GetCurrent(ctx, &azdext.EmptyRequest{}) + if envErr != nil && !hostedSourceAbsent(envErr) { + // The daemon answered, but not with "there is no current environment". + // Reading that as absence falls through to global config or the host + // variable, which can point at a different project -- and the command + // would then land there without anything having said so. + return out, envErr + } + if envErr == nil && envResp.GetEnvironment() != nil { for _, key := range []string{foundryEnvKey, azureAiEnvKey} { envVal, valErr := azdClient.Environment().GetValue(ctx, &azdext.GetEnvRequest{ EnvName: envResp.Environment.Name, Key: key, }) - if valErr == nil && envVal.GetValue() != "" { + if valErr != nil { + if !hostedSourceAbsent(valErr) { + return out, valErr + } + continue + } + if envVal.GetValue() != "" { out.EnvValue = envVal.Value out.EnvName = envResp.Environment.Name break @@ -66,6 +78,27 @@ func readAzdHostedSources(ctx context.Context) (AzdHostedSources, error) { return out, nil } +// hostedSourceAbsent reports whether an error from the azd daemon leaves the +// cascade free to carry on to the next level. +// +// azd answers the ordinary "nothing here" cases -- no default environment, no +// such key -- with plain Go errors, which its interceptor passes through +// untouched and grpc then encodes as Unknown. Unknown therefore has to read as +// absence: treating it as a failure would stop a project that simply has no +// environment selected from ever reaching the global config or the host +// variable, which is the whole point of the levels below. +// +// What must not read as absence is a daemon that refused to answer, or a read +// that never finished. An expired login is mapped to Unauthenticated, and a +// Ctrl-C arrives as Canceled; falling through on either would resolve quietly +// to some other project's endpoint. +func hostedSourceAbsent(err error) bool { + return !containsGRPCCode(err, codes.Unauthenticated) && + !containsGRPCCode(err, codes.PermissionDenied) && + !containsGRPCCode(err, codes.Canceled) && + !containsGRPCCode(err, codes.DeadlineExceeded) +} + // containsGRPCCode walks the error chain looking for a gRPC status with the // specified code. fmt.Errorf("%w", ...) wraps errors without forwarding the // GRPCStatus() method, so we must unwrap manually. diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/messages/agent_warnings_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/messages/agent_warnings_test.go index 144d8d7a25b..4ba33ca1b3f 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/messages/agent_warnings_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/messages/agent_warnings_test.go @@ -6,6 +6,7 @@ package messages import ( "errors" "net/http" + "path/filepath" "testing" "github.com/Azure/azure-sdk-for-go/sdk/azcore" @@ -39,7 +40,8 @@ func TestAgentWarningsAnswerA404InOneLine(t *testing.T) { // syscall name and says nothing about what to run. func TestDatasetNotGeneratedYet(t *testing.T) { err := DatasetProblem("support-agent-eval", - DatasetNotGeneratedYet("support-agent-eval", `evals\datasets\support-agent-eval.jsonl`)) + DatasetNotGeneratedYet("support-agent-eval", + filepath.Join("evals", "datasets", "support-agent-eval.jsonl"))) got := err.Error() assert.Contains(t, got, "evals/datasets/support-agent-eval.jsonl") diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/messages/evaluator_missing_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/messages/evaluator_missing_test.go index 15cc1ca92d3..95783056a73 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/messages/evaluator_missing_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/messages/evaluator_missing_test.go @@ -4,6 +4,7 @@ package messages import ( + "path/filepath" "testing" "github.com/stretchr/testify/assert" @@ -14,7 +15,8 @@ import ( // callers wrap it with EvaluatorProblem, so the evaluator is named once. func TestEvaluatorNotGeneratedYet(t *testing.T) { err := EvaluatorProblem("support-agent-quality", - EvaluatorNotGeneratedYet("support-agent-quality", `evals\evaluators\support-agent-quality.json`)) + EvaluatorNotGeneratedYet("support-agent-quality", + filepath.Join("evals", "evaluators", "support-agent-quality.json"))) got := err.Error() assert.Equal(t, 1, countOccurrences(got, `"support-agent-quality"`), diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go b/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go index 098d9af89c9..b4f8f4bfad3 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go @@ -418,10 +418,12 @@ func SamplesFailedAtLeastOne(samples int) string { } // GeneratedNameNotAFileName reports a generated artifact name that would not -// stay inside the output directory. +// stay inside the output directory, or that would produce a file whose name is +// read as a flag by whatever the path is handed to next. func GeneratedNameNotAFileName(kind, name string) error { return fmt.Errorf( - "%s name %q cannot be used as a file name: remove any of / \\ : and do not use . or ..", + "%s name %q cannot be used as a file name: remove any of / \\ : , "+ + "do not start with -, and do not use . or ..", kind, name) } @@ -1591,7 +1593,21 @@ func SelectingAgent(err error) error { func JudgeModelRequired() error { return errors.New( "a model deployment is required to judge with: pass --judge-model. " + - "This project declares no deployments: to read one from") + "This project declares no deployments, and the azd environment sets no " + + "AZURE_AI_MODEL_DEPLOYMENT_NAME") +} + +// EvaluatorRefEmpty reports an --evaluator that carries no name, which is what +// a stray comma leaves behind. +func EvaluatorRefEmpty() error { + return errors.New("--evaluator was given an empty reference: name an evaluator, " + + "or use builtin. for a built-in") +} + +// EvaluatorRefMalformed reports a reference no evaluator can be found under. +func EvaluatorRefMalformed(ref string) error { + return fmt.Errorf("%q is not an evaluator reference: repeat --evaluator, or separate "+ + "them with commas, and use builtin. for a built-in", ref) } // AmbiguousJudgeModel reports several deployments where only one can be used. @@ -1606,6 +1622,11 @@ func SelectJudgeModelPrompt() string { return "Select the model deployment the graders judge with:" } +// SelectEvalPrompt asks which of the declared evals a command means. +func SelectEvalPrompt() string { + return "Select the eval to use:" +} + // SelectingJudgeModel reports a failed judge model prompt. func SelectingJudgeModel(err error) error { return fmt.Errorf("selecting a judge model deployment: %w", err) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/messages/paths_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/messages/paths_test.go index 8cd54f240eb..29b97ba637e 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/messages/paths_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/messages/paths_test.go @@ -7,12 +7,28 @@ import ( "errors" "fmt" "io/fs" + "path/filepath" + "runtime" "strings" "testing" "github.com/stretchr/testify/assert" ) +// configPath and friends build the separator this OS actually uses. Hardcoding +// a backslash only exercises the escaping on Windows: filepath.ToSlash rewrites +// the platform separator, so on Linux a literal backslash is left alone -- it is +// part of a filename there, not a separator. +var ( + testConfigPath = filepath.Join("evals", "azure.eval.yaml") + testDatasetDir = filepath.Join("evals", "datasets") + testDatasetPath = filepath.Join("evals", "datasets", "d.jsonl") + testEvaluatorPath = filepath.Join("evals", "evaluators", "e.json") + testInstructions = filepath.Join("docs", "instructions.md") + testOutDir = filepath.Join("evals", "out") + testOutPath = filepath.Join("evals", "out", "x.json") +) + // Running `create` before `init` is the first thing anyone does wrong, and the // bare read failure underneath is a Windows syscall phrase naming neither // command. @@ -21,7 +37,7 @@ import ( // configuration decide that by asking, so a nicer sentence that stopped // answering would turn every one of those into a failure. func TestNoEvalConfigStaysDetectable(t *testing.T) { - err := ReadingEvalConfig(`evals\azure.eval.yaml`, fmt.Errorf("open x: %w", fs.ErrNotExist)) + err := ReadingEvalConfig(testConfigPath, fmt.Errorf("open x: %w", fs.ErrNotExist)) assert.Contains(t, err.Error(), "no eval configuration at evals/azure.eval.yaml") assert.Contains(t, err.Error(), "azd ai eval init") @@ -29,7 +45,7 @@ func TestNoEvalConfigStaysDetectable(t *testing.T) { assert.True(t, errors.Is(err, fs.ErrNotExist), "callers tolerate an absent config by asking, so it has to keep answering") - other := ReadingEvalConfig(`evals\azure.eval.yaml`, errors.New("permission denied")) + other := ReadingEvalConfig(testConfigPath, errors.New("permission denied")) assert.Contains(t, other.Error(), "reading eval config") assert.False(t, errors.Is(other, fs.ErrNotExist)) } @@ -40,23 +56,39 @@ func TestPathsInMessagesStayCopyable(t *testing.T) { boom := errors.New("no such file") for _, err := range []error{ - ReadingEvalConfig(`evals\azure.eval.yaml`, boom), - ParsingEvalConfig(`evals\azure.eval.yaml`, boom), - WritingEvalConfig(`evals\azure.eval.yaml`, boom), - ReadingFromFile(`evals\datasets\d.jsonl`, boom), - FromFileMustBeJSONL(`evals\datasets`), - DatasetSource(`evals\datasets\d.jsonl`, boom), - EvaluatorSource(`evals\evaluators\e.json`, boom), - DatasetFileEmpty(`evals\datasets\d.jsonl`), - ReadingInstructionFile(`docs\instructions.md`, boom), - InstructionFileEmpty(`docs\instructions.md`), - Hashing(`evals\evaluators\e.json`, boom), - Creating(`evals\out`, boom), - Serializing(`evals\out\x.json`, boom), - Writing(`evals\out\x.json`, boom), + ReadingEvalConfig(testConfigPath, boom), + ParsingEvalConfig(testConfigPath, boom), + WritingEvalConfig(testConfigPath, boom), + ReadingFromFile(testDatasetPath, boom), + FromFileMustBeJSONL(testDatasetDir), + DatasetSource(testDatasetPath, boom), + EvaluatorSource(testEvaluatorPath, boom), + DatasetFileEmpty(testDatasetPath), + ReadingInstructionFile(testInstructions, boom), + InstructionFileEmpty(testInstructions), + Hashing(testEvaluatorPath, boom), + Creating(testOutDir, boom), + Serializing(testOutPath, boom), + Writing(testOutPath, boom), } { got := err.Error() assert.NotContains(t, got, `\\`, "a doubled separator is not a path anyone can use: %s", got) assert.Truef(t, strings.Contains(got, "/"), "the path should read with forward slashes: %s", got) } } + +// The separator conversion only has anything to convert on Windows: on Linux a +// backslash is an ordinary character in a filename, and filepath.ToSlash leaves +// it alone -- correctly. Without this case the suite would stay green on Linux +// with every filepath.ToSlash call deleted. +func TestWindowsSeparatorsRenderAsForwardSlashes(t *testing.T) { + if runtime.GOOS != "windows" { + t.Skip("filepath.ToSlash only rewrites the platform separator") + } + + got := ReadingEvalConfig(`evals\azure.eval.yaml`, errors.New("no such file")).Error() + + assert.Contains(t, got, "evals/azure.eval.yaml") + assert.NotContains(t, got, `\\`, + "%%q escapes a Windows separator, so the path stops being copyable: %s", got) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/messages/rendered_text_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/messages/rendered_text_test.go new file mode 100644 index 00000000000..1a30d4c4245 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/messages/rendered_text_test.go @@ -0,0 +1,26 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package messages + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/assert" +) + +// JudgeModelRequired shipped ending at "to read one from" -- the half of the +// sentence naming the way out never rendered. It is the first error a new user +// hits, and nothing caught it because no test asserted the rendered string: +// checking the error type or a prefix cannot see a missing tail. +func TestJudgeModelRequiredNamesBothPlacesItLooked(t *testing.T) { + msg := JudgeModelRequired().Error() + + assert.Contains(t, msg, "--judge-model", "the flag is the whole of the way out") + assert.Contains(t, msg, "AZURE_AI_MODEL_DEPLOYMENT_NAME", + "the other place a deployment is read from has to be named") + assert.NotContains(t, msg, "to read one from", "the sentence must not stop mid-clause") + assert.False(t, strings.HasSuffix(strings.TrimRight(msg, " "), ":"), + "a trailing colon promises a clause that never arrives: %q", msg) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/operations.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/operations.go index 5fbd71d1c01..93b178288c7 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/operations.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/operations.go @@ -95,8 +95,8 @@ func (c *DatasetClient) CreateDataset( // almost always mean "the version after whatever is registered", which is what // this does. // -// The version listing is eventually consistent — it returns nothing for a -// second or two after a version is created — so an empty listing cannot be +// The version listing is eventually consistent -- it returns nothing for a +// second or two after a version is created -- so an empty listing cannot be // trusted to mean the dataset is new. A conflict is therefore treated as a // stale read: the listing is re-read, and when it is still behind, the version // just refused is taken as proof that it exists and the next one is tried. @@ -199,7 +199,7 @@ func IsNotFound(err error) bool { // UploadNewVersion reads the first JSONL file from localDir, computes the next // version from currentVersion, and uploads it as a new dataset version using // the 3-step pending upload flow: -// 1. startPendingUpload → get SAS URI +// 1. startPendingUpload -> get SAS URI // 2. Upload blob to SAS URI // 3. Finalize dataset version with dataUri func (c *DatasetClient) UploadNewVersion( @@ -442,7 +442,7 @@ func (c *DatasetClient) DownloadDataset(ctx context.Context, downloadURL string) return nil, messages.CreatingDownloadRequest(urlsafe.Error(err)) } - // Use a plain HTTP client for blob downloads — the SAS token in the URL provides + // Use a plain HTTP client for blob downloads -- the SAS token in the URL provides // authentication, and Azure SDK pipeline policies (bearer token, correlation ID) // should not be sent to Azure Blob Storage endpoints. httpClient := blobHTTPClient @@ -483,7 +483,7 @@ func (c *DatasetClient) ListContainerBlobs(ctx context.Context, containerSASUri for range maxListPages { page := *u q := page.Query() - q.Set("restype", "container") // cspell:ignore restype — Azure Storage API query parameter + q.Set("restype", "container") // cspell:ignore restype -- Azure Storage API query parameter q.Set("comp", "list") if marker != "" { q.Set("marker", marker) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/cursor_bound_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/cursor_bound_test.go new file mode 100644 index 00000000000..f9ca13c3ad6 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/cursor_bound_test.go @@ -0,0 +1,116 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package eval_api + +import ( + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "sync/atomic" + "testing" + "time" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// stuckCursorServer always answers has_more with the same last_id, which is +// what a service in a bad state does. Without a bound the client walks it +// forever, holding the command open and growing the slice until the process +// dies -- so this test would hang rather than fail if the guard were removed. +func stuckCursorServer(t *testing.T, calls *atomic.Int64) *EvalClient { + t.Helper() + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + calls.Add(1) + body, err := json.Marshal(map[string]any{ + "data": []map[string]any{ + {"id": "item_1", "status": "pass"}, + }, + // The cursor never advances. + "has_more": true, + "last_id": "cursor_that_never_moves", + }) + assert.NoError(t, err) + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write(body) + })) + t.Cleanup(srv.Close) + + return NewEvalClientFromPipeline( + srv.URL, runtime.NewPipeline("test", "v1", runtime.PipelineOptions{}, nil)) +} + +// TestListOutputItemsStopsOnACursorThatNeverMoves pins termination. The +// deadline exists so a regression reports a failure instead of hanging the +// whole suite. +func TestListOutputItemsStopsOnACursorThatNeverMoves(t *testing.T) { + var calls atomic.Int64 + client := stuckCursorServer(t, &calls) + + done := make(chan struct{}) + var items *OutputItemList + var err error + go func() { + defer close(done) + items, err = client.ListOutputItems(t.Context(), "eval_1", "run_1", 0) + }() + + select { + case <-done: + case <-time.After(30 * time.Second): + t.Fatal("the walk never terminated on a repeating cursor") + } + + require.NoError(t, err) + require.NotNil(t, items) + assert.Equal(t, int64(2), calls.Load(), + "the repeat is visible on the second read, so the walk stops there") + assert.NotEmpty(t, items.Data, "the rows it did read are still returned") +} + +// A cursor that always advances defeats the repeat check, so the page ceiling +// is the only thing left holding the walk open. A service paging one row at a +// time forever would otherwise never return. +func TestListOutputItemsStopsAtThePageCeiling(t *testing.T) { + var calls atomic.Int64 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + n := calls.Add(1) + body, err := json.Marshal(map[string]any{ + "data": []map[string]any{{"id": fmt.Sprintf("item_%d", n), "status": "pass"}}, + "has_more": true, + // Always a new cursor, so `seen` never fires. + "last_id": fmt.Sprintf("cursor_%d", n), + }) + assert.NoError(t, err) + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write(body) + })) + t.Cleanup(srv.Close) + + client := NewEvalClientFromPipeline( + srv.URL, runtime.NewPipeline("test", "v1", runtime.PipelineOptions{}, nil)) + + done := make(chan struct{}) + var items *OutputItemList + var err error + go func() { + defer close(done) + items, err = client.ListOutputItems(t.Context(), "eval_1", "run_1", 0) + }() + + select { + case <-done: + case <-time.After(60 * time.Second): + t.Fatal("the walk never terminated on an endlessly advancing cursor") + } + + require.NoError(t, err) + require.NotNil(t, items) + assert.Equal(t, int64(maxPages), calls.Load(), + "the walk has to stop at the ceiling rather than trust the service to end it") + assert.Len(t, items.Data, maxPages) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/evaluators.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/evaluators.go index b30227e26ae..dde85a6e179 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/evaluators.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/evaluators.go @@ -6,6 +6,7 @@ package eval_api import ( "context" "fmt" + "log" "net/http" "net/url" "sort" @@ -266,7 +267,11 @@ func (c *EvalClient) ListOutputItems( // them would be a sample of the run rather than the run. all := &OutputItemList{} after := "" - for { + // A cursor that repeats while still returning rows would spin forever and + // grow all.Data until the process dies, so the walk is bounded the same way + // the next-link walker in pages.go is. + seen := map[string]bool{} + for page := 0; page < maxPages; page++ { query := map[string]string{} if limit > 0 { query["limit"] = strconv.Itoa(limit - len(all.Data)) @@ -287,8 +292,15 @@ func (c *EvalClient) ListOutputItems( if limit > 0 && len(all.Data) >= limit { return all, nil } + if seen[page.LastID] { + log.Printf("[eval_api] cursor %q repeated; the listing may be incomplete", page.LastID) + return all, nil + } + seen[page.LastID] = true after = page.LastID } + log.Printf("[eval_api] stopped after %d pages; the listing may be incomplete", maxPages) + return all, nil } // GetOutputItem reads a single evaluated row. diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/generation.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/generation.go index 142183e8883..f0edca1ada0 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/generation.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/generation.go @@ -205,7 +205,10 @@ func IsBuiltinEvaluator(name string) bool { // built-in lists. func SplitEvaluators(evaluators evalcore.EvaluatorList) (generated, builtin evalcore.EvaluatorList) { for _, e := range evaluators { - if IsBuiltinEvaluator(e.Name) { + // Name labels the criterion in results and is empty for a plain + // `- evaluator: builtin.coherence`, so testing it classified every + // built-in as generated. Evaluator is the reference IsBuiltin reads. + if e.IsBuiltin() { builtin = append(builtin, e) } else { generated = append(generated, e) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/operations.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/operations.go index 6ea57290636..48138d4aa89 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/operations.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/operations.go @@ -265,9 +265,9 @@ func (p publishedVersion) writtenAt() time.Time { // // For a few seconds after a publish the service can answer the next one with // the version it just assigned, writing over that version's contents instead -// of adding one. It is a race rather than a fixed window — a second publish +// of adding one. It is a race rather than a fixed window -- a second publish // has been seen both colliding a quarter of a second later and succeeding -// immediately — and nothing observable marks its end. +// immediately -- and nothing observable marks its end. // // That matters because versions are the unit an eval binds to. `evaluator // create` followed by `evaluator update`, which is what a first authoring @@ -446,7 +446,12 @@ func collectPages( ) error { gathered := 0 after := "" - for { + // A cursor that keeps returning rows while pointing back at itself would + // spin forever, holding the command open and growing the slice until the + // process dies. The next-link walker in pages.go bounds itself the same + // way; the cursor listings simply never did. + seen := map[string]bool{} + for page := 0; page < maxPages; page++ { query := map[string]string{} if limit > 0 { query["limit"] = strconv.Itoa(limit - gathered) @@ -467,8 +472,15 @@ func collectPages( if limit > 0 && gathered >= limit { return nil } + if seen[lastID] { + log.Printf("[eval_api] cursor %q repeated; the listing may be incomplete", lastID) + return nil + } + seen[lastID] = true after = lastID } + log.Printf("[eval_api] stopped after %d pages; the listing may be incomplete", maxPages) + return nil } // GetOpenAIEval gets an OpenAI eval definition. @@ -485,7 +497,7 @@ func (c *EvalClient) DeleteOpenAIEval(ctx context.Context, evalID string) error } // UpdateOpenAIEval edits an eval in place. The route is a POST on the eval -// itself, matching how this surface spells run cancel — there is no PATCH verb +// itself, matching how this surface spells run cancel -- there is no PATCH verb // here. // // Only what UpdateEvalParametersBody reaches is editable: name, metadata and diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/portal_urls_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/portal_urls_test.go index 602b584ccd5..971331cb9b5 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/portal_urls_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/portal_urls_test.go @@ -113,17 +113,21 @@ func TestIsBuiltinEvaluator(t *testing.T) { } func TestSplitEvaluators(t *testing.T) { + // `- evaluator: builtin.coherence` fills Evaluator and leaves Name empty. + // Name is the criterion label in results, so a fixture that put the + // reference there agreed with the bug rather than with any real config. generated, builtin := SplitEvaluators(evalcore.EvaluatorList{ - {Name: "builtin.coherence"}, - {Name: "support-quality"}, - {Name: "builtin.task_adherence"}, + {Evaluator: "builtin.coherence"}, + {Evaluator: "support-quality"}, + {Evaluator: "builtin.task_adherence", Name: "adherence"}, }) require.Len(t, generated, 1) - assert.Equal(t, "support-quality", generated[0].Name) + assert.Equal(t, "support-quality", generated[0].Evaluator) require.Len(t, builtin, 2) - assert.Equal(t, "builtin.coherence", builtin[0].Name) - assert.Equal(t, "builtin.task_adherence", builtin[1].Name) + assert.Equal(t, "builtin.coherence", builtin[0].Evaluator) + assert.Equal(t, "builtin.task_adherence", builtin[1].Evaluator, + "a criterion label does not stop a built-in being a built-in") } // Both halves come back nil rather than empty for an empty input, so a caller From a7ee69f5814c6141b9071b5dc6533155353c1524 Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 18 Aug 2026 00:58:08 +0300 Subject: [PATCH 244/320] Satisfy the linters the first push tripped go fix prefers a range over an unused counter; gosec reads the test server echoing its own request path back as a taint sink; and the spell checker does not carry two of the words the comments used. --- .../internal/cmd/dataset_presence_test.go | 5 ++++- .../extensions/azure.ai.evaluations/internal/cmd/debug.go | 2 +- .../internal/foundry/projectctx/hosted_absence_test.go | 2 +- .../azure.ai.evaluations/internal/pkg/dataset_api/list.go | 4 ++-- .../azure.ai.evaluations/internal/pkg/eval_api/evaluators.go | 2 +- .../azure.ai.evaluations/internal/pkg/eval_api/operations.go | 2 +- 6 files changed, 10 insertions(+), 7 deletions(-) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset_presence_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset_presence_test.go index 5cc190d45db..a4fc77ac966 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset_presence_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset_presence_test.go @@ -61,7 +61,10 @@ func newPresenceClient( version := r.URL.Path[strings.LastIndex(r.URL.Path, "/")+1:] if found[version] { w.WriteHeader(http.StatusOK) - _, _ = w.Write([]byte(`{"name":"ds","version":"` + version + `"}`)) + // A fixed body: datasetPresence reads only whether the point + // read succeeded, and echoing the request path back would make + // this a taint sink for no benefit. + _, _ = w.Write([]byte(`{"name":"ds"}`)) return } http.Error(w, `{"error":{"code":"NotFound"}}`, http.StatusNotFound) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/debug.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/debug.go index 197e3feadad..edb3c0ef6c7 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/debug.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/debug.go @@ -33,7 +33,7 @@ func setupDebugLogging(flags *pflag.FlagSet) func() { // scaffolded .gitignore does not cover this name, and a routine `git add -A` // committed one. The private subdirectory matters because the temp directory // is shared on Linux -- at a predictable path another user could leave a - // symlink and have the HTTP trace appended to a file of their choosing. + // symbolic link and have the HTTP trace appended to a file of their choosing. logDir := filepath.Join(os.TempDir(), "azd-ai-eval") if err := os.MkdirAll(logDir, 0o700); err != nil { logDir = os.TempDir() diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/foundry/projectctx/hosted_absence_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/foundry/projectctx/hosted_absence_test.go index 096b4e33655..67aeeca57d9 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/foundry/projectctx/hosted_absence_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/foundry/projectctx/hosted_absence_test.go @@ -39,7 +39,7 @@ func TestUnansweredHostedSourcesLetTheCascadeCarryOn(t *testing.T) { // and nothing would have said so. func TestARefusedReadIsReportedRatherThanSkipped(t *testing.T) { for name, err := range map[string]error{ - "expired login": status.Error(codes.Unauthenticated, "reauthentication required"), + "expired login": status.Error(codes.Unauthenticated, "the login has expired"), "not allowed": status.Error(codes.PermissionDenied, "forbidden"), "the user hit ctrl-c": status.Error(codes.Canceled, "context canceled"), "the read timed out": status.Error(codes.DeadlineExceeded, "deadline exceeded"), diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/list.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/list.go index 0396241c53c..ab2b389b1d9 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/list.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/list.go @@ -99,8 +99,8 @@ func VersionGreater(a, b string) bool { func LatestVersion(datasets []Dataset) string { best := "" // VersionOrder returns -1 for anything it cannot order, so the sentinel has - // to be -1 rather than lower: below it, the first unorderable version becomes - // the running best and the fallback below never runs. + // to be -1 rather than lower: below it, the first version it cannot order + // becomes the running best and the fallback below never runs. bestOrder := -1.0 for _, d := range datasets { if o := VersionOrder(d.Version); o > bestOrder { diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/evaluators.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/evaluators.go index dde85a6e179..40df934ee14 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/evaluators.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/evaluators.go @@ -271,7 +271,7 @@ func (c *EvalClient) ListOutputItems( // grow all.Data until the process dies, so the walk is bounded the same way // the next-link walker in pages.go is. seen := map[string]bool{} - for page := 0; page < maxPages; page++ { + for range maxPages { query := map[string]string{} if limit > 0 { query["limit"] = strconv.Itoa(limit - len(all.Data)) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/operations.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/operations.go index 48138d4aa89..3be46995945 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/operations.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/operations.go @@ -451,7 +451,7 @@ func collectPages( // process dies. The next-link walker in pages.go bounds itself the same // way; the cursor listings simply never did. seen := map[string]bool{} - for page := 0; page < maxPages; page++ { + for range maxPages { query := map[string]string{} if limit > 0 { query["limit"] = strconv.Itoa(limit - gathered) From 3c78a779e957757c8ed036087aec434220f748c4 Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 18 Aug 2026 01:02:58 +0300 Subject: [PATCH 245/320] Name the codes that mean absence, rather than the ones that do not Inverting the test let Internal, ResourceExhausted and DataLoss read as "nothing here", so a daemon that broke resolved quietly to a lower-priority endpoint that can belong to a different project. Unavailable, NotFound and Unknown are the three that mean absence. Unknown is the one that is not obvious: azd answers its ordinary absences -- no default environment, no such key -- with plain Go errors that carry no status, and grpc encodes those as Unknown. --- .../foundry/projectctx/hosted_absence_test.go | 37 +++++++++++-------- .../internal/foundry/projectctx/resolver.go | 28 +++++++------- 2 files changed, 36 insertions(+), 29 deletions(-) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/foundry/projectctx/hosted_absence_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/foundry/projectctx/hosted_absence_test.go index 67aeeca57d9..6c20983c26d 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/foundry/projectctx/hosted_absence_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/foundry/projectctx/hosted_absence_test.go @@ -20,10 +20,9 @@ import ( // reaching the global config or the host variable. func TestUnansweredHostedSourcesLetTheCascadeCarryOn(t *testing.T) { for name, err := range map[string]error{ - "no daemon at all": status.Error(codes.Unavailable, "connection refused"), - "nothing under that key": status.Error(codes.NotFound, "key not found"), - "no default environment": status.Error(codes.Unknown, "default environment not found"), - "a plain error, unwrapped": errors.New("default environment not found"), + "no daemon at all": status.Error(codes.Unavailable, "connection refused"), + "nothing under that key": status.Error(codes.NotFound, "key not found"), + "no default environment": status.Error(codes.Unknown, "default environment not found"), "wrapped in context": fmt.Errorf("reading the environment: %w", status.Error(codes.Unknown, "default environment not found")), } { @@ -34,21 +33,29 @@ func TestUnansweredHostedSourcesLetTheCascadeCarryOn(t *testing.T) { } } -// A daemon that refused is not a daemon with nothing to say. Falling through -// here would resolve to a lower-priority endpoint belonging to another project, -// and nothing would have said so. -func TestARefusedReadIsReportedRatherThanSkipped(t *testing.T) { +// A daemon that refused, or one that broke, is not a daemon with nothing to +// say. Falling through here would resolve to a lower-priority endpoint that can +// belong to a different project, and nothing would have said so. +func TestAFailureToAnswerIsReportedRatherThanSkipped(t *testing.T) { for name, err := range map[string]error{ - "expired login": status.Error(codes.Unauthenticated, "the login has expired"), - "not allowed": status.Error(codes.PermissionDenied, "forbidden"), - "the user hit ctrl-c": status.Error(codes.Canceled, "context canceled"), - "the read timed out": status.Error(codes.DeadlineExceeded, "deadline exceeded"), - "wrapped expiry": fmt.Errorf("reading the environment: %w", status.Error(codes.Unauthenticated, "expired")), - "nested twice over": fmt.Errorf("outer: %w", fmt.Errorf("inner: %w", status.Error(codes.PermissionDenied, "no"))), + "the login has expired": status.Error(codes.Unauthenticated, "the login has expired"), + "not allowed": status.Error(codes.PermissionDenied, "forbidden"), + "the user hit ctrl-c": status.Error(codes.Canceled, "context canceled"), + "the read timed out": status.Error(codes.DeadlineExceeded, "deadline exceeded"), + "the daemon broke": status.Error(codes.Internal, "internal error"), + "the daemon is full": status.Error(codes.ResourceExhausted, "quota exceeded"), + "the answer was corrupt": status.Error(codes.DataLoss, "data loss"), + // A bare error carries no status at all, so it never travelled the wire + // as an absence the daemon reported. + "not a status at all": errors.New("something local went wrong"), + "wrapped expiry": fmt.Errorf("reading the environment: %w", + status.Error(codes.Unauthenticated, "expired")), + "nested twice over": fmt.Errorf("outer: %w", + fmt.Errorf("inner: %w", status.Error(codes.PermissionDenied, "no"))), } { t.Run(name, func(t *testing.T) { assert.False(t, hostedSourceAbsent(err), - "a refusal has to surface, not resolve to a different project") + "a failure to answer has to surface, not resolve to a different project") }) } } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/foundry/projectctx/resolver.go b/cli/azd/extensions/azure.ai.evaluations/internal/foundry/projectctx/resolver.go index 0e131334836..db8208679ba 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/foundry/projectctx/resolver.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/foundry/projectctx/resolver.go @@ -81,22 +81,22 @@ func readAzdHostedSources(ctx context.Context) (AzdHostedSources, error) { // hostedSourceAbsent reports whether an error from the azd daemon leaves the // cascade free to carry on to the next level. // -// azd answers the ordinary "nothing here" cases -- no default environment, no -// such key -- with plain Go errors, which its interceptor passes through -// untouched and grpc then encodes as Unknown. Unknown therefore has to read as -// absence: treating it as a failure would stop a project that simply has no -// environment selected from ever reaching the global config or the host -// variable, which is the whole point of the levels below. +// Three codes mean "nothing here". Unavailable is no daemon at all. NotFound is +// a daemon with nothing under that name. Unknown is the one that is not +// obvious: azd answers the ordinary absences -- no default environment, no such +// key -- with plain Go errors, its interceptor passes those through untouched, +// and grpc encodes an error carrying no status as Unknown. Without it, a +// project that simply has no environment selected could never reach the global +// config or the host variable. // -// What must not read as absence is a daemon that refused to answer, or a read -// that never finished. An expired login is mapped to Unauthenticated, and a -// Ctrl-C arrives as Canceled; falling through on either would resolve quietly -// to some other project's endpoint. +// Everything else is a failure to answer rather than an answer of "nothing": +// an expired login, a denial, a cancellation, or a server fault. Falling +// through on any of those would resolve quietly to a lower-priority endpoint +// that can belong to a different project. func hostedSourceAbsent(err error) bool { - return !containsGRPCCode(err, codes.Unauthenticated) && - !containsGRPCCode(err, codes.PermissionDenied) && - !containsGRPCCode(err, codes.Canceled) && - !containsGRPCCode(err, codes.DeadlineExceeded) + return containsGRPCCode(err, codes.Unavailable) || + containsGRPCCode(err, codes.NotFound) || + containsGRPCCode(err, codes.Unknown) } // containsGRPCCode walks the error chain looking for a gRPC status with the From ab485984bf28b4ca0c169a70e4299187adbe798e Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 18 Aug 2026 01:14:55 +0300 Subject: [PATCH 246/320] Gate only a run that has stopped moving The spec puts --fail-on on the commands that wait, but `run show --fail-on` applied the gate whether or not --wait was given. The counts of a run still in progress are partial, so the threshold could fail a run that would have passed, or trip GateNoResultCounts before the service had populated anything. Skipping the gate instead would be worse: a pipeline would believe it was protected when it was not. So the ambiguous case is refused, and the message names --wait as the way to gate once the run finishes. --- .../internal/cmd/gating_silent_test.go | 23 +++++++++++++++++++ .../azure.ai.evaluations/internal/cmd/run.go | 16 +++++++++++++ .../internal/cmd/run_ops.go | 7 ++++++ .../internal/messages/messages.go | 12 ++++++++++ 4 files changed, 58 insertions(+) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/gating_silent_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/gating_silent_test.go index 930f205da14..7b1699a6828 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/gating_silent_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/gating_silent_test.go @@ -6,6 +6,7 @@ package cmd import ( "testing" + "azureaieval/internal/messages" "azureaieval/internal/pkg/eval_api" "github.com/stretchr/testify/assert" @@ -72,3 +73,25 @@ func TestGatesStillJudgeRunsThatScoredSomething(t *testing.T) { // Counts the service never sent are not a pass. assert.NotEmpty(t, rate.breach(nil)) } + +// The spec puts --fail-on on the commands that wait. A run still in progress +// has partial counts, so gating it can fail a run that would have passed -- +// and silently skipping the gate would leave a pipeline believing it is +// protected when it is not. +func TestOnlyATerminalRunCanBeGated(t *testing.T) { + for _, status := range []string{"completed", "failed", "canceled", "cancelled", ""} { + assert.Truef(t, runIsTerminal(&eval_api.OpenAIEvalRun{Status: status}), + "%q has stopped moving, so its counts are final", status) + } + + for _, status := range []string{"in_progress", "queued", "running"} { + assert.Falsef(t, runIsTerminal(&eval_api.OpenAIEvalRun{Status: status}), + "%q is still moving, so its counts are partial", status) + } + + assert.False(t, runIsTerminal(nil), "no run is not a finished run") + + err := messages.GateNeedsATerminalRun("evalrun_1", "in_progress") + require.Error(t, err) + assert.Contains(t, err.Error(), "--wait", "the way out has to be named") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go index d1d678917b3..aeae5795e0d 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go @@ -54,6 +54,22 @@ func runCompleted(run *eval_api.OpenAIEvalRun) error { return messages.RunFinishedWithStatus(run.ID, run.Status) } +// runIsTerminal reports whether the run has stopped moving. +// +// A gate read from a run still in progress is read from partial counts: it can +// fail a run that would have passed, and it can pass one that has not finished +// failing. +func runIsTerminal(run *eval_api.OpenAIEvalRun) bool { + if run == nil { + return false + } + switch strings.ToLower(run.Status) { + case "completed", "failed", "canceled", "cancelled", "": + return true + } + return false +} + // newRunCommand builds the run group. // // `run` is a group, not an executable verb: once `run output` exists, a bare diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go index 8edaecb0b48..a3fe813e2c9 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go @@ -159,6 +159,13 @@ func newRunShowCommand() *cobra.Command { } } + // The spec puts --fail-on on the commands that wait. Gating a run + // that is still moving would read partial counts; ignoring the flag + // would leave a pipeline believing it is gated when it is not. + if threshold.set && !runIsTerminal(run) { + return messages.GateNeedsATerminalRun(run.ID, run.Status) + } + if isJSON(cmd) { if err := emitJSON(cmd.OutOrStdout(), run); err != nil { return err diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go b/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go index b4f8f4bfad3..585184f77b5 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go @@ -1610,6 +1610,18 @@ func EvaluatorRefMalformed(ref string) error { "them with commas, and use builtin. for a built-in", ref) } +// GateNeedsATerminalRun refuses to gate a run that is still moving. +// +// The counts are partial until the run stops, so a threshold read from them +// can fail a run that would have passed. Ignoring the flag instead would leave +// a pipeline believing it is gated when it is not. +func GateNeedsATerminalRun(runID, status string) error { + return fmt.Errorf( + "run %s is %s, so --fail-on has only partial results to judge: "+ + "add --wait to gate once it finishes", + runID, status) +} + // AmbiguousJudgeModel reports several deployments where only one can be used. func AmbiguousJudgeModel(models []string) error { return fmt.Errorf( From 281a7b5aa97774de5d74dfe7276e9639c19f62db Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 18 Aug 2026 02:05:05 +0300 Subject: [PATCH 247/320] Bump to 1.0.7-beta for the second bug bash build --- cli/azd/extensions/azure.ai.evaluations/CHANGELOG.md | 2 +- cli/azd/extensions/azure.ai.evaluations/extension.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cli/azd/extensions/azure.ai.evaluations/CHANGELOG.md b/cli/azd/extensions/azure.ai.evaluations/CHANGELOG.md index f793496a38d..eeab4b11adf 100644 --- a/cli/azd/extensions/azure.ai.evaluations/CHANGELOG.md +++ b/cli/azd/extensions/azure.ai.evaluations/CHANGELOG.md @@ -1,6 +1,6 @@ # Release History -## 1.0.4-beta (Unreleased) +## 1.0.7-beta (Unreleased) First release of the Foundry evaluations extension. diff --git a/cli/azd/extensions/azure.ai.evaluations/extension.yaml b/cli/azd/extensions/azure.ai.evaluations/extension.yaml index 2ba31a56d9b..da14c489457 100644 --- a/cli/azd/extensions/azure.ai.evaluations/extension.yaml +++ b/cli/azd/extensions/azure.ai.evaluations/extension.yaml @@ -5,7 +5,7 @@ displayName: Foundry evaluations (Beta) description: Define and run Foundry evaluations from your terminal. (Beta) usage: azd ai eval [options] # NOTE: Make sure version.txt is in sync with this version. -version: 1.0.4-beta +version: 1.0.7-beta requiredAzdVersion: ">=1.27.1" language: go capabilities: From 1a9418ee77a4f674d8188c82b78d870b8c2b826e Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 18 Aug 2026 02:21:18 +0300 Subject: [PATCH 248/320] Stop pflag reading a backquoted word as the flag placeholder The usage text for --path referred to the path "init" used, in backticks, and pflag takes a backquoted word in a usage string as the value name. So the flag rendered as "--path init" rather than "--path string". Also syncs version.txt with the manifest. The build stamps the binary from version.txt while the registry reads extension.yaml, so a drift ships a binary that misreports its own version -- which the manifest test exists to catch and just did. --- .../azure.ai.evaluations/internal/cmd/run_output.go | 4 +++- cli/azd/extensions/azure.ai.evaluations/version.txt | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output.go index fd98947745c..f942c983fb3 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output.go @@ -320,8 +320,10 @@ func addEvalFlag(cmd *cobra.Command, target *string) { // distinguishable from "given the default", which is what lets the path `init` // recorded take effect in between. func addEvalPathFlag(cmd *cobra.Command, target *string) { + // No backticks: pflag reads a backquoted word as the value placeholder, so + // `init` rendered the flag as "--path init" instead of "--path string". cmd.Flags().StringVar(target, "path", "", - "Directory holding azure.eval.yaml. Defaults to the path `init` used, then ./evals.") + "Directory holding azure.eval.yaml. Defaults to the path init used, then ./evals.") } // evalPathFlag reads --path from whichever command is resolving a declared diff --git a/cli/azd/extensions/azure.ai.evaluations/version.txt b/cli/azd/extensions/azure.ai.evaluations/version.txt index 12505f27e9d..33955bc06df 100644 --- a/cli/azd/extensions/azure.ai.evaluations/version.txt +++ b/cli/azd/extensions/azure.ai.evaluations/version.txt @@ -1 +1 @@ -1.0.4-beta +1.0.7-beta \ No newline at end of file From 11a7c100dc53c61064284534b19b8f80991ea1bb Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 18 Aug 2026 02:28:52 +0300 Subject: [PATCH 249/320] Use a word the spell checker carries --- .../azure.ai.evaluations/internal/cmd/run_output.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output.go index f942c983fb3..c7021ae31eb 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output.go @@ -320,8 +320,8 @@ func addEvalFlag(cmd *cobra.Command, target *string) { // distinguishable from "given the default", which is what lets the path `init` // recorded take effect in between. func addEvalPathFlag(cmd *cobra.Command, target *string) { - // No backticks: pflag reads a backquoted word as the value placeholder, so - // `init` rendered the flag as "--path init" instead of "--path string". + // No backticks: pflag reads a word in back quotes as the value placeholder, + // so `init` rendered the flag as "--path init" instead of "--path string". cmd.Flags().StringVar(target, "path", "", "Directory holding azure.eval.yaml. Defaults to the path init used, then ./evals.") } From 711eb2af7c5acc15dc06a89223757c49e06cf635 Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 18 Aug 2026 03:35:35 +0300 Subject: [PATCH 250/320] Make show a definition, and name the operation that failed `eval show` is documented as showing an eval definition and answered with the id, the name and who created it -- true of any eval, and none of the definition. The service returns data_source_config and testing_criteria on a read; the model simply did not decode them. It now shows where the rows come from and what grades them. A failed job delete reported "reading dataset generation job", which sends the reader looking for a read that never happened. The shared helper now takes the operation, so delete and cancel say so. --- .../internal/cmd/eval_group.go | 47 +++++++++++++++- .../internal/cmd/eval_show_test.go | 55 +++++++++++++++++++ .../internal/cmd/helpers_test.go | 23 +++++++- .../azure.ai.evaluations/internal/cmd/job.go | 16 ++++-- .../internal/messages/messages.go | 7 +++ .../internal/pkg/eval_api/models.go | 5 ++ 6 files changed, 145 insertions(+), 8 deletions(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/eval_show_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/eval_group.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/eval_group.go index 12b6bffcf14..de0a735b7f3 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/eval_group.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/eval_group.go @@ -6,6 +6,7 @@ package cmd import ( "fmt" "path/filepath" + "strings" "azureaieval/internal/messages" "azureaieval/internal/pkg/eval_api" @@ -224,7 +225,7 @@ func newEvalShowCommand() *cobra.Command { if isJSON(cmd) { return emitJSON(cmd.OutOrStdout(), group) } - return emitDetail(cmd.OutOrStdout(), []field{ + detail := []field{ {"Id", group.ID}, {"Name", group.Name}, // CreatedAt is `any` because the service sends epoch seconds here @@ -232,7 +233,16 @@ func newEvalShowCommand() *cobra.Command { // in scientific notation. {"Created", timestampString(group.CreatedAt)}, {"Created By", group.CreatedBy}, - }) + } + // Without these the command answers "does this id exist", which is + // not what a definition is, nor what its own help promises. + if source := evalSourceType(group); source != "" { + detail = append(detail, field{"Source", source}) + } + if graders := evalGraders(group); graders != "" { + detail = append(detail, field{"Evaluators", graders}) + } + return emitDetail(cmd.OutOrStdout(), detail) }, } @@ -240,6 +250,39 @@ func newEvalShowCommand() *cobra.Command { return cmd } +// evalSourceType reports where the eval's rows come from, as the service +// records it. Empty when the service sent no data source config. +func evalSourceType(group *eval_api.OpenAIEval) string { + if group == nil || group.DataSourceConfig == nil { + return "" + } + kind, _ := group.DataSourceConfig["type"].(string) + return kind +} + +// evalGraders lists the evaluators the eval grades with, preferring the +// reference a caller would recognize over the criterion label. +func evalGraders(group *eval_api.OpenAIEval) string { + if group == nil { + return "" + } + names := make([]string, 0, len(group.TestingCriteria)) + for _, c := range group.TestingCriteria { + name := c.EvaluatorName + if name == "" { + name = c.Name + } + if name == "" { + continue + } + if c.EvaluatorVersion != "" { + name += " (" + c.EvaluatorVersion + ")" + } + names = append(names, name) + } + return strings.Join(names, ", ") +} + func newEvalDeleteCommand() *cobra.Command { var endpointFlg string diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/eval_show_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/eval_show_test.go new file mode 100644 index 00000000000..3a86ab795b3 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/eval_show_test.go @@ -0,0 +1,55 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "testing" + + "azureaieval/internal/pkg/eval_api" + + "github.com/stretchr/testify/assert" +) + +// `show` is documented as showing an eval definition, and answered with the id, +// the name and who created it -- true of any eval, and none of the definition. +// The service returns both of these on a read. +func TestShowSurfacesWhatTheEvalGrades(t *testing.T) { + group := &eval_api.OpenAIEval{ + ID: "eval_1", + Name: "support-trace-eval", + DataSourceConfig: map[string]any{"type": "custom"}, + TestingCriteria: []eval_api.TestingCriterion{ + {Name: "task_adherence", EvaluatorName: "builtin.task_adherence"}, + {Name: "quality", EvaluatorName: "support-quality", EvaluatorVersion: "2"}, + }, + } + + assert.Equal(t, "custom", evalSourceType(group)) + assert.Equal(t, "builtin.task_adherence, support-quality (2)", evalGraders(group)) +} + +// The criterion label is what the service echoes when no evaluator reference +// was recorded, so it is better than printing nothing. +func TestShowFallsBackToTheCriterionLabel(t *testing.T) { + group := &eval_api.OpenAIEval{ + TestingCriteria: []eval_api.TestingCriterion{{Name: "custom-grader"}}, + } + + assert.Equal(t, "custom-grader", evalGraders(group)) +} + +// An older eval, or one the service answers without a definition, still shows +// its identity rather than blank rows. +func TestShowOmitsWhatTheServiceDidNotSend(t *testing.T) { + assert.Empty(t, evalSourceType(&eval_api.OpenAIEval{ID: "eval_1"})) + assert.Empty(t, evalGraders(&eval_api.OpenAIEval{ID: "eval_1"})) + assert.Empty(t, evalSourceType(nil)) + assert.Empty(t, evalGraders(nil)) + + // A criterion carrying no name at all contributes nothing rather than an + // empty entry with a stray separator. + assert.Empty(t, evalGraders(&eval_api.OpenAIEval{ + TestingCriteria: []eval_api.TestingCriterion{{Type: "azure_ai_evaluator"}}, + })) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/helpers_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/helpers_test.go index 4654145e853..806812b409f 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/helpers_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/helpers_test.go @@ -4,11 +4,13 @@ package cmd import ( + "net/http" "strings" "testing" "azureaieval/internal/project" + "github.com/Azure/azure-sdk-for-go/sdk/azcore" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -139,7 +141,7 @@ func TestDefaultEvalName(t *testing.T) { // group is a command that returns "not found". func TestJobLookupErrorNamesTheGroup(t *testing.T) { for _, kind := range []jobKind{datasetJobs, evaluatorJobs} { - err := jobLookupError(kind, "job_1", assert.AnError) + err := jobLookupError("reading", kind, "job_1", assert.AnError) require.Error(t, err) assert.Contains(t, err.Error(), "job_1") @@ -147,3 +149,22 @@ func TestJobLookupErrorNamesTheGroup(t *testing.T) { "the error must name the %q group so the retry goes to the right one", kind.name) } } + +// A failed delete used to report that a read failed, which sends the reader +// looking for a read that never happened. +func TestJobLookupErrorNamesWhatWasAttempted(t *testing.T) { + deleteErr := jobLookupError("deleting", datasetJobs, "job_1", assert.AnError) + require.Error(t, deleteErr) + assert.Contains(t, deleteErr.Error(), "deleting") + assert.NotContains(t, deleteErr.Error(), "reading") + + cancelErr := jobLookupError("cancelling", datasetJobs, "job_1", assert.AnError) + require.Error(t, cancelErr) + assert.Contains(t, cancelErr.Error(), "cancelling") + + // A genuine 404 still points at the sibling group whatever the verb was. + notFound := jobLookupError("deleting", datasetJobs, "job_1", + &azcore.ResponseError{StatusCode: http.StatusNotFound}) + require.Error(t, notFound) + assert.Contains(t, notFound.Error(), jobKindEvaluator) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/job.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/job.go index 62c31a95b22..24a911cea86 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/job.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/job.go @@ -201,7 +201,7 @@ func newJobShowCommand() *cobra.Command { job, err := kind.get(ctx, ec, jobID) if err != nil { - return jobLookupError(kind, jobID, err) + return jobLookupError("reading", kind, jobID, err) } if isJSON(cmd) { @@ -244,7 +244,7 @@ func newJobCancelCommand() *cobra.Command { canceled, err := kind.cancel(ctx, ec, jobID) if err != nil { - return jobLookupError(kind, jobID, err) + return jobLookupError("cancelling", kind, jobID, err) } if isJSON(cmd) { @@ -287,7 +287,7 @@ func newJobDeleteCommand() *cobra.Command { defer ec.Close() if err := kind.remove(ctx, ec, jobID); err != nil { - return jobLookupError(kind, jobID, err) + return jobLookupError("deleting", kind, jobID, err) } if isJSON(cmd) { @@ -307,7 +307,10 @@ func newJobDeleteCommand() *cobra.Command { // jobLookupError names the sibling group, because the two job types share an id // shape and reaching for the wrong one is the likely mistake. -func jobLookupError(kind jobKind, jobID string, err error) error { +// +// action is what the caller was doing, so a failed delete does not report that +// a read failed. +func jobLookupError(action string, kind jobKind, jobID string, err error) error { if eval_api.IsNotFound(err) { other := jobKindEvaluator if kind.name == jobKindEvaluator { @@ -315,5 +318,8 @@ func jobLookupError(kind jobKind, jobID string, err error) error { } return messages.JobNotFound(kind.name, jobID, other) } - return messages.ReadingJob(kind.name, jobID, err) + if action == "reading" { + return messages.ReadingJob(kind.name, jobID, err) + } + return messages.JobActionFailed(action, kind.name, jobID, err) } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go b/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go index 585184f77b5..7c0f05390b7 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go @@ -831,6 +831,13 @@ func ReadingJob(kind, jobID string, err error) error { return fmt.Errorf("reading %s generation job %s: %w", kind, jobID, err) } +// JobActionFailed reports a job operation that was not a read, so the sentence +// names what was attempted. A delete that reports "reading" sends the reader +// looking for a read that never happened. +func JobActionFailed(action, kind, jobID string, err error) error { + return fmt.Errorf("%s %s generation job %s: %w", action, kind, jobID, err) +} + // JobFailedWithReason reports a polled job that failed and said why. func JobFailedWithReason(status, message string) string { return fmt.Sprintf("job failed with status %q: %s", status, message) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/models.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/models.go index 9cc820c3bef..d0109405362 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/models.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/models.go @@ -278,6 +278,11 @@ type OpenAIEval struct { ModifiedAt any `json:"modified_at,omitempty"` CreatedBy string `json:"created_by,omitempty"` Metadata map[string]string `json:"metadata,omitempty"` + // The service returns these on a read, and they are what makes `show` a + // definition rather than an id: what the eval grades, and where its rows + // come from. Both are fixed at creation. + DataSourceConfig map[string]any `json:"data_source_config,omitempty"` + TestingCriteria []TestingCriterion `json:"testing_criteria,omitempty"` } // OpenAIEvalList is the response for listing OpenAI eval definitions. From 594c1fa5cf94013145df521257680d20d8274c6a Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 18 Aug 2026 04:58:29 +0300 Subject: [PATCH 251/320] Print one run link, and stop claiming a connection init never saw The service sends report_url and the extension builds its own portal URL for the same page, so a run carried two labels for one destination. One link now, the service's when it sent one. Separately, init defaults --source from the Application Insights connection recorded in the azd environment, which is the signal generate --from already uses and a local read rather than a service call, and it only names Application Insights when it found one. --- .../azure.ai.evaluations/internal/cmd/init.go | 17 ++++++-- .../internal/cmd/init_model.go | 17 +++++++- .../internal/cmd/output.go | 14 +++++++ .../internal/cmd/portal_test.go | 39 ++++++++++++++++--- .../azure.ai.evaluations/internal/cmd/run.go | 5 +-- .../internal/cmd/run_ops.go | 5 +-- .../internal/cmd/run_output.go | 4 +- .../internal/messages/messages.go | 14 ++++++- .../internal/messages/trace_source_test.go | 28 +++++++++++++ 9 files changed, 124 insertions(+), 19 deletions(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/messages/trace_source_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go index 722b05df970..6375e1da3ba 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go @@ -76,7 +76,15 @@ func newInitCommand() *cobra.Command { return err } if source == "" { - source = initSourceDataset + // The same signal `generate --from` defaults on, read from the azd + // environment rather than the service, so init still makes no + // service calls. Traces are real conversations; a project wired to + // collect them should not have to ask for them by flag. + if tracesConnected(commandContext(cmd)) { + source = initSourceTraces + } else { + source = initSourceDataset + } } if path == "" { path = project.DefaultEvalDir @@ -224,7 +232,9 @@ func newInitCommand() *cobra.Command { fmt.Fprint(out, messages.DetectedTarget(target)) if source == initSourceTraces { - fmt.Fprint(out, messages.UsingTraceSource()) + // Claiming the connection is only honest when it was found. init + // makes no service calls, so it cannot verify one it did not see. + fmt.Fprint(out, messages.UsingTraceSource(tracesConnected(commandContext(cmd)))) } // Only what was settled without asking: a reader who just picked // from a list does not need it read back to them. @@ -261,7 +271,8 @@ func newInitCommand() *cobra.Command { cmd.Flags().StringVar(&target, "target", "", "Name of the agent to evaluate. Detected when the project has one agent; prompts when it has several.") cmd.Flags().StringVar(&source, "source", "", - "Where rows come from: dataset or traces. Defaults to dataset.") + "Where rows come from: dataset or traces. Defaults to traces when the azd "+ + "environment records an Application Insights connection, otherwise dataset.") cmd.Flags().StringVar(&dataset, "dataset", "", "Path to a local .jsonl, or the name of a registered dataset.") cmd.Flags().IntVar(&maxTraces, "max-traces", project.DefaultScaffoldMaxTraces, diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_model.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_model.go index a93084d5945..ab55ff8bcfd 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_model.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_model.go @@ -97,10 +97,25 @@ func resolveJudgeModel(cmd *cobra.Command, proj *azdext.ProjectConfig) (string, return promptJudgeModel(cmd, deployments) } +// tracesConnected reports whether the azd environment records an Application +// Insights connection, which is what `generate --from` already defaults on. +// +// A local read, so `init` keeps its promise to make no service calls. Absence +// is ordinary: init runs outside an azd project too. +func tracesConnected(ctx context.Context) bool { + return azdEnvValue(ctx, appInsightsEnvKey) != "" +} + // modelDeploymentFromAzdEnv reads the deployment `azd ai agent init` recorded // in the active azd environment. Absence is ordinary: `init` runs outside an // azd project too, and the caller falls back to naming --judge-model. func modelDeploymentFromAzdEnv(ctx context.Context) string { + return azdEnvValue(ctx, judgeModelEnvKey) +} + +// azdEnvValue reads one key from the active azd environment, answering empty +// whenever there is no daemon, no environment, or no such key. +func azdEnvValue(ctx context.Context, key string) string { azdClient, err := azdext.NewAzdClient() if err != nil { return "" @@ -113,7 +128,7 @@ func modelDeploymentFromAzdEnv(ctx context.Context) string { } val, err := azdClient.Environment().GetValue(ctx, &azdext.GetEnvRequest{ EnvName: envResp.Environment.Name, - Key: judgeModelEnvKey, + Key: key, }) if err != nil { return "" diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/output.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/output.go index db1bed539f5..aab83da518a 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/output.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/output.go @@ -35,6 +35,20 @@ func writePortalLink(w io.Writer, url string) { fmt.Fprint(w, messages.PortalLink(color.CyanString(url))) } +// runLink is the one link a run has. +// +// The service sends report_url and the extension builds its own portal URL, and +// the two resolve to the same page. Printing both put two labels on one +// destination with no rule a reader could infer, so the service's value wins and +// ours is the fallback that keeps the link from going missing. Callers format +// it themselves, because the three views that show it are laid out differently. +func runLink(reportURL, portalURL string) string { + if reportURL != "" { + return reportURL + } + return portalURL +} + // outputFormat reads the inherited -o/--output flag. func outputFormat(cmd *cobra.Command) string { if cmd == nil { diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/portal_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/portal_test.go index 2d40dfc8533..73e64e289c5 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/portal_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/portal_test.go @@ -81,7 +81,10 @@ func TestPortalRunURLShape(t *testing.T) { // The report line and the portal line are different things: one is the // service's own link, the other is built here. A run carrying both prints both. -func TestRenderRunPrintsReportAndPortalSeparately(t *testing.T) { +// A run has one destination. The service's report_url and the portal URL the +// extension builds resolve to the same page, and printing both put two labels +// on it with no rule a reader could infer. +func TestRenderRunPrintsOneLink(t *testing.T) { run := &eval_api.OpenAIEvalRun{ ID: "evalrun_1", Status: "completed", @@ -93,8 +96,34 @@ func TestRenderRunPrintsReportAndPortalSeparately(t *testing.T) { require.NoError(t, renderRun(&buf, run, nil)) out := buf.String() - assert.Contains(t, out, "Report: https://service.example/report/1") - assert.Contains(t, out, "Portal: ") - assert.Less(t, strings.Index(out, "Report:"), strings.Index(out, "Portal:"), - "the portal link closes the view") + assert.Contains(t, out, "Report: https://service.example/report/1", + "the service's url wins where it sent one") + assert.NotContains(t, out, "Portal: ", + "the second label named the same destination") + assert.NotContains(t, out, run.PortalURL) +} + +// Ours is the fallback, so a service that sends no report_url does not leave +// the reader with no way to open the run. +func TestRenderRunFallsBackToTheBuiltLink(t *testing.T) { + run := &eval_api.OpenAIEvalRun{ + ID: "evalrun_1", + Status: "completed", + PortalURL: "https://ai.azure.com/nextgen/r/x,y,,z,p/build/evaluations/e/run/r", + } + + var buf bytes.Buffer + require.NoError(t, renderRun(&buf, run, nil)) + + assert.Contains(t, buf.String(), "Report: "+run.PortalURL) +} + +// A run with neither prints no link rather than an empty label. +func TestRenderRunOmitsAnAbsentLink(t *testing.T) { + var buf bytes.Buffer + require.NoError(t, renderRun(&buf, &eval_api.OpenAIEvalRun{ + ID: "evalrun_1", Status: "completed", + }, nil)) + + assert.NotContains(t, buf.String(), "Report:") } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go index aeae5795e0d..0b1bf457815 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go @@ -849,10 +849,9 @@ func renderRun( } } - if run.ReportURL != "" { - fmt.Fprint(out, messages.ReportLink(run.ReportURL)) + if url := runLink(run.ReportURL, run.PortalURL); url != "" { + fmt.Fprint(out, messages.ReportLink(url)) } - writePortalLink(out, run.PortalURL) return nil } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go index a3fe813e2c9..1a0ff8e1d5f 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go @@ -186,10 +186,9 @@ func newRunShowCommand() *cobra.Command { if counts := summarizeCounts(run.ResultCounts); counts != "" { fmt.Fprint(out, messages.RunResultsLine(counts)) } - if run.ReportURL != "" { - fmt.Fprint(out, messages.RunReportLine(run.ReportURL)) + if url := runLink(run.ReportURL, run.PortalURL); url != "" { + fmt.Fprint(out, messages.RunReportLine(url)) } - writePortalLink(out, run.PortalURL) if gateOnStatus { if err := runCompleted(run); err != nil { return err diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output.go index c7021ae31eb..ae3f738cef5 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output.go @@ -568,8 +568,8 @@ func renderResults( } } - if run.ReportURL != "" { - fmt.Fprint(w, messages.ReportLinkAfterRows(run.ReportURL)) + if url := runLink(run.ReportURL, run.PortalURL); url != "" { + fmt.Fprint(w, messages.ReportLinkAfterRows(url)) } return nil } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go b/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go index 7c0f05390b7..c79a86de799 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go @@ -1652,8 +1652,18 @@ func SelectingJudgeModel(err error) error { } // UsingTraceSource reports a scaffold that reads production traces. -func UsingTraceSource() string { - return fmt.Sprintf("%s Using data source: traces (Application Insights)\n", DoneMark) +// +// Naming Application Insights is a claim about the project, so it is only made +// when a connection was actually found. `init` makes no service calls and +// cannot verify one it did not see. +func UsingTraceSource(connected bool) string { + if connected { + return fmt.Sprintf("%s Using data source: traces (Application Insights)\n", DoneMark) + } + return fmt.Sprintf( + "%s Using data source: traces. No Application Insights connection is recorded "+ + "in this environment, so the run finds rows only if the project has one\n", + DoneMark) } // JudgeModelDeployment reports the deployment the graders will judge with. diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/messages/trace_source_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/messages/trace_source_test.go new file mode 100644 index 00000000000..6a121f289dd --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/messages/trace_source_test.go @@ -0,0 +1,28 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package messages + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/assert" +) + +// `init` printed "Using data source: traces (Application Insights)" whether or +// not the project had one, so a scaffold that could not produce a sample was +// reported with a green check. init makes no service calls, so it cannot +// verify a connection it never saw. +func TestTraceSourceOnlyClaimsAConnectionItFound(t *testing.T) { + connected := UsingTraceSource(true) + assert.Contains(t, connected, "Application Insights") + + unverified := UsingTraceSource(false) + assert.Contains(t, unverified, "traces", + "the source is still what was chosen") + assert.NotContains(t, unverified, "(Application Insights)", + "naming the connection asserts something nobody checked") + assert.Truef(t, strings.Contains(unverified, "only if"), + "the reader has to learn the run may find no rows: %s", unverified) +} From c29eca1570a604e69f0a1a31cac155cd3ce0d3b3 Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 18 Aug 2026 05:13:08 +0300 Subject: [PATCH 252/320] Keep what a trace run was asked for azd emitted the legacy azure_ai_traces data source, which discards agent_version and start_time without a word and re-applies its own lookback. So an agent that had been redeployed was graded against whichever version the service chose, and a caller could not say otherwise. It now emits azure_ai_trace_data_source_preview with a nested trace_source of type agent_filter, and source: gains agent_version, start_time and end_time. lookback_hours stays supported, read as a start bound relative to now. Verified against the service, which changed the shape: start_time, end_time and max_traces have to be nested inside trace_source. A run submitted with them beside the data source type came back with the nested values null, so the window was dropped silently -- the same defect, one level in. A test pins the emitted JSON so the nesting cannot regress. The reconciler fingerprints the configuration rather than the payload, so an unchanged azure.eval.yaml keeps its digest and no eval is recreated. --- .../azure.ai.evaluations/internal/cmd/run.go | 39 ++++++++- .../internal/cmd/run_datasource_test.go | 83 +++++++++++++++++-- .../internal/messages/messages.go | 14 ++++ .../internal/pkg/eval_api/models.go | 54 ++++++++++++ .../pkg/eval_api/trace_source_test.go | 66 +++++++++++++++ .../internal/project/config_keys_test.go | 5 +- .../internal/project/eval_config.go | 8 ++ 7 files changed, 260 insertions(+), 9 deletions(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/trace_source_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go index 0b1bf457815..44b8c3ead8e 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go @@ -475,14 +475,47 @@ func tracesDataSource(group *project.Eval) (*eval_api.EvalRunDataSource, error) if agent == "" { return nil, messages.TracesNeedAgentName(group.Name) } - return eval_api.NewTracesDataSource( + + start, end, err := traceWindow(group.Source) + if err != nil { + return nil, err + } + return eval_api.NewTracePreviewDataSource( agent, - group.Source.LookbackHours, - time.Time{}, + group.Source.AgentVersion, + start, + end, group.Source.MaxTraces, ), nil } +// traceWindow resolves the bounds of the span a trace run reads. +// +// lookback_hours predates start_time and stays supported, read as a start bound +// relative to now. An explicit start_time wins, because it says the same thing +// without drifting every time the command runs. +func traceWindow(source *project.SourceDecl) (start, end time.Time, err error) { + if source.StartTime != "" { + start, err = time.Parse(time.RFC3339, source.StartTime) + if err != nil { + return time.Time{}, time.Time{}, messages.TraceWindowNotATime("start_time", source.StartTime) + } + } else if source.LookbackHours > 0 { + start = time.Now().Add(-time.Duration(source.LookbackHours) * time.Hour) + } + + if source.EndTime != "" { + end, err = time.Parse(time.RFC3339, source.EndTime) + if err != nil { + return time.Time{}, time.Time{}, messages.TraceWindowNotATime("end_time", source.EndTime) + } + } + if !start.IsZero() && !end.IsZero() && !end.After(start) { + return time.Time{}, time.Time{}, messages.TraceWindowEndsBeforeItStarts(source.StartTime, source.EndTime) + } + return start, end, nil +} + // responsesDataSource evaluates responses the project already stored. func responsesDataSource(group *project.Eval) (*eval_api.EvalRunDataSource, error) { if len(group.Source.ResponseIDs) == 0 { diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_datasource_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_datasource_test.go index 7eb727af747..bee8bd85afd 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_datasource_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_datasource_test.go @@ -50,15 +50,87 @@ func TestBuildRunDataSource_Traces(t *testing.T) { ds, err := ec.buildRunDataSource(context.Background(), group, "", 0) require.NoError(t, err) - assert.Equal(t, eval_api.EvalRunDataSourceTypeTraces, ds.Type) - assert.Equal(t, "support-agent", ds.AgentName) - assert.Equal(t, 24, ds.LookbackHours) - assert.Equal(t, 500, ds.MaxTraces) + // The legacy azure_ai_traces shape discarded agent_version and start_time + // without saying so, and re-imposed its own lookback. + assert.Equal(t, eval_api.EvalRunDataSourceTypeTracePreview, ds.Type) + require.NotNil(t, ds.TraceSource) + assert.Equal(t, "agent_filter", ds.TraceSource.Type) + assert.Equal(t, "support-agent", ds.TraceSource.AgentName) + assert.Equal(t, 500, ds.TraceSource.MaxTraces) + // lookback_hours is still honoured, as the window's start bound. + assert.NotZero(t, ds.TraceSource.StartTime) + assert.Zero(t, ds.TraceSource.EndTime, "an open end means up to now") // Nothing is invoked and nothing local is sent. assert.Nil(t, ds.Target) assert.Nil(t, ds.Source) } +// Pinning the version is the whole reason the preview shape is used: without +// it a redeployed agent is graded on whichever version the service picked. +func TestBuildRunDataSource_TracesPinsTheAgentVersion(t *testing.T) { + ec := &evalContext{} + group := &project.Eval{ + Name: "trace-eval", + Source: &project.SourceDecl{ + Type: project.SourceTypeTraces, + AgentName: "support-agent", + AgentVersion: "2", + }, + } + + ds, err := ec.buildRunDataSource(context.Background(), group, "", 0) + + require.NoError(t, err) + require.NotNil(t, ds.TraceSource) + assert.Equal(t, "2", ds.TraceSource.AgentVersion) +} + +// An explicit window travels intact, and wins over lookback_hours, which says +// the same thing but drifts every time the command runs. +func TestBuildRunDataSource_TracesCarriesAnExplicitWindow(t *testing.T) { + ec := &evalContext{} + group := &project.Eval{ + Name: "trace-eval", + Source: &project.SourceDecl{ + Type: project.SourceTypeTraces, + AgentName: "support-agent", + LookbackHours: 999, + StartTime: "2026-08-01T00:00:00Z", + EndTime: "2026-08-02T00:00:00Z", + }, + } + + ds, err := ec.buildRunDataSource(context.Background(), group, "", 0) + + require.NoError(t, err) + assert.Equal(t, int64(1785542400), ds.TraceSource.StartTime) + assert.Equal(t, int64(1785628800), ds.TraceSource.EndTime) +} + +// A window nobody can read, or one that holds nothing, is refused here rather +// than by a service that answers with no rows and no reason. +func TestBuildRunDataSource_TracesRefusesAnUnusableWindow(t *testing.T) { + ec := &evalContext{} + build := func(source *project.SourceDecl) error { + _, err := ec.buildRunDataSource(context.Background(), + &project.Eval{Name: "trace-eval", Source: source}, "", 0) + return err + } + + err := build(&project.SourceDecl{ + Type: project.SourceTypeTraces, AgentName: "a", StartTime: "yesterday", + }) + require.Error(t, err) + assert.Contains(t, err.Error(), "start_time") + + err = build(&project.SourceDecl{ + Type: project.SourceTypeTraces, AgentName: "a", + StartTime: "2026-08-02T00:00:00Z", EndTime: "2026-08-01T00:00:00Z", + }) + require.Error(t, err) + assert.Contains(t, err.Error(), "holds no traces") +} + // agent_name under source: is a filter, but an eval that names a target and // leaves the filter off still means "this agent's traces". func TestBuildRunDataSource_TracesFallsBackToTargetName(t *testing.T) { @@ -72,7 +144,8 @@ func TestBuildRunDataSource_TracesFallsBackToTargetName(t *testing.T) { ds, err := ec.buildRunDataSource(context.Background(), group, "", 0) require.NoError(t, err) - assert.Equal(t, "support-agent", ds.AgentName) + require.NotNil(t, ds.TraceSource) + assert.Equal(t, "support-agent", ds.TraceSource.AgentName) } // With neither, the run cannot say whose conversations to read, and saying so diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go b/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go index c79a86de799..c2eadb0796b 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go @@ -1629,6 +1629,20 @@ func GateNeedsATerminalRun(runID, status string) error { runID, status) } +// TraceWindowNotATime reports a window bound that is not a timestamp. +func TraceWindowNotATime(field, value string) error { + return fmt.Errorf( + "source.%s is %q, which is not a time: use RFC 3339, for example 2026-08-18T09:00:00Z", + field, value) +} + +// TraceWindowEndsBeforeItStarts reports a window that can hold no traces. +func TraceWindowEndsBeforeItStarts(start, end string) error { + return fmt.Errorf( + "source.end_time %q is not after source.start_time %q, so the window holds no traces", + end, start) +} + // AmbiguousJudgeModel reports several deployments where only one can be used. func AmbiguousJudgeModel(models []string) error { return fmt.Errorf( diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/models.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/models.go index d0109405362..f9f7f52857f 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/models.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/models.go @@ -316,8 +316,16 @@ const ( // EvalRunDataSourceTypeTraces evaluates an agent's recorded traces instead of // a dataset. The service reads them from Application Insights, so the agent // must be emitting gen_ai.input.messages / gen_ai.output.messages. + // + // Deprecated in favour of EvalRunDataSourceTypeTracePreview, which is the + // only shape that carries an agent version. This one silently discards + // agent_version and start_time and re-imposes its own lookback. EvalRunDataSourceTypeTraces EvalRunDataSourceType = "azure_ai_traces" + // EvalRunDataSourceTypeTracePreview is the shape that honours what the + // caller asks for: a pinned agent version, and a window with both bounds. + EvalRunDataSourceTypeTracePreview EvalRunDataSourceType = "azure_ai_trace_data_source_preview" + // EvalRunDataSourceTypeResponses evaluates responses the project already // stored, addressed by id. EvalRunDataSourceTypeResponses EvalRunDataSourceType = "azure_ai_responses" @@ -349,6 +357,10 @@ type EvalRunDataSource struct { EndTime int64 `json:"end_time,omitempty"` MaxTraces int `json:"max_traces,omitempty"` + // Trace preview only. Everything it carries is nested in the filter, which + // is where the service reads it. + TraceSource *TraceSourceFilter `json:"trace_source,omitempty"` + // Responses only. ItemGenerationParams *ItemGenerationParams `json:"item_generation_params,omitempty"` } @@ -435,6 +447,48 @@ func NewTracesDataSource(agentName string, lookbackHours int, end time.Time, max return ds } +// TraceSourceFilter selects which spans a trace run reads. +// +// The window and the cap live here, not beside the data source type: verified +// against the service, which echoes them back only from inside this object and +// silently ignores them at the top level. +type TraceSourceFilter struct { + Type string `json:"type"` + AgentName string `json:"agent_name,omitempty"` + AgentVersion string `json:"agent_version,omitempty"` + StartTime int64 `json:"start_time,omitempty"` + EndTime int64 `json:"end_time,omitempty"` + MaxTraces int `json:"max_traces,omitempty"` +} + +// NewTracePreviewDataSource builds the shape that keeps what the caller sent. +// +// The legacy azure_ai_traces source drops agent_version and start_time without +// saying so and re-applies its own lookback, so a redeployed agent was +// evaluated against whichever version the service picked. +func NewTracePreviewDataSource( + agentName, agentVersion string, + start, end time.Time, + maxTraces int, +) *EvalRunDataSource { + filter := &TraceSourceFilter{ + Type: "agent_filter", + AgentName: agentName, + AgentVersion: agentVersion, + MaxTraces: maxTraces, + } + if !start.IsZero() { + filter.StartTime = start.Unix() + } + if !end.IsZero() { + filter.EndTime = end.Unix() + } + return &EvalRunDataSource{ + Type: EvalRunDataSourceTypeTracePreview, + TraceSource: filter, + } +} + // NewDatasetOnlyDataSource scores the dataset as it stands, invoking nothing. // // Used when an eval declares no target: the rows already hold both sides of the diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/trace_source_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/trace_source_test.go new file mode 100644 index 00000000000..8f572baa0d8 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/trace_source_test.go @@ -0,0 +1,66 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package eval_api + +import ( + "encoding/json" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// The service reads the window and the cap from inside trace_source and +// ignores them beside the data source type. Verified live: a run submitted with +// them at the top level came back with the nested ones null, so the caller's +// window was dropped without a word -- the same defect the legacy +// azure_ai_traces shape has, moved one level in. +func TestTracePreviewNestsEverythingInsideTheFilter(t *testing.T) { + ds := NewTracePreviewDataSource( + "support-agent", "2", + time.Unix(1785542400, 0), time.Unix(1785628800, 0), + 20, + ) + + raw, err := json.Marshal(ds) + require.NoError(t, err) + + var got map[string]any + require.NoError(t, json.Unmarshal(raw, &got)) + + assert.Equal(t, "azure_ai_trace_data_source_preview", got["type"]) + assert.NotContains(t, got, "start_time", "the service ignores it here") + assert.NotContains(t, got, "end_time", "the service ignores it here") + assert.NotContains(t, got, "max_traces", "the service ignores it here") + + filter, ok := got["trace_source"].(map[string]any) + require.True(t, ok, "trace_source has to be an object") + assert.Equal(t, "agent_filter", filter["type"]) + assert.Equal(t, "support-agent", filter["agent_name"]) + assert.Equal(t, "2", filter["agent_version"]) + assert.EqualValues(t, 1785542400, filter["start_time"]) + assert.EqualValues(t, 1785628800, filter["end_time"]) + assert.EqualValues(t, 20, filter["max_traces"]) +} + +// An unpinned version and an open window are omitted rather than sent as zero, +// which the service would read as a bound of 1970. +func TestTracePreviewOmitsWhatWasNotAskedFor(t *testing.T) { + ds := NewTracePreviewDataSource("support-agent", "", time.Time{}, time.Time{}, 0) + + raw, err := json.Marshal(ds) + require.NoError(t, err) + + var got map[string]any + require.NoError(t, json.Unmarshal(raw, &got)) + + filter, ok := got["trace_source"].(map[string]any) + require.True(t, ok) + assert.Equal(t, "support-agent", filter["agent_name"]) + assert.NotContains(t, filter, "agent_version") + assert.NotContains(t, filter, "start_time") + assert.NotContains(t, filter, "end_time") + assert.NotContains(t, filter, "max_traces") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/config_keys_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/config_keys_test.go index 76ef12d5adc..1711765f7ca 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/config_keys_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/config_keys_test.go @@ -73,7 +73,10 @@ func TestEvaluatorRefKeys(t *testing.T) { // source: says where rows come from when they are not a dataset. func TestSourceDeclKeys(t *testing.T) { assert.ElementsMatch(t, - []string{"type", "lookback_hours", "max_traces", "agent_name", "response_ids", "max_turns"}, + []string{ + "type", "lookback_hours", "max_traces", "agent_name", "response_ids", "max_turns", + "agent_version", "start_time", "end_time", + }, yamlKeys(t, SourceDecl{})) } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go index fb9abc2aee3..7b20b769b77 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go @@ -70,6 +70,14 @@ type SourceDecl struct { AgentName string `yaml:"agent_name,omitempty" json:"agent_name,omitempty"` ResponseIDs []string `yaml:"response_ids,omitempty" json:"response_ids,omitempty"` MaxTurns int `yaml:"max_turns,omitempty" json:"max_turns,omitempty"` + // AgentVersion pins which deployment's spans are read. Without it the + // service chooses, and a redeployed agent is evaluated against whichever + // version it picked. + AgentVersion string `yaml:"agent_version,omitempty" json:"agent_version,omitempty"` + // StartTime and EndTime bound the window explicitly. LookbackHours stays + // supported and is read as a start bound relative to now. + StartTime string `yaml:"start_time,omitempty" json:"start_time,omitempty"` + EndTime string `yaml:"end_time,omitempty" json:"end_time,omitempty"` } // Source types an eval can read rows from. From 594939b58aa5a6bd487a4f6218053e95c9189a99 Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 18 Aug 2026 05:39:01 +0300 Subject: [PATCH 253/320] Refuse a trace window before it is silently ignored A window bound the service cannot read is dropped without comment: the run grades a default seven days and reports success. The bounds are now parsed where the rest of the configuration is checked, so a mistyped timestamp, a window that ends before it starts, a start declared twice over as both start_time and lookback_hours, and a lookback that reaches forwards all fail before the eval is created. The run path keeps its own parse, because a run reached by id has no configuration that was ever validated. Also from this review round: - `eval show` no longer prints a Source row. It read the type back out of the request this extension builds, which hardcodes "custom", so the row said "custom" for every eval that has ever existed. - `--eval-id` re-submitted whatever data source the last run carried, which for an eval created before this change is the legacy trace shape that drops agent_version and the window. It is upgraded on the way through. - The run link is cyan again, as the portal link was before the two were merged into one, and `run list` stamps the portal URL so the fallback is reachable there rather than only on the views that wait. - `init` reads the App Insights connection once instead of opening an azd connection for each of the two questions the answer settles. - Deleted NewTracesDataSource, which nothing calls now. The trace window test asserted only that the start was non-zero, which a lookback that reached forwards would also satisfy; it now asserts the distance back from now. --- .../internal/cmd/eval_group.go | 19 ++---- .../internal/cmd/eval_show_test.go | 10 +--- .../azure.ai.evaluations/internal/cmd/init.go | 11 +++- .../azure.ai.evaluations/internal/cmd/job.go | 3 - .../internal/cmd/portal_test.go | 2 - .../azure.ai.evaluations/internal/cmd/run.go | 59 ++++++++++++++++--- .../internal/cmd/run_datasource_test.go | 12 ++-- .../internal/cmd/run_ops.go | 3 +- .../internal/cmd/run_output.go | 3 +- .../internal/messages/messages.go | 39 +++++++++--- .../internal/pkg/eval_api/models.go | 19 ------ .../internal/project/eval_config.go | 39 ++++++++++++ .../internal/project/eval_config_test.go | 28 +++++++++ 13 files changed, 179 insertions(+), 68 deletions(-) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/eval_group.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/eval_group.go index de0a735b7f3..9f939a1bff7 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/eval_group.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/eval_group.go @@ -236,9 +236,7 @@ func newEvalShowCommand() *cobra.Command { } // Without these the command answers "does this id exist", which is // not what a definition is, nor what its own help promises. - if source := evalSourceType(group); source != "" { - detail = append(detail, field{"Source", source}) - } + if graders := evalGraders(group); graders != "" { detail = append(detail, field{"Evaluators", graders}) } @@ -250,18 +248,13 @@ func newEvalShowCommand() *cobra.Command { return cmd } -// evalSourceType reports where the eval's rows come from, as the service -// records it. Empty when the service sent no data source config. -func evalSourceType(group *eval_api.OpenAIEval) string { - if group == nil || group.DataSourceConfig == nil { - return "" - } - kind, _ := group.DataSourceConfig["type"].(string) - return kind -} - // evalGraders lists the evaluators the eval grades with, preferring the // reference a caller would recognize over the criterion label. +// +// data_source_config is deliberately not shown beside it: every eval this +// extension creates carries type "custom", which describes the item schema +// rather than where the rows come from, so a "Source" row would read as an +// answer while always saying the same thing. func evalGraders(group *eval_api.OpenAIEval) string { if group == nil { return "" diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/eval_show_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/eval_show_test.go index 3a86ab795b3..fbe5267fb61 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/eval_show_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/eval_show_test.go @@ -13,19 +13,17 @@ import ( // `show` is documented as showing an eval definition, and answered with the id, // the name and who created it -- true of any eval, and none of the definition. -// The service returns both of these on a read. +// The graders are the part of the definition the service does return. func TestShowSurfacesWhatTheEvalGrades(t *testing.T) { group := &eval_api.OpenAIEval{ - ID: "eval_1", - Name: "support-trace-eval", - DataSourceConfig: map[string]any{"type": "custom"}, + ID: "eval_1", + Name: "support-trace-eval", TestingCriteria: []eval_api.TestingCriterion{ {Name: "task_adherence", EvaluatorName: "builtin.task_adherence"}, {Name: "quality", EvaluatorName: "support-quality", EvaluatorVersion: "2"}, }, } - assert.Equal(t, "custom", evalSourceType(group)) assert.Equal(t, "builtin.task_adherence, support-quality (2)", evalGraders(group)) } @@ -42,9 +40,7 @@ func TestShowFallsBackToTheCriterionLabel(t *testing.T) { // An older eval, or one the service answers without a definition, still shows // its identity rather than blank rows. func TestShowOmitsWhatTheServiceDidNotSend(t *testing.T) { - assert.Empty(t, evalSourceType(&eval_api.OpenAIEval{ID: "eval_1"})) assert.Empty(t, evalGraders(&eval_api.OpenAIEval{ID: "eval_1"})) - assert.Empty(t, evalSourceType(nil)) assert.Empty(t, evalGraders(nil)) // A criterion carrying no name at all contributes nothing rather than an diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go index 6375e1da3ba..08b4e364c0a 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go @@ -10,6 +10,7 @@ import ( "os" "path/filepath" "strings" + "sync" "azureaieval/internal/messages" "azureaieval/internal/pkg/evalcore" @@ -75,12 +76,18 @@ func newInitCommand() *cobra.Command { if err := validateEvaluatorRefs(evaluators); err != nil { return err } + // Asked twice -- once to pick the default source, once to say so -- + // and each call opens an azd connection. The answer cannot change + // mid-command, and a run that never asks never connects. + tracesWired := sync.OnceValue(func() bool { + return tracesConnected(commandContext(cmd)) + }) if source == "" { // The same signal `generate --from` defaults on, read from the azd // environment rather than the service, so init still makes no // service calls. Traces are real conversations; a project wired to // collect them should not have to ask for them by flag. - if tracesConnected(commandContext(cmd)) { + if tracesWired() { source = initSourceTraces } else { source = initSourceDataset @@ -234,7 +241,7 @@ func newInitCommand() *cobra.Command { if source == initSourceTraces { // Claiming the connection is only honest when it was found. init // makes no service calls, so it cannot verify one it did not see. - fmt.Fprint(out, messages.UsingTraceSource(tracesConnected(commandContext(cmd)))) + fmt.Fprint(out, messages.UsingTraceSource(tracesWired())) } // Only what was settled without asking: a reader who just picked // from a list does not need it read back to them. diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/job.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/job.go index 24a911cea86..5f299788cb3 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/job.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/job.go @@ -318,8 +318,5 @@ func jobLookupError(action string, kind jobKind, jobID string, err error) error } return messages.JobNotFound(kind.name, jobID, other) } - if action == "reading" { - return messages.ReadingJob(kind.name, jobID, err) - } return messages.JobActionFailed(action, kind.name, jobID, err) } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/portal_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/portal_test.go index 73e64e289c5..60a18cade0a 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/portal_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/portal_test.go @@ -79,8 +79,6 @@ func TestPortalRunURLShape(t *testing.T) { "/build/evaluations/eval_1/run/evalrun_9")) } -// The report line and the portal line are different things: one is the -// service's own link, the other is built here. A run carrying both prints both. // A run has one destination. The service's report_url and the portal URL the // extension builds resolve to the same page, and printing both put two labels // on it with no rule a reader could infer. diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go index 44b8c3ead8e..aedf74e5bf6 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go @@ -22,6 +22,7 @@ import ( "azureaieval/internal/pkg/eval_api" "azureaieval/internal/project" + "github.com/fatih/color" "github.com/spf13/cobra" ) @@ -386,7 +387,35 @@ func (ec *evalContext) reuseDataSourceFromLastRun( if list == nil || len(list.Data) == 0 || list.Data[0].DataSource == nil { return nil, messages.EvalHasNoPreviousRun(evalID) } - return list.Data[0].DataSource, nil + return upgradeLegacyTraceSource(list.Data[0].DataSource), nil +} + +// upgradeLegacyTraceSource carries a run recorded under the old trace shape +// onto the one that keeps what it is given. +// +// A run reattached by id repeats whatever the last one sent, so without this an +// eval whose last run predates the change would keep the version-blind, +// drifting-lookback source for good, and nothing would say so. +func upgradeLegacyTraceSource(ds *eval_api.EvalRunDataSource) *eval_api.EvalRunDataSource { + if ds == nil || ds.Type != eval_api.EvalRunDataSourceTypeTraces { + return ds + } + var end time.Time + if ds.EndTime > 0 { + end = time.Unix(ds.EndTime, 0) + } + var start time.Time + if ds.LookbackHours > 0 { + from := end + if from.IsZero() { + from = time.Now() + } + start = from.Add(-time.Duration(ds.LookbackHours) * time.Hour) + } + // The old shape carried no version, so this pins nothing that was not + // pinned before; it stops the service choosing differently run to run only + // once the declaration names one. + return eval_api.NewTracePreviewDataSource(ds.AgentName, "", start, end, ds.MaxTraces) } // buildRunDataSource binds the eval's rows to the run. @@ -492,13 +521,21 @@ func tracesDataSource(group *project.Eval) (*eval_api.EvalRunDataSource, error) // traceWindow resolves the bounds of the span a trace run reads. // // lookback_hours predates start_time and stays supported, read as a start bound -// relative to now. An explicit start_time wins, because it says the same thing -// without drifting every time the command runs. +// relative to now. The pair is refused by config validation, so only one of the +// two is ever set here. +// +// Bounds are parsed rather than validated: a run reached by id has no config to +// have been validated, so a malformed value still has to fail rather than be +// dropped. func traceWindow(source *project.SourceDecl) (start, end time.Time, err error) { + if source == nil { + return time.Time{}, time.Time{}, nil + } if source.StartTime != "" { start, err = time.Parse(time.RFC3339, source.StartTime) if err != nil { - return time.Time{}, time.Time{}, messages.TraceWindowNotATime("start_time", source.StartTime) + return time.Time{}, time.Time{}, messages.TraceWindowNotATime( + 0, source.AgentName, "start_time", source.StartTime) } } else if source.LookbackHours > 0 { start = time.Now().Add(-time.Duration(source.LookbackHours) * time.Hour) @@ -507,11 +544,19 @@ func traceWindow(source *project.SourceDecl) (start, end time.Time, err error) { if source.EndTime != "" { end, err = time.Parse(time.RFC3339, source.EndTime) if err != nil { - return time.Time{}, time.Time{}, messages.TraceWindowNotATime("end_time", source.EndTime) + return time.Time{}, time.Time{}, messages.TraceWindowNotATime( + 0, source.AgentName, "end_time", source.EndTime) } } if !start.IsZero() && !end.IsZero() && !end.After(start) { - return time.Time{}, time.Time{}, messages.TraceWindowEndsBeforeItStarts(source.StartTime, source.EndTime) + // The start is named as it was written, which is lookback_hours when + // that is where it came from rather than an empty start_time. + written := source.StartTime + if written == "" { + written = fmt.Sprintf("%dh before now", source.LookbackHours) + } + return time.Time{}, time.Time{}, messages.TraceWindowEndsBeforeItStarts( + 0, source.AgentName, written, source.EndTime) } return start, end, nil } @@ -883,7 +928,7 @@ func renderRun( } if url := runLink(run.ReportURL, run.PortalURL); url != "" { - fmt.Fprint(out, messages.ReportLink(url)) + fmt.Fprint(out, messages.ReportLink(color.CyanString(url))) } return nil } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_datasource_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_datasource_test.go index bee8bd85afd..bc7c2a743dc 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_datasource_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_datasource_test.go @@ -8,6 +8,7 @@ import ( "os" "path/filepath" "testing" + "time" "azureaieval/internal/pkg/eval_api" "azureaieval/internal/project" @@ -57,8 +58,10 @@ func TestBuildRunDataSource_Traces(t *testing.T) { assert.Equal(t, "agent_filter", ds.TraceSource.Type) assert.Equal(t, "support-agent", ds.TraceSource.AgentName) assert.Equal(t, 500, ds.TraceSource.MaxTraces) - // lookback_hours is still honoured, as the window's start bound. - assert.NotZero(t, ds.TraceSource.StartTime) + // lookback_hours is still honoured, as the window's start bound. Asserted + // as a distance from now, because a merely non-zero start is also what a + // lookback that reached forwards would produce. + assert.InDelta(t, time.Now().Add(-24*time.Hour).Unix(), ds.TraceSource.StartTime, 60) assert.Zero(t, ds.TraceSource.EndTime, "an open end means up to now") // Nothing is invoked and nothing local is sent. assert.Nil(t, ds.Target) @@ -85,8 +88,9 @@ func TestBuildRunDataSource_TracesPinsTheAgentVersion(t *testing.T) { assert.Equal(t, "2", ds.TraceSource.AgentVersion) } -// An explicit window travels intact, and wins over lookback_hours, which says -// the same thing but drifts every time the command runs. +// An explicit window travels intact. Declaring it beside lookback_hours is +// refused by config validation, but a run reached by id has no config that was +// validated, so the absolute bound still has to win rather than be dropped. func TestBuildRunDataSource_TracesCarriesAnExplicitWindow(t *testing.T) { ec := &evalContext{} group := &project.Eval{ diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go index 1a0ff8e1d5f..36db38b6da9 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go @@ -11,6 +11,7 @@ import ( "azureaieval/internal/messages" "azureaieval/internal/pkg/eval_api" + "github.com/fatih/color" "github.com/spf13/cobra" ) @@ -187,7 +188,7 @@ func newRunShowCommand() *cobra.Command { fmt.Fprint(out, messages.RunResultsLine(counts)) } if url := runLink(run.ReportURL, run.PortalURL); url != "" { - fmt.Fprint(out, messages.RunReportLine(url)) + fmt.Fprint(out, messages.RunReportLine(color.CyanString(url))) } if gateOnStatus { if err := runCompleted(run); err != nil { diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output.go index ae3f738cef5..ae1a961c8b7 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output.go @@ -15,6 +15,7 @@ import ( "azureaieval/internal/messages" "azureaieval/internal/pkg/eval_api" + "github.com/fatih/color" "github.com/spf13/cobra" ) @@ -569,7 +570,7 @@ func renderResults( } if url := runLink(run.ReportURL, run.PortalURL); url != "" { - fmt.Fprint(w, messages.ReportLinkAfterRows(url)) + fmt.Fprint(w, messages.ReportLinkAfterRows(color.CyanString(url))) } return nil } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go b/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go index c2eadb0796b..d3655f783cd 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go @@ -249,7 +249,7 @@ func ErroredNotScored(errored int) string { return fmt.Sprintf("(%d errored, not scored)", errored) } -// ReportLink closes a run summary with the service's own report. +// ReportLink closes a run summary with the one link the run has. func ReportLink(url string) string { return fmt.Sprintf("Report: %s\n", url) } @@ -334,7 +334,7 @@ func RunResultsLine(counts string) string { return fmt.Sprintf(" results : %s\n", counts) } -// RunReportLine reports the run's report URL in the detail view. +// RunReportLine reports the run's one link in the detail view. func RunReportLine(url string) string { return fmt.Sprintf(" report : %s\n", url) } @@ -468,7 +468,7 @@ func OutputItemReason(reason string) string { return fmt.Sprintf(" %s\n", reason) } -// ReportLinkAfterRows closes a per-sample listing with the service's report. +// ReportLinkAfterRows closes a per-sample listing with the run's one link. func ReportLinkAfterRows(url string) string { return fmt.Sprintf("\nReport: %s\n", url) } @@ -1630,17 +1630,38 @@ func GateNeedsATerminalRun(runID, status string) error { } // TraceWindowNotATime reports a window bound that is not a timestamp. -func TraceWindowNotATime(field, value string) error { +func TraceWindowNotATime(i int, eval, field, value string) error { return fmt.Errorf( - "source.%s is %q, which is not a time: use RFC 3339, for example 2026-08-18T09:00:00Z", - field, value) + "evals[%d] %q: source.%s is %q, which is not a time: use RFC 3339, "+ + "for example 2026-08-18T09:00:00Z", + i, eval, field, value) } // TraceWindowEndsBeforeItStarts reports a window that can hold no traces. -func TraceWindowEndsBeforeItStarts(start, end string) error { +func TraceWindowEndsBeforeItStarts(i int, eval, start, end string) error { return fmt.Errorf( - "source.end_time %q is not after source.start_time %q, so the window holds no traces", - end, start) + "evals[%d] %q: source.end_time %q is not after source.start_time %q, "+ + "so the window holds no traces", + i, eval, end, start) +} + +// TraceWindowOverSpecified reports a window declared twice over. +// +// lookback_hours is a start bound relative to now and start_time is an absolute +// one, so a file carrying both does not say which window was meant. +func TraceWindowOverSpecified(i int, eval string) error { + return fmt.Errorf( + "evals[%d] %q: source declares both start_time and lookback_hours, "+ + "which are two ways of saying where the window starts: keep one", + i, eval) +} + +// NegativeLookbackHours reports a window that reaches forwards. +func NegativeLookbackHours(i int, eval string, hours int) error { + return fmt.Errorf( + "evals[%d] %q: source.lookback_hours is %d, and a window cannot reach "+ + "into the future: give the hours to look back", + i, eval, hours) } // AmbiguousJudgeModel reports several deployments where only one can be used. diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/models.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/models.go index f9f7f52857f..9249a8e35ac 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/models.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/models.go @@ -428,25 +428,6 @@ func NewAgentTargetDataSource(agentName string, agentVersion *string) *EvalRunDa } } -// NewTracesDataSource evaluates an agent's recorded traces instead of a dataset. -// -// The window is a lookback in hours. The service has no start bound on this -// data source: a start_time is accepted and dropped, leaving the default seven -// days in place, so the conversion happens here rather than being left to look -// like it worked. -func NewTracesDataSource(agentName string, lookbackHours int, end time.Time, maxTraces int) *EvalRunDataSource { - ds := &EvalRunDataSource{ - Type: EvalRunDataSourceTypeTraces, - AgentName: agentName, - LookbackHours: lookbackHours, - MaxTraces: maxTraces, - } - if !end.IsZero() { - ds.EndTime = end.Unix() - } - return ds -} - // TraceSourceFilter selects which spans a trace run reads. // // The window and the cap live here, not beside the data source type: verified diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go index 7b20b769b77..aa06ffd2bcd 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go @@ -7,6 +7,7 @@ package project import ( "strings" + "time" "azureaieval/internal/messages" "azureaieval/internal/pkg/evalcore" @@ -309,6 +310,41 @@ func (c *EvalConfig) validateCatalogs() error { return nil } +// validateTraceWindow refuses a window a run could not use. +// +// Checked with the rest of the configuration rather than at run time, so a +// mistyped timestamp is caught before the eval is created rather than after. +func validateTraceWindow(i int, name string, source *SourceDecl) error { + start, end := time.Time{}, time.Time{} + + if source.StartTime != "" { + parsed, err := time.Parse(time.RFC3339, source.StartTime) + if err != nil { + return messages.TraceWindowNotATime(i, name, "start_time", source.StartTime) + } + start = parsed + } + if source.EndTime != "" { + parsed, err := time.Parse(time.RFC3339, source.EndTime) + if err != nil { + return messages.TraceWindowNotATime(i, name, "end_time", source.EndTime) + } + end = parsed + } + if !start.IsZero() && !end.IsZero() && !end.After(start) { + return messages.TraceWindowEndsBeforeItStarts(i, name, source.StartTime, source.EndTime) + } + // Two ways of saying the same thing, and the file cannot say which was + // meant. Every other contradictory pair here is refused rather than ranked. + if source.StartTime != "" && source.LookbackHours != 0 { + return messages.TraceWindowOverSpecified(i, name) + } + if source.LookbackHours < 0 { + return messages.NegativeLookbackHours(i, name, source.LookbackHours) + } + return nil +} + func (c *EvalConfig) validateEval(i int, eval Eval) error { if eval.Dataset != "" && eval.Source != nil { return messages.DatasetAndSourceBothDeclared(i, eval.Name) @@ -332,6 +368,9 @@ func (c *EvalConfig) validateEval(i int, eval Eval) error { if eval.Source.AgentName == "" && (eval.Target == nil || eval.Target.Name == "") { return messages.TracesSourceNeedsAgentName(i, eval.Name) } + if err := validateTraceWindow(i, eval.Name, eval.Source); err != nil { + return err + } case SourceTypeResponses: if len(eval.Source.ResponseIDs) == 0 { return messages.ResponsesSourceNeedsIDs(i, eval.Name) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_test.go index 96898445f47..bc3f6af9db0 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_test.go @@ -229,6 +229,34 @@ func TestValidate_Rejects(t *testing.T) { " evaluators:\n - evaluator: builtin.relevance\n", wantErr: "source.response_ids is required", }, + { + // A window bound the run path cannot parse is dropped by the + // service, which then grades a default seven days and says nothing. + name: "window bound that is not a time", + body: "evals:\n - name: e\n source:\n type: traces\n agent_name: a\n" + + " start_time: yesterday\n evaluators:\n - evaluator: builtin.relevance\n", + wantErr: "which is not a time", + }, + { + name: "window ending before it starts", + body: "evals:\n - name: e\n source:\n type: traces\n agent_name: a\n" + + " start_time: \"2026-08-02T00:00:00Z\"\n end_time: \"2026-08-01T00:00:00Z\"\n" + + " evaluators:\n - evaluator: builtin.relevance\n", + wantErr: "holds no traces", + }, + { + name: "window declared twice over", + body: "evals:\n - name: e\n source:\n type: traces\n agent_name: a\n" + + " start_time: \"2026-08-01T00:00:00Z\"\n lookback_hours: 24\n" + + " evaluators:\n - evaluator: builtin.relevance\n", + wantErr: "keep one", + }, + { + name: "lookback reaching forwards", + body: "evals:\n - name: e\n source:\n type: traces\n agent_name: a\n" + + " lookback_hours: -24\n evaluators:\n - evaluator: builtin.relevance\n", + wantErr: "cannot reach into the future", + }, { name: "dataset without a name", body: "datasets:\n - source: ./d.jsonl\n" + oneEval, From fbde53cd250c5c0243f810bc7624eb35f5eba4ce Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 18 Aug 2026 06:07:54 +0300 Subject: [PATCH 254/320] Close the holes the window validation left open A review of the previous commit found the new validation agreed with the run path on everything except the one case a file can actually express. Both ends of the window are now resolved the same way in both places. - `lookback_hours` beside an `end_time` in the past passed validation and then failed at `run start`, which is the failure the validation exists to move earlier. The lookback is resolved to a start bound before the ordering check. - A lookback large enough to overflow the nanosecond duration it becomes wrapped into a start bound in the future, and the run read nothing and said nothing. Bounded at ten years. - A bound that parsed to the zero time was accepted and then read as "no bound" by everything after it, so a declared bound was dropped from the request in silence. - `max_traces` was guarded on the flag but not in the file, so a hand-edited negative cap was sent as-is. - A negative lookback declared beside a `start_time` reported "keep one", and following that advice left the file wrong in a different way. The run-time window errors named `evals[0]` and the agent name where the eval name belonged, and told anyone who wrote `lookback_hours` to go and look at their `start_time`. They are now their own messages, without an index a run does not have, naming the field the file actually contains. Reattaching a legacy trace run by id dropped the window entirely unless the old source carried a lookback. The old shape had no start bound and the service graded seven days, so an upgrade with no start widened the run to all of history, graded a different set of traces and made the query much more expensive, with nothing to say so. The default it ran under is written down. Colour is pinned off for the cmd package's tests. fatih/color decides once at init from whether the process's stdout is a terminal, not from the writer the renderer was handed, so the assertions added with the coloured run link passed under `go test` and failed when the compiled test binary was run from a terminal. Verified by flipping the pin: the link assertion fails with the escape codes in place. `upgradeLegacyTraceSource` had no test at all. It has one per quadrant now, plus pass-through for nil and for a source that was never the legacy shape. --- .../internal/cmd/eval_group.go | 3 +- .../internal/cmd/legacy_trace_test.go | 87 +++++++++++++++++++ .../internal/cmd/main_test.go | 24 +++++ .../azure.ai.evaluations/internal/cmd/run.go | 73 ++++++++++------ .../internal/cmd/run_datasource_test.go | 29 +++++-- .../internal/messages/messages.go | 60 ++++++++++++- .../internal/project/eval_config.go | 82 +++++++++++++---- .../internal/project/eval_config_test.go | 32 +++++++ 8 files changed, 331 insertions(+), 59 deletions(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/legacy_trace_test.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/main_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/eval_group.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/eval_group.go index 9f939a1bff7..66750890f43 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/eval_group.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/eval_group.go @@ -234,9 +234,8 @@ func newEvalShowCommand() *cobra.Command { {"Created", timestampString(group.CreatedAt)}, {"Created By", group.CreatedBy}, } - // Without these the command answers "does this id exist", which is + // Without this the command answers "does this id exist", which is // not what a definition is, nor what its own help promises. - if graders := evalGraders(group); graders != "" { detail = append(detail, field{"Evaluators", graders}) } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/legacy_trace_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/legacy_trace_test.go new file mode 100644 index 00000000000..1005244bb31 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/legacy_trace_test.go @@ -0,0 +1,87 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "testing" + "time" + + "azureaieval/internal/pkg/eval_api" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// A run reattached by id repeats the data source the last one sent, so an eval +// whose last run predates the preview shape would keep sending the old one for +// good. The old shape carried no agent version, which is the whole reason to +// move off it. +func TestUpgradeLegacyTraceSource_MovesToThePreviewShape(t *testing.T) { + ds := upgradeLegacyTraceSource(&eval_api.EvalRunDataSource{ + Type: eval_api.EvalRunDataSourceTypeTraces, + AgentName: "support-agent", + LookbackHours: 24, + MaxTraces: 500, + }) + + require.NotNil(t, ds.TraceSource) + assert.Equal(t, eval_api.EvalRunDataSourceTypeTracePreview, ds.Type) + assert.Equal(t, "agent_filter", ds.TraceSource.Type) + assert.Equal(t, "support-agent", ds.TraceSource.AgentName) + assert.Equal(t, 500, ds.TraceSource.MaxTraces) + assert.InDelta(t, time.Now().Add(-24*time.Hour).Unix(), ds.TraceSource.StartTime, 60) + assert.Zero(t, ds.TraceSource.EndTime) + // Nothing was pinned before, so nothing is pinned now: the upgrade must not + // invent a version the previous runs were never graded against. + assert.Empty(t, ds.TraceSource.AgentVersion) +} + +// The old shape had no start bound at all: the service applied its own default +// of seven days. Carrying such a run forward with an open start would widen it +// to all of history, grading a different set of traces than the run being +// repeated and making the query far more expensive, with nothing to say so. +func TestUpgradeLegacyTraceSource_KeepsTheWindowItRanUnder(t *testing.T) { + ds := upgradeLegacyTraceSource(&eval_api.EvalRunDataSource{ + Type: eval_api.EvalRunDataSourceTypeTraces, + AgentName: "support-agent", + }) + + require.NotNil(t, ds.TraceSource) + assert.InDelta(t, time.Now().Add(-24*7*time.Hour).Unix(), ds.TraceSource.StartTime, 60) + assert.Zero(t, ds.TraceSource.EndTime) +} + +// An end bound anchors the window it closes, rather than being read alongside a +// start measured from now: a run that ended a month ago covered the week before +// that, not the week before today. +func TestUpgradeLegacyTraceSource_MeasuresBackFromTheEnd(t *testing.T) { + end := time.Date(2026, 8, 2, 0, 0, 0, 0, time.UTC) + + ds := upgradeLegacyTraceSource(&eval_api.EvalRunDataSource{ + Type: eval_api.EvalRunDataSourceTypeTraces, + AgentName: "support-agent", + LookbackHours: 24, + EndTime: end.Unix(), + }) + + require.NotNil(t, ds.TraceSource) + assert.Equal(t, end.Unix(), ds.TraceSource.EndTime) + assert.Equal(t, end.Add(-24*time.Hour).Unix(), ds.TraceSource.StartTime) +} + +// Only the legacy shape is rewritten. Anything else is repeated exactly, so a +// source this extension has never heard of is not quietly replaced with one it +// made up. +func TestUpgradeLegacyTraceSource_LeavesEverythingElseAlone(t *testing.T) { + assert.Nil(t, upgradeLegacyTraceSource(nil)) + + preview := &eval_api.EvalRunDataSource{ + Type: eval_api.EvalRunDataSourceTypeTracePreview, + TraceSource: &eval_api.TraceSourceFilter{Type: "agent_filter", AgentName: "a", StartTime: 7}, + } + assert.Same(t, preview, upgradeLegacyTraceSource(preview)) + + jsonl := &eval_api.EvalRunDataSource{Type: eval_api.EvalRunDataSourceTypeJSONL} + assert.Same(t, jsonl, upgradeLegacyTraceSource(jsonl)) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/main_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/main_test.go new file mode 100644 index 00000000000..b726e669783 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/main_test.go @@ -0,0 +1,24 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "os" + "testing" + + "github.com/fatih/color" +) + +// TestMain pins colour off for the whole package. +// +// fatih/color decides once, at init, from whether the process's stdout is a +// terminal -- not from the writer a renderer was handed. `go test` pipes +// stdout, so a test asserting on plain text passes under `go test` and fails +// when the compiled test binary is run from a terminal. Pinning it here makes +// the expected output the same either way, rather than leaving every assertion +// on a rendered line to depend on how the suite was started. +func TestMain(m *testing.M) { + color.NoColor = true + os.Exit(m.Run()) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go index aedf74e5bf6..18251180b4c 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go @@ -14,6 +14,7 @@ import ( "os" "path/filepath" "sort" + "strconv" "strings" "time" @@ -390,12 +391,27 @@ func (ec *evalContext) reuseDataSourceFromLastRun( return upgradeLegacyTraceSource(list.Data[0].DataSource), nil } +// legacyTraceLookbackHours is the window the old trace shape fell back to. +// +// The old data source had no start bound: a start_time was accepted and +// dropped, and the service graded its own default of seven days. Carrying a run +// forward with no start at all would quietly widen it to all of history, so the +// default it was actually run under is what gets written down. +const legacyTraceLookbackHours = 24 * 7 + // upgradeLegacyTraceSource carries a run recorded under the old trace shape // onto the one that keeps what it is given. // // A run reattached by id repeats whatever the last one sent, so without this an -// eval whose last run predates the change would keep the version-blind, -// drifting-lookback source for good, and nothing would say so. +// eval whose last run predates the change would keep the version-blind source +// for good, and nothing would say so. +// +// The window it produces is anchored at the moment of the upgrade rather than +// rolling: the new shape has no lookback to carry, and the upgraded source is +// what the next reattach reads back. A window that should move with each run +// has to come from the declaration, which is where `run start --eval ` +// reads it from anyway; this path exists only for an eval reached by id, with +// no declaration to read. func upgradeLegacyTraceSource(ds *eval_api.EvalRunDataSource) *eval_api.EvalRunDataSource { if ds == nil || ds.Type != eval_api.EvalRunDataSourceTypeTraces { return ds @@ -404,14 +420,15 @@ func upgradeLegacyTraceSource(ds *eval_api.EvalRunDataSource) *eval_api.EvalRunD if ds.EndTime > 0 { end = time.Unix(ds.EndTime, 0) } - var start time.Time - if ds.LookbackHours > 0 { - from := end - if from.IsZero() { - from = time.Now() - } - start = from.Add(-time.Duration(ds.LookbackHours) * time.Hour) + from := end + if from.IsZero() { + from = time.Now() } + hours := ds.LookbackHours + if hours <= 0 { + hours = legacyTraceLookbackHours + } + start := from.Add(-time.Duration(hours) * time.Hour) // The old shape carried no version, so this pins nothing that was not // pinned before; it stops the service choosing differently run to run only // once the declaration names one. @@ -505,7 +522,7 @@ func tracesDataSource(group *project.Eval) (*eval_api.EvalRunDataSource, error) return nil, messages.TracesNeedAgentName(group.Name) } - start, end, err := traceWindow(group.Source) + start, end, err := traceWindow(group.Name, group.Source) if err != nil { return nil, err } @@ -521,42 +538,42 @@ func tracesDataSource(group *project.Eval) (*eval_api.EvalRunDataSource, error) // traceWindow resolves the bounds of the span a trace run reads. // // lookback_hours predates start_time and stays supported, read as a start bound -// relative to now. The pair is refused by config validation, so only one of the -// two is ever set here. +// relative to now. Config validation refuses the pair, so in practice only one +// of the two is set; the precedence here is what makes that a validation rule +// rather than the only thing standing between the file and a dropped bound. // -// Bounds are parsed rather than validated: a run reached by id has no config to -// have been validated, so a malformed value still has to fail rather than be -// dropped. -func traceWindow(source *project.SourceDecl) (start, end time.Time, err error) { +// The bounds are parsed again rather than trusted, because this is the last +// place they can be refused before the request is built, and a value that fails +// here would otherwise be sent as a zero and read as "no bound". +func traceWindow(evalName string, source *project.SourceDecl) (start, end time.Time, err error) { if source == nil { return time.Time{}, time.Time{}, nil } + + startField, startValue := "start_time", source.StartTime if source.StartTime != "" { start, err = time.Parse(time.RFC3339, source.StartTime) if err != nil { - return time.Time{}, time.Time{}, messages.TraceWindowNotATime( - 0, source.AgentName, "start_time", source.StartTime) + return time.Time{}, time.Time{}, messages.RunTraceWindowNotATime( + evalName, "start_time", source.StartTime) } } else if source.LookbackHours > 0 { start = time.Now().Add(-time.Duration(source.LookbackHours) * time.Hour) + // Named as the file spells it, so the error does not send a reader + // looking for a start_time they never wrote. + startField, startValue = "lookback_hours", strconv.Itoa(source.LookbackHours) } if source.EndTime != "" { end, err = time.Parse(time.RFC3339, source.EndTime) if err != nil { - return time.Time{}, time.Time{}, messages.TraceWindowNotATime( - 0, source.AgentName, "end_time", source.EndTime) + return time.Time{}, time.Time{}, messages.RunTraceWindowNotATime( + evalName, "end_time", source.EndTime) } } if !start.IsZero() && !end.IsZero() && !end.After(start) { - // The start is named as it was written, which is lookback_hours when - // that is where it came from rather than an empty start_time. - written := source.StartTime - if written == "" { - written = fmt.Sprintf("%dh before now", source.LookbackHours) - } - return time.Time{}, time.Time{}, messages.TraceWindowEndsBeforeItStarts( - 0, source.AgentName, written, source.EndTime) + return time.Time{}, time.Time{}, messages.RunTraceWindowEndsBeforeItStarts( + evalName, startField, startValue, source.EndTime) } return start, end, nil } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_datasource_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_datasource_test.go index bc7c2a743dc..b69ece4318c 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_datasource_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_datasource_test.go @@ -88,19 +88,18 @@ func TestBuildRunDataSource_TracesPinsTheAgentVersion(t *testing.T) { assert.Equal(t, "2", ds.TraceSource.AgentVersion) } -// An explicit window travels intact. Declaring it beside lookback_hours is -// refused by config validation, but a run reached by id has no config that was -// validated, so the absolute bound still has to win rather than be dropped. +// An explicit window travels intact, in seconds and in UTC. The epochs are +// spelled out because a drift into local time, or into milliseconds, would +// still produce a window the service accepts and grades the wrong span of. func TestBuildRunDataSource_TracesCarriesAnExplicitWindow(t *testing.T) { ec := &evalContext{} group := &project.Eval{ Name: "trace-eval", Source: &project.SourceDecl{ - Type: project.SourceTypeTraces, - AgentName: "support-agent", - LookbackHours: 999, - StartTime: "2026-08-01T00:00:00Z", - EndTime: "2026-08-02T00:00:00Z", + Type: project.SourceTypeTraces, + AgentName: "support-agent", + StartTime: "2026-08-01T00:00:00Z", + EndTime: "2026-08-02T00:00:00Z", }, } @@ -125,6 +124,9 @@ func TestBuildRunDataSource_TracesRefusesAnUnusableWindow(t *testing.T) { Type: project.SourceTypeTraces, AgentName: "a", StartTime: "yesterday", }) require.Error(t, err) + // The eval is named, not the agent: the reader has to know which entry to + // go and edit, and a file can declare several trace evals over one agent. + assert.Contains(t, err.Error(), `eval "trace-eval"`) assert.Contains(t, err.Error(), "start_time") err = build(&project.SourceDecl{ @@ -132,7 +134,18 @@ func TestBuildRunDataSource_TracesRefusesAnUnusableWindow(t *testing.T) { StartTime: "2026-08-02T00:00:00Z", EndTime: "2026-08-01T00:00:00Z", }) require.Error(t, err) + assert.Contains(t, err.Error(), `eval "trace-eval"`) assert.Contains(t, err.Error(), "holds no traces") + + // A window written as a lookback is named as a lookback. Reporting an empty + // start_time sends the reader looking for a key their file does not have. + err = build(&project.SourceDecl{ + Type: project.SourceTypeTraces, AgentName: "a", + LookbackHours: 1, EndTime: "2020-01-01T00:00:00Z", + }) + require.Error(t, err) + assert.Contains(t, err.Error(), "source.lookback_hours") + assert.NotContains(t, err.Error(), "start_time") } // agent_name under source: is a filter, but an eval that names a target and diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go b/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go index d3655f783cd..3fa0e364cb7 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go @@ -1637,12 +1637,46 @@ func TraceWindowNotATime(i int, eval, field, value string) error { i, eval, field, value) } +// TraceWindowBoundUnusable reports a bound that parses but says nothing. +// +// The zero time is what the window logic reads as "no bound", so a bound that +// resolves to it would be dropped from the request rather than applied. +func TraceWindowBoundUnusable(i int, eval, field, value string) error { + return fmt.Errorf( + "evals[%d] %q: source.%s is %q, which is not a window any traces fall in: "+ + "give a time the agent was running", + i, eval, field, value) +} + // TraceWindowEndsBeforeItStarts reports a window that can hold no traces. -func TraceWindowEndsBeforeItStarts(i int, eval, start, end string) error { +// +// The start is named by the field that set it, which is lookback_hours when the +// window was written that way: naming start_time there sends a reader looking +// for a key their file does not contain. +func TraceWindowEndsBeforeItStarts(i int, eval, startField, startValue, end string) error { return fmt.Errorf( - "evals[%d] %q: source.end_time %q is not after source.start_time %q, "+ + "evals[%d] %q: source.end_time %q is not after source.%s %q, "+ "so the window holds no traces", - i, eval, end, start) + i, eval, end, startField, startValue) +} + +// RunTraceWindowNotATime reports a window bound a run cannot read. +// +// Separate from the configuration message because a run has no index to name: +// it was reached by id, or by a name that already resolved. +func RunTraceWindowNotATime(eval, field, value string) error { + return fmt.Errorf( + "eval %q: source.%s is %q, which is not a time: use RFC 3339, "+ + "for example 2026-08-18T09:00:00Z", + eval, field, value) +} + +// RunTraceWindowEndsBeforeItStarts reports a window a run would read nothing from. +func RunTraceWindowEndsBeforeItStarts(eval, startField, startValue, end string) error { + return fmt.Errorf( + "eval %q: source.end_time %q is not after source.%s %q, "+ + "so the window holds no traces", + eval, end, startField, startValue) } // TraceWindowOverSpecified reports a window declared twice over. @@ -1664,6 +1698,26 @@ func NegativeLookbackHours(i int, eval string, hours int) error { i, eval, hours) } +// LookbackTooLarge reports a lookback that would overflow into the future. +func LookbackTooLarge(i int, eval string, hours, max int) error { + return fmt.Errorf( + "evals[%d] %q: source.lookback_hours is %d, which is beyond the %d hours "+ + "a window can reach back: give a shorter lookback, or a start_time", + i, eval, hours, max) +} + +// MaxTracesMustBePositiveIn reports a negative cap written into the file. +// +// The flag that writes it is already guarded; this catches the file being +// edited afterwards, where a negative value is sent as-is and the run comes +// back empty. +func MaxTracesMustBePositiveIn(i int, eval string, maxTraces int) error { + return fmt.Errorf( + "evals[%d] %q: source.max_traces is %d, and a run cannot read fewer than "+ + "no traces: give a positive cap, or leave it out for the default", + i, eval, maxTraces) +} + // AmbiguousJudgeModel reports several deployments where only one can be used. func AmbiguousJudgeModel(models []string) error { return fmt.Errorf( diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go index aa06ffd2bcd..94493bdae08 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go @@ -6,6 +6,7 @@ package project import ( + "strconv" "strings" "time" @@ -310,41 +311,86 @@ func (c *EvalConfig) validateCatalogs() error { return nil } +// maxLookbackHours bounds `lookback_hours` at ten years. +// +// The bound exists because the value becomes a time.Duration in nanoseconds, +// which wraps negative above about 2.5 million hours: a lookback large enough +// to overflow produces a start bound in the future, and the run then reads no +// traces and says nothing about why. +const maxLookbackHours = 24 * 365 * 10 + // validateTraceWindow refuses a window a run could not use. // // Checked with the rest of the configuration rather than at run time, so a // mistyped timestamp is caught before the eval is created rather than after. +// +// The window has two spellings for where it starts -- an absolute start_time +// and a lookback in hours -- and both are resolved here, so that a window is +// judged the same way however it was written. func validateTraceWindow(i int, name string, source *SourceDecl) error { - start, end := time.Time{}, time.Time{} - - if source.StartTime != "" { - parsed, err := time.Parse(time.RFC3339, source.StartTime) - if err != nil { - return messages.TraceWindowNotATime(i, name, "start_time", source.StartTime) - } - start = parsed + // Ordered before the pair check, so a file that is wrong twice over is not + // told to delete the field that would have left it wrong in another way. + if source.LookbackHours < 0 { + return messages.NegativeLookbackHours(i, name, source.LookbackHours) } - if source.EndTime != "" { - parsed, err := time.Parse(time.RFC3339, source.EndTime) - if err != nil { - return messages.TraceWindowNotATime(i, name, "end_time", source.EndTime) - } - end = parsed + if source.LookbackHours > maxLookbackHours { + return messages.LookbackTooLarge(i, name, source.LookbackHours, maxLookbackHours) } - if !start.IsZero() && !end.IsZero() && !end.After(start) { - return messages.TraceWindowEndsBeforeItStarts(i, name, source.StartTime, source.EndTime) + if source.MaxTraces < 0 { + return messages.MaxTracesMustBePositiveIn(i, name, source.MaxTraces) } // Two ways of saying the same thing, and the file cannot say which was // meant. Every other contradictory pair here is refused rather than ranked. if source.StartTime != "" && source.LookbackHours != 0 { return messages.TraceWindowOverSpecified(i, name) } - if source.LookbackHours < 0 { - return messages.NegativeLookbackHours(i, name, source.LookbackHours) + + start, err := traceBound(i, name, "start_time", source.StartTime) + if err != nil { + return err + } + end, err := traceBound(i, name, "end_time", source.EndTime) + if err != nil { + return err + } + // The lookback is resolved to the same kind of bound the run will resolve + // it to. Without this, `lookback_hours: 24` beside an end_time from last + // year passes here and fails at `run start`, which is the failure this + // function exists to move earlier. + startField := "start_time" + startValue := source.StartTime + if start.IsZero() && source.LookbackHours > 0 { + start = time.Now().Add(-time.Duration(source.LookbackHours) * time.Hour) + startField = "lookback_hours" + startValue = strconv.Itoa(source.LookbackHours) + } + + if !start.IsZero() && !end.IsZero() && !end.After(start) { + return messages.TraceWindowEndsBeforeItStarts(i, name, startField, startValue, source.EndTime) } return nil } +// traceBound reads one end of the window. +// +// A bound that parses to the zero time is refused rather than returned: the +// zero value is what the rest of the window logic reads as "not set", so +// year one would be accepted here and then dropped from the request without +// a word, which is the whole defect this validation exists to close. +func traceBound(i int, name, field, value string) (time.Time, error) { + if value == "" { + return time.Time{}, nil + } + parsed, err := time.Parse(time.RFC3339, value) + if err != nil { + return time.Time{}, messages.TraceWindowNotATime(i, name, field, value) + } + if parsed.IsZero() { + return time.Time{}, messages.TraceWindowBoundUnusable(i, name, field, value) + } + return parsed, nil +} + func (c *EvalConfig) validateEval(i int, eval Eval) error { if eval.Dataset != "" && eval.Source != nil { return messages.DatasetAndSourceBothDeclared(i, eval.Name) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_test.go index bc3f6af9db0..c13f7eb2e17 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_test.go @@ -257,6 +257,38 @@ func TestValidate_Rejects(t *testing.T) { " lookback_hours: -24\n evaluators:\n - evaluator: builtin.relevance\n", wantErr: "cannot reach into the future", }, + { + // Large enough to overflow the nanosecond duration the hours become, + // which wraps the start bound into the future and reads no traces. + name: "lookback beyond what a duration holds", + body: "evals:\n - name: e\n source:\n type: traces\n agent_name: a\n" + + " lookback_hours: 100000000\n evaluators:\n - evaluator: builtin.relevance\n", + wantErr: "beyond the", + }, + { + // A window written as a lookback is named as one: reporting an + // empty start_time sends the reader to a key their file lacks. + name: "lookback that ends before it starts", + body: "evals:\n - name: e\n source:\n type: traces\n agent_name: a\n" + + " lookback_hours: 1\n end_time: \"2020-01-01T00:00:00Z\"\n" + + " evaluators:\n - evaluator: builtin.relevance\n", + wantErr: "source.lookback_hours", + }, + { + // Parses, then reads as "no bound" everywhere after, so the bound + // the file declared would be dropped from the request in silence. + name: "window bound at the zero time", + body: "evals:\n - name: e\n source:\n type: traces\n agent_name: a\n" + + " start_time: \"0001-01-01T00:00:00Z\"\n" + + " evaluators:\n - evaluator: builtin.relevance\n", + wantErr: "not a window any traces fall in", + }, + { + name: "negative trace cap", + body: "evals:\n - name: e\n source:\n type: traces\n agent_name: a\n" + + " max_traces: -5\n evaluators:\n - evaluator: builtin.relevance\n", + wantErr: "source.max_traces", + }, { name: "dataset without a name", body: "datasets:\n - source: ./d.jsonl\n" + oneEval, From 8d57cfca3be2fc060bf95d508d9efd8251f82a08 Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 18 Aug 2026 06:14:49 +0300 Subject: [PATCH 255/320] Say which end of the window is wrong in a sentence that parses Running the built binary over every new rule turned up one that reads as nonsense: source.end_time "2020-01-01T00:00:00Z" is not after source.lookback_hours "24" A lookback is a length and an end time is an instant, so nothing is "after" the other. The shared message took the start field as a parameter, which named the right key and still could not be read. The two cases now have their own sentence, at config time and at run time: source.lookback_hours is 24, which opens the window after source.end_time "2020-01-01T00:00:00Z", so it holds no traces The negative max_traces message lost "a run cannot read fewer than no traces", which was a riddle rather than an explanation. --- .../azure.ai.evaluations/internal/cmd/run.go | 13 +++--- .../internal/cmd/run_datasource_test.go | 7 ++-- .../internal/messages/messages.go | 40 +++++++++++++------ .../internal/project/eval_config.go | 12 +++--- .../internal/project/eval_config_test.go | 8 ++-- 5 files changed, 50 insertions(+), 30 deletions(-) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go index 18251180b4c..3f818196b62 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go @@ -14,7 +14,6 @@ import ( "os" "path/filepath" "sort" - "strconv" "strings" "time" @@ -550,7 +549,7 @@ func traceWindow(evalName string, source *project.SourceDecl) (start, end time.T return time.Time{}, time.Time{}, nil } - startField, startValue := "start_time", source.StartTime + fromLookback := false if source.StartTime != "" { start, err = time.Parse(time.RFC3339, source.StartTime) if err != nil { @@ -559,9 +558,9 @@ func traceWindow(evalName string, source *project.SourceDecl) (start, end time.T } } else if source.LookbackHours > 0 { start = time.Now().Add(-time.Duration(source.LookbackHours) * time.Hour) - // Named as the file spells it, so the error does not send a reader + // Reported as the file spells it, so the error does not send a reader // looking for a start_time they never wrote. - startField, startValue = "lookback_hours", strconv.Itoa(source.LookbackHours) + fromLookback = true } if source.EndTime != "" { @@ -572,8 +571,12 @@ func traceWindow(evalName string, source *project.SourceDecl) (start, end time.T } } if !start.IsZero() && !end.IsZero() && !end.After(start) { + if fromLookback { + return time.Time{}, time.Time{}, messages.RunTraceWindowOpensAfterItEnds( + evalName, source.LookbackHours, source.EndTime) + } return time.Time{}, time.Time{}, messages.RunTraceWindowEndsBeforeItStarts( - evalName, startField, startValue, source.EndTime) + evalName, source.StartTime, source.EndTime) } return start, end, nil } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_datasource_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_datasource_test.go index b69ece4318c..c539a09de09 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_datasource_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_datasource_test.go @@ -137,14 +137,15 @@ func TestBuildRunDataSource_TracesRefusesAnUnusableWindow(t *testing.T) { assert.Contains(t, err.Error(), `eval "trace-eval"`) assert.Contains(t, err.Error(), "holds no traces") - // A window written as a lookback is named as a lookback. Reporting an empty - // start_time sends the reader looking for a key their file does not have. + // A window written as a lookback is reported as a lookback. Reporting an + // empty start_time sends the reader looking for a key their file lacks, and + // "end_time is not after lookback_hours" compares a moment with a length. err = build(&project.SourceDecl{ Type: project.SourceTypeTraces, AgentName: "a", LookbackHours: 1, EndTime: "2020-01-01T00:00:00Z", }) require.Error(t, err) - assert.Contains(t, err.Error(), "source.lookback_hours") + assert.Contains(t, err.Error(), "source.lookback_hours is 1, which opens the window after") assert.NotContains(t, err.Error(), "start_time") } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go b/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go index 3fa0e364cb7..0e78b87f5be 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go @@ -1649,15 +1649,23 @@ func TraceWindowBoundUnusable(i int, eval, field, value string) error { } // TraceWindowEndsBeforeItStarts reports a window that can hold no traces. -// -// The start is named by the field that set it, which is lookback_hours when the -// window was written that way: naming start_time there sends a reader looking -// for a key their file does not contain. -func TraceWindowEndsBeforeItStarts(i int, eval, startField, startValue, end string) error { +func TraceWindowEndsBeforeItStarts(i int, eval, start, end string) error { return fmt.Errorf( - "evals[%d] %q: source.end_time %q is not after source.%s %q, "+ + "evals[%d] %q: source.end_time %q is not after source.start_time %q, "+ "so the window holds no traces", - i, eval, end, startField, startValue) + i, eval, end, start) +} + +// TraceWindowOpensAfterItEnds reports the same emptiness for a lookback. +// +// Its own sentence rather than the one above with a field substituted: a +// lookback is a length, not an instant, so "end_time is not after +// lookback_hours" reads as a comparison nobody can make. +func TraceWindowOpensAfterItEnds(i int, eval string, hours int, end string) error { + return fmt.Errorf( + "evals[%d] %q: source.lookback_hours is %d, which opens the window after "+ + "source.end_time %q, so it holds no traces", + i, eval, hours, end) } // RunTraceWindowNotATime reports a window bound a run cannot read. @@ -1672,11 +1680,19 @@ func RunTraceWindowNotATime(eval, field, value string) error { } // RunTraceWindowEndsBeforeItStarts reports a window a run would read nothing from. -func RunTraceWindowEndsBeforeItStarts(eval, startField, startValue, end string) error { +func RunTraceWindowEndsBeforeItStarts(eval, start, end string) error { return fmt.Errorf( - "eval %q: source.end_time %q is not after source.%s %q, "+ + "eval %q: source.end_time %q is not after source.start_time %q, "+ "so the window holds no traces", - eval, end, startField, startValue) + eval, end, start) +} + +// RunTraceWindowOpensAfterItEnds reports the same emptiness for a lookback. +func RunTraceWindowOpensAfterItEnds(eval string, hours int, end string) error { + return fmt.Errorf( + "eval %q: source.lookback_hours is %d, which opens the window after "+ + "source.end_time %q, so it holds no traces", + eval, hours, end) } // TraceWindowOverSpecified reports a window declared twice over. @@ -1713,8 +1729,8 @@ func LookbackTooLarge(i int, eval string, hours, max int) error { // back empty. func MaxTracesMustBePositiveIn(i int, eval string, maxTraces int) error { return fmt.Errorf( - "evals[%d] %q: source.max_traces is %d, and a run cannot read fewer than "+ - "no traces: give a positive cap, or leave it out for the default", + "evals[%d] %q: source.max_traces is %d: give a positive cap, or leave it "+ + "out to read as many as the service allows", i, eval, maxTraces) } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go index 94493bdae08..ebba3d8e111 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go @@ -6,7 +6,6 @@ package project import ( - "strconv" "strings" "time" @@ -357,16 +356,17 @@ func validateTraceWindow(i int, name string, source *SourceDecl) error { // it to. Without this, `lookback_hours: 24` beside an end_time from last // year passes here and fails at `run start`, which is the failure this // function exists to move earlier. - startField := "start_time" - startValue := source.StartTime + fromLookback := false if start.IsZero() && source.LookbackHours > 0 { start = time.Now().Add(-time.Duration(source.LookbackHours) * time.Hour) - startField = "lookback_hours" - startValue = strconv.Itoa(source.LookbackHours) + fromLookback = true } if !start.IsZero() && !end.IsZero() && !end.After(start) { - return messages.TraceWindowEndsBeforeItStarts(i, name, startField, startValue, source.EndTime) + if fromLookback { + return messages.TraceWindowOpensAfterItEnds(i, name, source.LookbackHours, source.EndTime) + } + return messages.TraceWindowEndsBeforeItStarts(i, name, source.StartTime, source.EndTime) } return nil } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_test.go index c13f7eb2e17..70b209de6b2 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_test.go @@ -266,13 +266,13 @@ func TestValidate_Rejects(t *testing.T) { wantErr: "beyond the", }, { - // A window written as a lookback is named as one: reporting an - // empty start_time sends the reader to a key their file lacks. - name: "lookback that ends before it starts", + // A window written as a lookback is reported as a lookback: naming + // start_time sends the reader to a key their file lacks. + name: "lookback that opens after the window ends", body: "evals:\n - name: e\n source:\n type: traces\n agent_name: a\n" + " lookback_hours: 1\n end_time: \"2020-01-01T00:00:00Z\"\n" + " evaluators:\n - evaluator: builtin.relevance\n", - wantErr: "source.lookback_hours", + wantErr: "source.lookback_hours is 1, which opens the window after", }, { // Parses, then reads as "no bound" everywhere after, so the bound From 1ea0c2aae5fd74fe0cf25a52dfc721b56dcf6e20 Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 18 Aug 2026 06:38:09 +0300 Subject: [PATCH 256/320] Resolve a trace window in one place instead of two The config check and the request builder each had their own copy of the window rules, and they had drifted on six inputs. A start_time at year one was refused by the config and dropped by the request. A negative lookback_hours was refused by the config and ignored by the request. A lookback past the bound, a negative max_traces, and a start_time declared beside a lookback were all refused by one and accepted by the other. Which rules applied depended on which door the eval came through, and nothing tested the two against each other. There is now one resolver, in the project package, called by both. The messages carry no prefix of their own; the caller says whether it has an index to name. That also settles a spelling split, since the window errors said `evals[0] "name":` where every neighbouring config error says `evals[0] (name):`. Two rules changed while unifying them: - `lookback_hours` now measures back from where the window closes, not from now. Measuring from now made validation a function of the clock: a file with a lookback and an end_time passed today and failed tomorrow, unedited, once now minus the lookback drifted past the end. It also made `azd up` non-deterministic, and one drifted window blocked read-only commands over every other eval in the file. - A bound at or before the Unix epoch is refused, not just Go's zero time. A `start_time` of 1970-01-01T00:00:00Z parses, is not Go's zero, and then serializes to a unix zero that omitempty drops from the request -- the same silent discard the check was added to close, one representation over. Reattaching a legacy trace run now pins both ends. The old shape said "the last n hours" and was re-read on every run; the new one has no lookback to carry, so an upgrade that left the end open replayed the same start against a later now on each reattach and graded a wider span than the run before it, without limit and without saying so. The recorded lookback and cap are clamped rather than trusted, because they come from builds that had no bounds on them. Window fields under `type: responses` are refused instead of silently ignored. Tests: the run layer is now checked against the same inputs as the config layer, so a future divergence fails rather than hides; the lookback bound is asserted at the boundary and against the overflow it exists to prevent; and one test turns colour back on, since pinning it off for the package left the branch that runs in a real terminal with no coverage at all. --- .../internal/cmd/legacy_trace_test.go | 50 +++++- .../internal/cmd/portal_test.go | 16 ++ .../azure.ai.evaluations/internal/cmd/run.go | 93 ++++------- .../internal/cmd/run_datasource_test.go | 69 +++++---- .../internal/messages/messages.go | 122 +++++++-------- .../internal/project/eval_config.go | 87 ++--------- .../internal/project/eval_config_test.go | 42 +++-- .../internal/project/trace_window.go | 102 ++++++++++++ .../internal/project/trace_window_test.go | 146 ++++++++++++++++++ 9 files changed, 476 insertions(+), 251 deletions(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/project/trace_window.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/project/trace_window_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/legacy_trace_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/legacy_trace_test.go index 1005244bb31..2d22a439a81 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/legacy_trace_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/legacy_trace_test.go @@ -31,16 +31,35 @@ func TestUpgradeLegacyTraceSource_MovesToThePreviewShape(t *testing.T) { assert.Equal(t, "support-agent", ds.TraceSource.AgentName) assert.Equal(t, 500, ds.TraceSource.MaxTraces) assert.InDelta(t, time.Now().Add(-24*time.Hour).Unix(), ds.TraceSource.StartTime, 60) - assert.Zero(t, ds.TraceSource.EndTime) // Nothing was pinned before, so nothing is pinned now: the upgrade must not // invent a version the previous runs were never graded against. assert.Empty(t, ds.TraceSource.AgentVersion) } -// The old shape had no start bound at all: the service applied its own default -// of seven days. Carrying such a run forward with an open start would widen it -// to all of history, grading a different set of traces than the run being -// repeated and making the query far more expensive, with nothing to say so. +// The old shape said "the last n hours" and was re-read on every run. The new +// one has no lookback, so an upgrade that left the end open would replay the +// same start against a later now on each reattach and grade a wider span than +// the run before it, without limit and without saying so. Both ends are pinned, +// which keeps the length of the window the run was actually graded over. +func TestUpgradeLegacyTraceSource_PinsBothEndsSoItCannotWiden(t *testing.T) { + ds := upgradeLegacyTraceSource(&eval_api.EvalRunDataSource{ + Type: eval_api.EvalRunDataSourceTypeTraces, + AgentName: "support-agent", + LookbackHours: 24, + }) + + require.NotNil(t, ds.TraceSource) + assert.InDelta(t, time.Now().Unix(), ds.TraceSource.EndTime, 60) + assert.Equal(t, int64(24*3600), ds.TraceSource.EndTime-ds.TraceSource.StartTime) + + // Upgrading the result again is a no-op, so the length cannot creep. + again := upgradeLegacyTraceSource(ds) + assert.Same(t, ds, again) +} + +// The old shape had no start bound, so a run that set no lookback was graded +// over whatever the service chose. Carrying it forward with no start would +// widen it to all of history instead. func TestUpgradeLegacyTraceSource_KeepsTheWindowItRanUnder(t *testing.T) { ds := upgradeLegacyTraceSource(&eval_api.EvalRunDataSource{ Type: eval_api.EvalRunDataSourceTypeTraces, @@ -48,8 +67,25 @@ func TestUpgradeLegacyTraceSource_KeepsTheWindowItRanUnder(t *testing.T) { }) require.NotNil(t, ds.TraceSource) - assert.InDelta(t, time.Now().Add(-24*7*time.Hour).Unix(), ds.TraceSource.StartTime, 60) - assert.Zero(t, ds.TraceSource.EndTime) + assert.Equal(t, int64(24*7*3600), ds.TraceSource.EndTime-ds.TraceSource.StartTime) +} + +// The recorded value comes from an older build, from before the bound existed. +// A lookback large enough to overflow the duration it becomes would put the +// start in the future, and the reattached run would read nothing. +func TestUpgradeLegacyTraceSource_ClampsWhatAnOlderBuildRecorded(t *testing.T) { + ds := upgradeLegacyTraceSource(&eval_api.EvalRunDataSource{ + Type: eval_api.EvalRunDataSourceTypeTraces, + AgentName: "support-agent", + LookbackHours: 100000000, + MaxTraces: -5, + }) + + require.NotNil(t, ds.TraceSource) + assert.Greater(t, ds.TraceSource.EndTime, ds.TraceSource.StartTime, + "an overflowed lookback puts the start after the end") + assert.Equal(t, int64(24*7*3600), ds.TraceSource.EndTime-ds.TraceSource.StartTime) + assert.Zero(t, ds.TraceSource.MaxTraces, "a negative cap is dropped, not forwarded") } // An end bound anchors the window it closes, rather than being read alongside a diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/portal_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/portal_test.go index 60a18cade0a..f4173a516e9 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/portal_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/portal_test.go @@ -11,6 +11,7 @@ import ( "azureaieval/internal/pkg/eval_api" + "github.com/fatih/color" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -37,6 +38,21 @@ func TestWritePortalLink_SilentWithoutAURL(t *testing.T) { assert.Empty(t, buf.String()) } +// Colour is pinned off for the rest of the package, which leaves nothing +// exercising the branch that actually runs in a terminal. The escape codes have +// to wrap the URL and nothing else: one leaking into the label, or past the +// newline, follows the link into whatever a reader pastes it in. +func TestWritePortalLink_WrapsOnlyTheURL(t *testing.T) { + restore := color.NoColor + color.NoColor = false + t.Cleanup(func() { color.NoColor = restore }) + + var buf bytes.Buffer + writePortalLink(&buf, "https://ai.azure.com/x") + + assert.Equal(t, "Portal: \x1b[36mhttps://ai.azure.com/x\x1b[0m\n", buf.String()) +} + // `-o json` carries the same link the terminal prints, so a pipeline reading // JSON is not the one consumer that cannot find the run in the portal. func TestRunPortalURLTravelsInJSON(t *testing.T) { diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go index 3f818196b62..e237bcf9d13 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go @@ -390,12 +390,14 @@ func (ec *evalContext) reuseDataSourceFromLastRun( return upgradeLegacyTraceSource(list.Data[0].DataSource), nil } -// legacyTraceLookbackHours is the window the old trace shape fell back to. +// legacyTraceLookbackHours is the window a legacy source with no lookback ran +// under: the service's own default of seven days. // -// The old data source had no start bound: a start_time was accepted and -// dropped, and the service graded its own default of seven days. Carrying a run -// forward with no start at all would quietly widen it to all of history, so the -// default it was actually run under is what gets written down. +// Recorded here because the old data source had no start bound of its own -- +// it carried agent_name, lookback_hours, end_time and max_traces, and nothing +// else -- so a run that set no lookback was graded over whatever the service +// chose. Carrying such a run forward with no start at all would widen it to all +// of history instead. const legacyTraceLookbackHours = 24 * 7 // upgradeLegacyTraceSource carries a run recorded under the old trace shape @@ -405,33 +407,37 @@ const legacyTraceLookbackHours = 24 * 7 // eval whose last run predates the change would keep the version-blind source // for good, and nothing would say so. // -// The window it produces is anchored at the moment of the upgrade rather than -// rolling: the new shape has no lookback to carry, and the upgraded source is -// what the next reattach reads back. A window that should move with each run -// has to come from the declaration, which is where `run start --eval ` -// reads it from anyway; this path exists only for an eval reached by id, with -// no declaration to read. +// Both bounds are written down, including the end. The old shape said "the last +// n hours" and was re-read on every run; the new one has no lookback to carry, +// so leaving the end open would let each reattach replay the same start against +// a later now and grade a wider span than the run before it, without limit and +// without saying so. Pinning both keeps the length of the window, which is the +// closest a shape with no lookback can come to one. func upgradeLegacyTraceSource(ds *eval_api.EvalRunDataSource) *eval_api.EvalRunDataSource { if ds == nil || ds.Type != eval_api.EvalRunDataSourceTypeTraces { return ds } - var end time.Time - if ds.EndTime > 0 { + end := time.Now() + if ds.EndTime != 0 { end = time.Unix(ds.EndTime, 0) } - from := end - if from.IsZero() { - from = time.Now() - } + // The recorded value is whatever an older build sent, from before the + // bound existed, so it is clamped rather than trusted: a lookback large + // enough to overflow the duration puts the start in the future, and the + // reattached run reads nothing. hours := ds.LookbackHours - if hours <= 0 { + if hours <= 0 || hours > project.MaxLookbackHours { hours = legacyTraceLookbackHours } - start := from.Add(-time.Duration(hours) * time.Hour) + maxTraces := ds.MaxTraces + if maxTraces < 0 { + maxTraces = 0 + } // The old shape carried no version, so this pins nothing that was not // pinned before; it stops the service choosing differently run to run only // once the declaration names one. - return eval_api.NewTracePreviewDataSource(ds.AgentName, "", start, end, ds.MaxTraces) + return eval_api.NewTracePreviewDataSource( + ds.AgentName, "", end.Add(-time.Duration(hours)*time.Hour), end, maxTraces) } // buildRunDataSource binds the eval's rows to the run. @@ -536,47 +542,14 @@ func tracesDataSource(group *project.Eval) (*eval_api.EvalRunDataSource, error) // traceWindow resolves the bounds of the span a trace run reads. // -// lookback_hours predates start_time and stays supported, read as a start bound -// relative to now. Config validation refuses the pair, so in practice only one -// of the two is set; the precedence here is what makes that a validation rule -// rather than the only thing standing between the file and a dropped bound. -// -// The bounds are parsed again rather than trusted, because this is the last -// place they can be refused before the request is built, and a value that fails -// here would otherwise be sent as a zero and read as "no bound". +// The rules live in the project package, with the check the configuration runs, +// so a window is judged the same way whether it is being validated or sent. The +// two used to be separate and had drifted on six inputs, each accepted by one +// and refused by the other. func traceWindow(evalName string, source *project.SourceDecl) (start, end time.Time, err error) { - if source == nil { - return time.Time{}, time.Time{}, nil - } - - fromLookback := false - if source.StartTime != "" { - start, err = time.Parse(time.RFC3339, source.StartTime) - if err != nil { - return time.Time{}, time.Time{}, messages.RunTraceWindowNotATime( - evalName, "start_time", source.StartTime) - } - } else if source.LookbackHours > 0 { - start = time.Now().Add(-time.Duration(source.LookbackHours) * time.Hour) - // Reported as the file spells it, so the error does not send a reader - // looking for a start_time they never wrote. - fromLookback = true - } - - if source.EndTime != "" { - end, err = time.Parse(time.RFC3339, source.EndTime) - if err != nil { - return time.Time{}, time.Time{}, messages.RunTraceWindowNotATime( - evalName, "end_time", source.EndTime) - } - } - if !start.IsZero() && !end.IsZero() && !end.After(start) { - if fromLookback { - return time.Time{}, time.Time{}, messages.RunTraceWindowOpensAfterItEnds( - evalName, source.LookbackHours, source.EndTime) - } - return time.Time{}, time.Time{}, messages.RunTraceWindowEndsBeforeItStarts( - evalName, source.StartTime, source.EndTime) + start, end, err = project.ResolveTraceWindow(source) + if err != nil { + return time.Time{}, time.Time{}, messages.InEval(evalName, err) } return start, end, nil } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_datasource_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_datasource_test.go index c539a09de09..ecd8de620c3 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_datasource_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_datasource_test.go @@ -112,41 +112,54 @@ func TestBuildRunDataSource_TracesCarriesAnExplicitWindow(t *testing.T) { // A window nobody can read, or one that holds nothing, is refused here rather // than by a service that answers with no rows and no reason. -func TestBuildRunDataSource_TracesRefusesAnUnusableWindow(t *testing.T) { +// +// Every input the configuration refuses has to be refused here too. The two +// used to be separate checks and had drifted on six inputs, each accepted by +// one door and refused by the other, so which rules applied depended on how the +// eval was reached. +func TestBuildRunDataSource_TracesRefusesEveryWindowTheConfigWould(t *testing.T) { ec := &evalContext{} build := func(source *project.SourceDecl) error { + source.Type = project.SourceTypeTraces + source.AgentName = "a" _, err := ec.buildRunDataSource(context.Background(), &project.Eval{Name: "trace-eval", Source: source}, "", 0) return err } - err := build(&project.SourceDecl{ - Type: project.SourceTypeTraces, AgentName: "a", StartTime: "yesterday", - }) - require.Error(t, err) - // The eval is named, not the agent: the reader has to know which entry to - // go and edit, and a file can declare several trace evals over one agent. - assert.Contains(t, err.Error(), `eval "trace-eval"`) - assert.Contains(t, err.Error(), "start_time") - - err = build(&project.SourceDecl{ - Type: project.SourceTypeTraces, AgentName: "a", - StartTime: "2026-08-02T00:00:00Z", EndTime: "2026-08-01T00:00:00Z", - }) - require.Error(t, err) - assert.Contains(t, err.Error(), `eval "trace-eval"`) - assert.Contains(t, err.Error(), "holds no traces") - - // A window written as a lookback is reported as a lookback. Reporting an - // empty start_time sends the reader looking for a key their file lacks, and - // "end_time is not after lookback_hours" compares a moment with a length. - err = build(&project.SourceDecl{ - Type: project.SourceTypeTraces, AgentName: "a", - LookbackHours: 1, EndTime: "2020-01-01T00:00:00Z", - }) - require.Error(t, err) - assert.Contains(t, err.Error(), "source.lookback_hours is 1, which opens the window after") - assert.NotContains(t, err.Error(), "start_time") + cases := []struct { + name string + source project.SourceDecl + wantErr string + }{ + {"start that is not a time", project.SourceDecl{StartTime: "yesterday"}, "not a time"}, + {"start at year one", project.SourceDecl{StartTime: "0001-01-01T00:00:00Z"}, "traces were recorded at"}, + {"start at the unix epoch", project.SourceDecl{StartTime: "1970-01-01T00:00:00Z"}, "traces were recorded at"}, + {"negative lookback", project.SourceDecl{LookbackHours: -24}, "cannot reach into the future"}, + {"lookback past the bound", project.SourceDecl{LookbackHours: project.MaxLookbackHours + 1}, "beyond the"}, + {"negative cap", project.SourceDecl{MaxTraces: -5}, "source.max_traces"}, + { + "window declared twice over", + project.SourceDecl{StartTime: "2026-08-01T00:00:00Z", LookbackHours: 24}, + "keep one", + }, + { + "end before start", + project.SourceDecl{StartTime: "2026-08-02T00:00:00Z", EndTime: "2026-08-01T00:00:00Z"}, + "holds no traces", + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + err := build(&tc.source) + require.Error(t, err) + // The eval is named, not the agent: the reader has to know which + // entry to edit, and a file can declare several evals over one agent. + assert.Contains(t, err.Error(), `eval "trace-eval"`) + assert.Contains(t, err.Error(), tc.wantErr) + }) + } } // agent_name under source: is a filter, but an eval that names a target and diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go b/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go index 0e78b87f5be..08179bf2b77 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go @@ -1629,109 +1629,91 @@ func GateNeedsATerminalRun(runID, status string) error { runID, status) } -// TraceWindowNotATime reports a window bound that is not a timestamp. -func TraceWindowNotATime(i int, eval, field, value string) error { - return fmt.Errorf( - "evals[%d] %q: source.%s is %q, which is not a time: use RFC 3339, "+ - "for example 2026-08-18T09:00:00Z", - i, eval, field, value) -} - -// TraceWindowBoundUnusable reports a bound that parses but says nothing. +// InEvalAt says which declaration an error came from. // -// The zero time is what the window logic reads as "no bound", so a bound that -// resolves to it would be dropped from the request rather than applied. -func TraceWindowBoundUnusable(i int, eval, field, value string) error { - return fmt.Errorf( - "evals[%d] %q: source.%s is %q, which is not a window any traces fall in: "+ - "give a time the agent was running", - i, eval, field, value) +// The window rules are checked in one place and reported from two, because the +// same file is read when it is validated and again when a run is built. Only +// the first of those has an index to name. +func InEvalAt(i int, eval string, err error) error { + return fmt.Errorf("evals[%d] (%s): %w", i, eval, err) } -// TraceWindowEndsBeforeItStarts reports a window that can hold no traces. -func TraceWindowEndsBeforeItStarts(i int, eval, start, end string) error { - return fmt.Errorf( - "evals[%d] %q: source.end_time %q is not after source.start_time %q, "+ - "so the window holds no traces", - i, eval, end, start) +// InEval says which eval an error came from, where there is no index. +func InEval(eval string, err error) error { + return fmt.Errorf("eval %q: %w", eval, err) } -// TraceWindowOpensAfterItEnds reports the same emptiness for a lookback. -// -// Its own sentence rather than the one above with a field substituted: a -// lookback is a length, not an instant, so "end_time is not after -// lookback_hours" reads as a comparison nobody can make. -func TraceWindowOpensAfterItEnds(i int, eval string, hours int, end string) error { +// TraceWindowNotATime reports a window bound that is not a timestamp. +func TraceWindowNotATime(field, value string) error { return fmt.Errorf( - "evals[%d] %q: source.lookback_hours is %d, which opens the window after "+ - "source.end_time %q, so it holds no traces", - i, eval, hours, end) + "source.%s is %q, which is not a time: use RFC 3339, "+ + "for example 2026-08-18T09:00:00Z", + field, value) } -// RunTraceWindowNotATime reports a window bound a run cannot read. +// TraceWindowBoundUnusable reports a bound that parses but says nothing. // -// Separate from the configuration message because a run has no index to name: -// it was reached by id, or by a name that already resolved. -func RunTraceWindowNotATime(eval, field, value string) error { +// Both this layer and the wire read a zero as "no bound", so a bound that +// resolves to one would be dropped from the request rather than applied. +func TraceWindowBoundUnusable(field, value string) error { return fmt.Errorf( - "eval %q: source.%s is %q, which is not a time: use RFC 3339, "+ - "for example 2026-08-18T09:00:00Z", - eval, field, value) + "source.%s is %q, which is not a time any traces were recorded at: "+ + "give a time the agent was running", + field, value) } -// RunTraceWindowEndsBeforeItStarts reports a window a run would read nothing from. -func RunTraceWindowEndsBeforeItStarts(eval, start, end string) error { +// TraceWindowEndsBeforeItStarts reports a window that can hold no traces. +func TraceWindowEndsBeforeItStarts(start, end string) error { return fmt.Errorf( - "eval %q: source.end_time %q is not after source.start_time %q, "+ + "source.end_time %q is not after source.start_time %q, "+ "so the window holds no traces", - eval, end, start) -} - -// RunTraceWindowOpensAfterItEnds reports the same emptiness for a lookback. -func RunTraceWindowOpensAfterItEnds(eval string, hours int, end string) error { - return fmt.Errorf( - "eval %q: source.lookback_hours is %d, which opens the window after "+ - "source.end_time %q, so it holds no traces", - eval, hours, end) + end, start) } // TraceWindowOverSpecified reports a window declared twice over. // -// lookback_hours is a start bound relative to now and start_time is an absolute -// one, so a file carrying both does not say which window was meant. -func TraceWindowOverSpecified(i int, eval string) error { +// lookback_hours measures back from where the window closes and start_time is +// an absolute bound, so a file carrying both does not say which was meant. +func TraceWindowOverSpecified() error { return fmt.Errorf( - "evals[%d] %q: source declares both start_time and lookback_hours, "+ - "which are two ways of saying where the window starts: keep one", - i, eval) + "source declares both start_time and lookback_hours, which are two ways " + + "of saying where the window opens: keep one") } // NegativeLookbackHours reports a window that reaches forwards. -func NegativeLookbackHours(i int, eval string, hours int) error { +func NegativeLookbackHours(hours int) error { return fmt.Errorf( - "evals[%d] %q: source.lookback_hours is %d, and a window cannot reach "+ - "into the future: give the hours to look back", - i, eval, hours) + "source.lookback_hours is %d, and a window cannot reach into the future: "+ + "give the hours to look back", + hours) } -// LookbackTooLarge reports a lookback that would overflow into the future. -func LookbackTooLarge(i int, eval string, hours, max int) error { +// LookbackTooLarge reports a lookback beyond the span a window may cover. +func LookbackTooLarge(hours, max int) error { return fmt.Errorf( - "evals[%d] %q: source.lookback_hours is %d, which is beyond the %d hours "+ - "a window can reach back: give a shorter lookback, or a start_time", - i, eval, hours, max) + "source.lookback_hours is %d, which is beyond the %d hours a window can "+ + "reach back: give a shorter lookback, or replace it with a start_time", + hours, max) } -// MaxTracesMustBePositiveIn reports a negative cap written into the file. +// MaxTracesUnusable reports a negative cap written into the file. // // The flag that writes it is already guarded; this catches the file being // edited afterwards, where a negative value is sent as-is and the run comes // back empty. -func MaxTracesMustBePositiveIn(i int, eval string, maxTraces int) error { +func MaxTracesUnusable(maxTraces int) error { + return fmt.Errorf( + "source.max_traces is %d: give a positive cap, or leave it out to use "+ + "the service's default", + maxTraces) +} + +// WindowOnANonTraceSource reports window fields a source cannot use. +func WindowOnANonTraceSource(sourceType, traces string) error { return fmt.Errorf( - "evals[%d] %q: source.max_traces is %d: give a positive cap, or leave it "+ - "out to read as many as the service allows", - i, eval, maxTraces) + "source declares a trace window, which a %q source does not read: "+ + "remove it, or set type to %q", + sourceType, traces) } // AmbiguousJudgeModel reports several deployments where only one can be used. diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go index ebba3d8e111..7bf46b03220 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go @@ -7,7 +7,6 @@ package project import ( "strings" - "time" "azureaieval/internal/messages" "azureaieval/internal/pkg/evalcore" @@ -310,85 +309,30 @@ func (c *EvalConfig) validateCatalogs() error { return nil } -// maxLookbackHours bounds `lookback_hours` at ten years. -// -// The bound exists because the value becomes a time.Duration in nanoseconds, -// which wraps negative above about 2.5 million hours: a lookback large enough -// to overflow produces a start bound in the future, and the run then reads no -// traces and says nothing about why. -const maxLookbackHours = 24 * 365 * 10 - // validateTraceWindow refuses a window a run could not use. // // Checked with the rest of the configuration rather than at run time, so a // mistyped timestamp is caught before the eval is created rather than after. -// -// The window has two spellings for where it starts -- an absolute start_time -// and a lookback in hours -- and both are resolved here, so that a window is -// judged the same way however it was written. +// The rules themselves live with the resolver the run also uses, so the two +// cannot come to different conclusions about the same file. func validateTraceWindow(i int, name string, source *SourceDecl) error { - // Ordered before the pair check, so a file that is wrong twice over is not - // told to delete the field that would have left it wrong in another way. - if source.LookbackHours < 0 { - return messages.NegativeLookbackHours(i, name, source.LookbackHours) - } - if source.LookbackHours > maxLookbackHours { - return messages.LookbackTooLarge(i, name, source.LookbackHours, maxLookbackHours) - } - if source.MaxTraces < 0 { - return messages.MaxTracesMustBePositiveIn(i, name, source.MaxTraces) - } - // Two ways of saying the same thing, and the file cannot say which was - // meant. Every other contradictory pair here is refused rather than ranked. - if source.StartTime != "" && source.LookbackHours != 0 { - return messages.TraceWindowOverSpecified(i, name) - } - - start, err := traceBound(i, name, "start_time", source.StartTime) - if err != nil { - return err - } - end, err := traceBound(i, name, "end_time", source.EndTime) - if err != nil { - return err - } - // The lookback is resolved to the same kind of bound the run will resolve - // it to. Without this, `lookback_hours: 24` beside an end_time from last - // year passes here and fails at `run start`, which is the failure this - // function exists to move earlier. - fromLookback := false - if start.IsZero() && source.LookbackHours > 0 { - start = time.Now().Add(-time.Duration(source.LookbackHours) * time.Hour) - fromLookback = true - } - - if !start.IsZero() && !end.IsZero() && !end.After(start) { - if fromLookback { - return messages.TraceWindowOpensAfterItEnds(i, name, source.LookbackHours, source.EndTime) - } - return messages.TraceWindowEndsBeforeItStarts(i, name, source.StartTime, source.EndTime) + if _, _, err := ResolveTraceWindow(source); err != nil { + return messages.InEvalAt(i, name, err) } return nil } -// traceBound reads one end of the window. +// validateNoTraceWindow refuses window fields on a source that cannot read one. // -// A bound that parses to the zero time is refused rather than returned: the -// zero value is what the rest of the window logic reads as "not set", so -// year one would be accepted here and then dropped from the request without -// a word, which is the whole defect this validation exists to close. -func traceBound(i int, name, field, value string) (time.Time, error) { - if value == "" { - return time.Time{}, nil - } - parsed, err := time.Parse(time.RFC3339, value) - if err != nil { - return time.Time{}, messages.TraceWindowNotATime(i, name, field, value) - } - if parsed.IsZero() { - return time.Time{}, messages.TraceWindowBoundUnusable(i, name, field, value) - } - return parsed, nil +// Silently inert fields are how a file comes to say something it does not do: +// a `lookback_hours` under `type: responses` looks like it bounds the run and +// never has. +func validateNoTraceWindow(i int, name string, source *SourceDecl) error { + if source.StartTime == "" && source.EndTime == "" && + source.LookbackHours == 0 && source.MaxTraces == 0 { + return nil + } + return messages.InEvalAt(i, name, messages.WindowOnANonTraceSource(source.Type, SourceTypeTraces)) } func (c *EvalConfig) validateEval(i int, eval Eval) error { @@ -421,6 +365,9 @@ func (c *EvalConfig) validateEval(i int, eval Eval) error { if len(eval.Source.ResponseIDs) == 0 { return messages.ResponsesSourceNeedsIDs(i, eval.Name) } + if err := validateNoTraceWindow(i, eval.Name, eval.Source); err != nil { + return err + } case "": return messages.SourceTypeRequired(i, eval.Name) default: diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_test.go index 70b209de6b2..99f851ed622 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_test.go @@ -258,21 +258,13 @@ func TestValidate_Rejects(t *testing.T) { wantErr: "cannot reach into the future", }, { - // Large enough to overflow the nanosecond duration the hours become, - // which wraps the start bound into the future and reads no traces. - name: "lookback beyond what a duration holds", + // The bound exists to keep a typo from becoming a query over every + // trace ever recorded. Checked one past it, so raising the constant + // without meaning to fails here. + name: "lookback beyond what a window may cover", body: "evals:\n - name: e\n source:\n type: traces\n agent_name: a\n" + - " lookback_hours: 100000000\n evaluators:\n - evaluator: builtin.relevance\n", - wantErr: "beyond the", - }, - { - // A window written as a lookback is reported as a lookback: naming - // start_time sends the reader to a key their file lacks. - name: "lookback that opens after the window ends", - body: "evals:\n - name: e\n source:\n type: traces\n agent_name: a\n" + - " lookback_hours: 1\n end_time: \"2020-01-01T00:00:00Z\"\n" + - " evaluators:\n - evaluator: builtin.relevance\n", - wantErr: "source.lookback_hours is 1, which opens the window after", + " lookback_hours: 87601\n evaluators:\n - evaluator: builtin.relevance\n", + wantErr: "is beyond the 87600 hours a window can reach back", }, { // Parses, then reads as "no bound" everywhere after, so the bound @@ -281,13 +273,31 @@ func TestValidate_Rejects(t *testing.T) { body: "evals:\n - name: e\n source:\n type: traces\n agent_name: a\n" + " start_time: \"0001-01-01T00:00:00Z\"\n" + " evaluators:\n - evaluator: builtin.relevance\n", - wantErr: "not a window any traces fall in", + wantErr: "not a time any traces were recorded at", + }, + { + // The wire drops a zero as readily as Go does, so an end bound at + // the epoch is the same silence one field over. + name: "end bound at the unix epoch", + body: "evals:\n - name: e\n source:\n type: traces\n agent_name: a\n" + + " end_time: \"1970-01-01T00:00:00Z\"\n" + + " evaluators:\n - evaluator: builtin.relevance\n", + wantErr: "source.end_time", }, { name: "negative trace cap", body: "evals:\n - name: e\n source:\n type: traces\n agent_name: a\n" + " max_traces: -5\n evaluators:\n - evaluator: builtin.relevance\n", - wantErr: "source.max_traces", + wantErr: "source.max_traces is -5", + }, + { + // A responses source reads no traces, so a window on it bounds + // nothing and only looks as though it does. + name: "trace window on a responses source", + body: "evals:\n - name: e\n source:\n type: responses\n" + + " response_ids: [resp_1]\n lookback_hours: 24\n" + + " evaluators:\n - evaluator: builtin.relevance\n", + wantErr: "does not read", }, { name: "dataset without a name", diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/trace_window.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/trace_window.go new file mode 100644 index 00000000000..dc8898d3e30 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/trace_window.go @@ -0,0 +1,102 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package project + +import ( + "time" + + "azureaieval/internal/messages" +) + +// MaxLookbackHours bounds `lookback_hours` at ten years. +// +// Ten years is a policy bound, not an arithmetic one: the hours become a +// time.Duration in nanoseconds, which does not overflow until about 2,562,047 +// hours. The tighter bound is here because a lookback in that range is a typo +// rather than a window, and the run it produces is expensive and empty. +const MaxLookbackHours = 24 * 365 * 10 + +// ResolveTraceWindow reads the span of traces an eval grades. +// +// One definition, called by the configuration check and again when the request +// is built. Two copies drifted apart on every axis they were not both tested +// on: the config refused a bound the request then dropped, and the request +// accepted values the config had already refused, so which rules applied +// depended on which door the eval came through. +// +// The window is resolved as well as checked, because a rule about a window can +// only be stated once the window is known, and both callers need the answer. +// +// A zero start or end means unbounded at that end. +func ResolveTraceWindow(source *SourceDecl) (start, end time.Time, err error) { + if source == nil { + return time.Time{}, time.Time{}, nil + } + + // Parsed first, so a file that is wrong in two ways names the value that + // cannot be read at all rather than the pair it also got wrong. + start, err = traceBound("start_time", source.StartTime) + if err != nil { + return time.Time{}, time.Time{}, err + } + end, err = traceBound("end_time", source.EndTime) + if err != nil { + return time.Time{}, time.Time{}, err + } + + if source.LookbackHours < 0 { + return time.Time{}, time.Time{}, messages.NegativeLookbackHours(source.LookbackHours) + } + if source.LookbackHours > MaxLookbackHours { + return time.Time{}, time.Time{}, messages.LookbackTooLarge(source.LookbackHours, MaxLookbackHours) + } + if source.MaxTraces < 0 { + return time.Time{}, time.Time{}, messages.MaxTracesUnusable(source.MaxTraces) + } + // Two ways of saying where the window opens, and the file cannot say which + // was meant. Every other contradictory pair here is refused rather than + // ranked. + if source.StartTime != "" && source.LookbackHours != 0 { + return time.Time{}, time.Time{}, messages.TraceWindowOverSpecified() + } + + // The lookback is measured back from where the window closes, which is now + // when nothing closed it. Measuring from now regardless made the window a + // function of the clock: `lookback_hours` beside an `end_time` validated + // today and failed tomorrow, with the file unchanged. + if start.IsZero() && source.LookbackHours > 0 { + from := end + if from.IsZero() { + from = time.Now() + } + return from.Add(-time.Duration(source.LookbackHours) * time.Hour), end, nil + } + + if !start.IsZero() && !end.IsZero() && !end.After(start) { + return time.Time{}, time.Time{}, messages.TraceWindowEndsBeforeItStarts( + source.StartTime, source.EndTime) + } + return start, end, nil +} + +// traceBound reads one end of the window. +// +// A bound at or before the Unix epoch is refused rather than returned. Zero is +// what every layer below reads as "no bound" -- Go's zero time here, and an +// omitted field on the wire -- so a bound that resolves to it would be dropped +// from the request without a word, which is the silence this check exists to +// break rather than to join. +func traceBound(field, value string) (time.Time, error) { + if value == "" { + return time.Time{}, nil + } + parsed, err := time.Parse(time.RFC3339, value) + if err != nil { + return time.Time{}, messages.TraceWindowNotATime(field, value) + } + if parsed.Unix() <= 0 { + return time.Time{}, messages.TraceWindowBoundUnusable(field, value) + } + return parsed, nil +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/trace_window_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/trace_window_test.go new file mode 100644 index 00000000000..db127de97ec --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/trace_window_test.go @@ -0,0 +1,146 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package project + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// The bound exists so a lookback cannot reach far enough back to overflow the +// duration it becomes, which would wrap the start into the future and read +// nothing. Asserted rather than reasoned about, so raising the constant to a +// value that does overflow fails here instead of in a run that comes back empty. +func TestLookbackBoundCannotOverflowTheDurationItBecomes(t *testing.T) { + assert.Positive(t, time.Duration(MaxLookbackHours)*time.Hour) + assert.Equal(t, MaxLookbackHours, 24*365*10) +} + +// A lookback beside an end_time used to be measured from now, which made the +// same file valid today and invalid tomorrow with nothing edited: once now +// minus the lookback drifted past the end, the window was empty for good. +// Measuring back from where the window closes takes the clock out of it. +func TestResolveTraceWindow_LookbackMeasuresBackFromTheEnd(t *testing.T) { + start, end, err := ResolveTraceWindow(&SourceDecl{ + Type: SourceTypeTraces, + AgentName: "a", + LookbackHours: 24, + EndTime: "2020-01-01T00:00:00Z", + }) + + require.NoError(t, err) + assert.Equal(t, time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC), end.UTC()) + assert.Equal(t, time.Date(2019, 12, 31, 0, 0, 0, 0, time.UTC), start.UTC()) + assert.True(t, end.After(start)) +} + +// With nothing closing the window, the lookback measures back from now. +func TestResolveTraceWindow_LookbackWithNoEndMeasuresBackFromNow(t *testing.T) { + start, end, err := ResolveTraceWindow(&SourceDecl{ + Type: SourceTypeTraces, AgentName: "a", LookbackHours: 24, + }) + + require.NoError(t, err) + assert.InDelta(t, time.Now().Add(-24*time.Hour).Unix(), start.Unix(), 60) + assert.True(t, end.IsZero(), "an open end means up to now") +} + +// A source with no window at all is not an error: both ends open is what an +// eval that never mentioned a window means. +func TestResolveTraceWindow_OpenWindowIsFine(t *testing.T) { + start, end, err := ResolveTraceWindow(&SourceDecl{Type: SourceTypeTraces, AgentName: "a"}) + + require.NoError(t, err) + assert.True(t, start.IsZero()) + assert.True(t, end.IsZero()) + + start, end, err = ResolveTraceWindow(nil) + require.NoError(t, err) + assert.True(t, start.IsZero()) + assert.True(t, end.IsZero()) +} + +// Every rule, at the boundary rather than well past it, so a bound that is +// moved by one still fails. +func TestResolveTraceWindow_Refuses(t *testing.T) { + cases := []struct { + name string + source SourceDecl + wantErr string + }{ + { + name: "start that is not a time", + source: SourceDecl{StartTime: "yesterday"}, + wantErr: "source.start_time is \"yesterday\", which is not a time", + }, + { + name: "end that is not a time", + source: SourceDecl{EndTime: "tomorrow"}, + wantErr: "source.end_time is \"tomorrow\", which is not a time", + }, + { + name: "start at year one", + source: SourceDecl{StartTime: "0001-01-01T00:00:00Z"}, + wantErr: "not a time any traces were recorded at", + }, + { + // Parses, is not Go's zero time, and still serializes to a unix + // zero that omitempty drops from the request. + name: "start at the unix epoch", + source: SourceDecl{StartTime: "1970-01-01T00:00:00Z"}, + wantErr: "not a time any traces were recorded at", + }, + { + name: "negative lookback", + source: SourceDecl{LookbackHours: -1}, + wantErr: "cannot reach into the future", + }, + { + name: "lookback one past the bound", + source: SourceDecl{LookbackHours: MaxLookbackHours + 1}, + wantErr: "beyond the 87600 hours", + }, + { + name: "negative cap", + source: SourceDecl{MaxTraces: -1}, + wantErr: "source.max_traces is -1", + }, + { + name: "window declared twice over", + source: SourceDecl{StartTime: "2026-08-01T00:00:00Z", LookbackHours: 1}, + wantErr: "keep one", + }, + { + name: "end before start", + source: SourceDecl{StartTime: "2026-08-02T00:00:00Z", EndTime: "2026-08-01T00:00:00Z"}, + wantErr: "holds no traces", + }, + { + // An instant is not a window, and a run over it reads nothing. + name: "end equal to start", + source: SourceDecl{StartTime: "2026-08-01T00:00:00Z", EndTime: "2026-08-01T00:00:00Z"}, + wantErr: "holds no traces", + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + _, _, err := ResolveTraceWindow(&tc.source) + require.Error(t, err) + assert.Contains(t, err.Error(), tc.wantErr) + }) + } +} + +// The bound is exactly on the line, so the check is `>` and not `>=`. +func TestResolveTraceWindow_AcceptsTheLargestLookbackAllowed(t *testing.T) { + start, _, err := ResolveTraceWindow(&SourceDecl{LookbackHours: MaxLookbackHours}) + + require.NoError(t, err) + assert.False(t, start.IsZero()) + assert.True(t, start.Before(time.Now()), "the window has to open in the past") +} From 9cae41cd8d940289d812222c314b4a5097d76f02 Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 18 Aug 2026 06:59:33 +0300 Subject: [PATCH 257/320] Close a reused trace window whatever shape it arrived in The last commit stopped a reattached legacy run from widening its window, and left the shape this extension writes today doing exactly that. A trace window with a start and no end means "up to now", so `run start --eval ` replayed the recorded start against a later now: the second run graded a week more than the first, the third more again, unbounded, on a per-row billed operation, with the code comment next to it asserting the opposite. Both ends are now written down for any reused trace window, not only the legacy one. The source rules moved up a level with it. `ResolveTraceWindow` only ever knew about the window, so which fields a source type actually reads was still split across two files: the config refused a `lookback_hours` under `type: responses` and the run path accepted and ignored it. `ValidateSource` now covers both, and both doors call it. What that turned up, once the rule was stated in one place: - `max_turns` was never checked for sign anywhere, so `max_turns: -3` went to the service verbatim while `max_traces: -5` was refused. Same kind of value, opposite treatment. - `response_ids` and `max_turns` under a traces source, and `agent_name` and `agent_version` under a responses source, were inert and unreported. The message names the fields rather than saying a window was declared, which was not true of a source carrying only `max_traces`. - A `target:` of `type: model` was accepted as the agent whose traces to read. A deployment name matches no spans, so the run came back empty with no reason given. Only an agent target names an agent. Two more holes in the window rules themselves: - A lookback long enough to reach past the Unix epoch produced a start the wire then drops, which is the silence a written bound at the epoch is refused for. The computed bound is held to the same rule as a written one. - A negative `max_traces` on a reattached run was clamped to zero, which is not a cap of zero: the field is omitempty, so it means the service's own default of a thousand traces, a bigger run than the one being repeated. It falls back to the bounded cap `init` writes instead. The overflow test asserted that the bound times an hour is positive, and that the bound equals its own definition. Both are constant expressions: the first would stop the package compiling rather than fail, and the second is a change detector. It goes through the resolver now. Also documented: `lookback_hours` measures back from `end_time`. The field's own doc comment still said "relative to now", which is what the last commit changed. --- .../internal/cmd/legacy_trace_test.go | 117 +++++++++++------- .../azure.ai.evaluations/internal/cmd/run.go | 74 ++++++++--- .../internal/cmd/run_datasource_test.go | 2 +- .../internal/messages/messages.go | 38 ++++-- .../internal/project/eval_config.go | 33 ++--- .../internal/project/eval_config_test.go | 4 +- .../internal/project/trace_window.go | 90 +++++++++++++- .../internal/project/trace_window_test.go | 82 ++++++++++-- 8 files changed, 329 insertions(+), 111 deletions(-) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/legacy_trace_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/legacy_trace_test.go index 2d22a439a81..9c0c9b733e3 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/legacy_trace_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/legacy_trace_test.go @@ -8,6 +8,7 @@ import ( "time" "azureaieval/internal/pkg/eval_api" + "azureaieval/internal/project" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -17,8 +18,8 @@ import ( // whose last run predates the preview shape would keep sending the old one for // good. The old shape carried no agent version, which is the whole reason to // move off it. -func TestUpgradeLegacyTraceSource_MovesToThePreviewShape(t *testing.T) { - ds := upgradeLegacyTraceSource(&eval_api.EvalRunDataSource{ +func TestPinReusedTraceWindow_MovesTheLegacyShapeOn(t *testing.T) { + ds := pinReusedTraceWindow(&eval_api.EvalRunDataSource{ Type: eval_api.EvalRunDataSourceTypeTraces, AgentName: "support-agent", LookbackHours: 24, @@ -36,32 +37,61 @@ func TestUpgradeLegacyTraceSource_MovesToThePreviewShape(t *testing.T) { assert.Empty(t, ds.TraceSource.AgentVersion) } -// The old shape said "the last n hours" and was re-read on every run. The new -// one has no lookback, so an upgrade that left the end open would replay the -// same start against a later now on each reattach and grade a wider span than -// the run before it, without limit and without saying so. Both ends are pinned, -// which keeps the length of the window the run was actually graded over. -func TestUpgradeLegacyTraceSource_PinsBothEndsSoItCannotWiden(t *testing.T) { - ds := upgradeLegacyTraceSource(&eval_api.EvalRunDataSource{ - Type: eval_api.EvalRunDataSourceTypeTraces, - AgentName: "support-agent", - LookbackHours: 24, - }) +// A window with a start and no end means "up to now", so replaying it a week +// later grades a week more than the run it was copied from, and the run after +// that more again. Both ends are written down, whatever shape the window +// arrived in, so the span cannot grow with each reattach. +func TestPinReusedTraceWindow_ClosesAWindowSoItCannotWiden(t *testing.T) { + cases := []struct { + name string + ds *eval_api.EvalRunDataSource + }{ + { + "a legacy source", + &eval_api.EvalRunDataSource{ + Type: eval_api.EvalRunDataSourceTypeTraces, + AgentName: "support-agent", + LookbackHours: 24, + }, + }, + { + // The shape this extension writes today. Pinning only the legacy + // one left the current one growing in exactly the way the legacy + // handling exists to prevent. + "a source this extension wrote", + &eval_api.EvalRunDataSource{ + Type: eval_api.EvalRunDataSourceTypeTracePreview, + TraceSource: &eval_api.TraceSourceFilter{ + Type: "agent_filter", + AgentName: "support-agent", + StartTime: time.Now().Add(-24 * time.Hour).Unix(), + }, + }, + }, + } - require.NotNil(t, ds.TraceSource) - assert.InDelta(t, time.Now().Unix(), ds.TraceSource.EndTime, 60) - assert.Equal(t, int64(24*3600), ds.TraceSource.EndTime-ds.TraceSource.StartTime) + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + ds := pinReusedTraceWindow(tc.ds) - // Upgrading the result again is a no-op, so the length cannot creep. - again := upgradeLegacyTraceSource(ds) - assert.Same(t, ds, again) + require.NotNil(t, ds.TraceSource) + assert.InDelta(t, time.Now().Unix(), ds.TraceSource.EndTime, 60) + assert.InDelta(t, int64(24*3600), ds.TraceSource.EndTime-ds.TraceSource.StartTime, 60) + + // Reusing the result again changes nothing, so the span cannot + // creep run after run. + pinned := *ds.TraceSource + again := pinReusedTraceWindow(ds) + assert.Equal(t, pinned, *again.TraceSource) + }) + } } // The old shape had no start bound, so a run that set no lookback was graded // over whatever the service chose. Carrying it forward with no start would // widen it to all of history instead. -func TestUpgradeLegacyTraceSource_KeepsTheWindowItRanUnder(t *testing.T) { - ds := upgradeLegacyTraceSource(&eval_api.EvalRunDataSource{ +func TestPinReusedTraceWindow_KeepsTheWindowALegacyRunRanUnder(t *testing.T) { + ds := pinReusedTraceWindow(&eval_api.EvalRunDataSource{ Type: eval_api.EvalRunDataSourceTypeTraces, AgentName: "support-agent", }) @@ -70,31 +100,33 @@ func TestUpgradeLegacyTraceSource_KeepsTheWindowItRanUnder(t *testing.T) { assert.Equal(t, int64(24*7*3600), ds.TraceSource.EndTime-ds.TraceSource.StartTime) } -// The recorded value comes from an older build, from before the bound existed. -// A lookback large enough to overflow the duration it becomes would put the -// start in the future, and the reattached run would read nothing. -func TestUpgradeLegacyTraceSource_ClampsWhatAnOlderBuildRecorded(t *testing.T) { - ds := upgradeLegacyTraceSource(&eval_api.EvalRunDataSource{ +// The recorded values come from an older build, from before the bounds existed. +// A lookback past the bound reaches back further than a window may cover, and a +// negative cap is no cap at all: left as zero it is dropped from the request, +// which means the service's own default of a thousand traces -- a bigger and +// costlier run than the one being repeated. +func TestPinReusedTraceWindow_ClampsWhatAnOlderBuildRecorded(t *testing.T) { + ds := pinReusedTraceWindow(&eval_api.EvalRunDataSource{ Type: eval_api.EvalRunDataSourceTypeTraces, AgentName: "support-agent", - LookbackHours: 100000000, + LookbackHours: project.MaxLookbackHours + 1, MaxTraces: -5, }) require.NotNil(t, ds.TraceSource) - assert.Greater(t, ds.TraceSource.EndTime, ds.TraceSource.StartTime, - "an overflowed lookback puts the start after the end") + assert.Greater(t, ds.TraceSource.EndTime, ds.TraceSource.StartTime) assert.Equal(t, int64(24*7*3600), ds.TraceSource.EndTime-ds.TraceSource.StartTime) - assert.Zero(t, ds.TraceSource.MaxTraces, "a negative cap is dropped, not forwarded") + assert.Equal(t, project.DefaultScaffoldMaxTraces, ds.TraceSource.MaxTraces, + "a bounded cap, rather than none at all") } // An end bound anchors the window it closes, rather than being read alongside a // start measured from now: a run that ended a month ago covered the week before // that, not the week before today. -func TestUpgradeLegacyTraceSource_MeasuresBackFromTheEnd(t *testing.T) { +func TestPinReusedTraceWindow_MeasuresBackFromTheEnd(t *testing.T) { end := time.Date(2026, 8, 2, 0, 0, 0, 0, time.UTC) - ds := upgradeLegacyTraceSource(&eval_api.EvalRunDataSource{ + ds := pinReusedTraceWindow(&eval_api.EvalRunDataSource{ Type: eval_api.EvalRunDataSourceTypeTraces, AgentName: "support-agent", LookbackHours: 24, @@ -106,18 +138,19 @@ func TestUpgradeLegacyTraceSource_MeasuresBackFromTheEnd(t *testing.T) { assert.Equal(t, end.Add(-24*time.Hour).Unix(), ds.TraceSource.StartTime) } -// Only the legacy shape is rewritten. Anything else is repeated exactly, so a -// source this extension has never heard of is not quietly replaced with one it -// made up. -func TestUpgradeLegacyTraceSource_LeavesEverythingElseAlone(t *testing.T) { - assert.Nil(t, upgradeLegacyTraceSource(nil)) +// Anything that is not an open trace window is repeated exactly, so a source +// this extension has never heard of is not quietly replaced with one it made up. +func TestPinReusedTraceWindow_LeavesEverythingElseAlone(t *testing.T) { + assert.Nil(t, pinReusedTraceWindow(nil)) - preview := &eval_api.EvalRunDataSource{ - Type: eval_api.EvalRunDataSourceTypeTracePreview, - TraceSource: &eval_api.TraceSourceFilter{Type: "agent_filter", AgentName: "a", StartTime: 7}, + closed := &eval_api.EvalRunDataSource{ + Type: eval_api.EvalRunDataSourceTypeTracePreview, + TraceSource: &eval_api.TraceSourceFilter{ + Type: "agent_filter", AgentName: "a", StartTime: 7, EndTime: 8, + }, } - assert.Same(t, preview, upgradeLegacyTraceSource(preview)) + assert.Same(t, closed, pinReusedTraceWindow(closed)) jsonl := &eval_api.EvalRunDataSource{Type: eval_api.EvalRunDataSourceTypeJSONL} - assert.Same(t, jsonl, upgradeLegacyTraceSource(jsonl)) + assert.Same(t, jsonl, pinReusedTraceWindow(jsonl)) } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go index e237bcf9d13..a199933f40e 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go @@ -387,7 +387,7 @@ func (ec *evalContext) reuseDataSourceFromLastRun( if list == nil || len(list.Data) == 0 || list.Data[0].DataSource == nil { return nil, messages.EvalHasNoPreviousRun(evalID) } - return upgradeLegacyTraceSource(list.Data[0].DataSource), nil + return pinReusedTraceWindow(list.Data[0].DataSource), nil } // legacyTraceLookbackHours is the window a legacy source with no lookback ran @@ -400,38 +400,63 @@ func (ec *evalContext) reuseDataSourceFromLastRun( // of history instead. const legacyTraceLookbackHours = 24 * 7 -// upgradeLegacyTraceSource carries a run recorded under the old trace shape -// onto the one that keeps what it is given. +// pinReusedTraceWindow closes the window a reattached run repeats. // -// A run reattached by id repeats whatever the last one sent, so without this an -// eval whose last run predates the change would keep the version-blind source -// for good, and nothing would say so. +// A run reached by id repeats whatever data source the last one sent, and a +// trace window with a start and no end means "up to now". Replaying it a week +// later grades a week more than the run it was copied from, and the run after +// that more again, so the span grows without limit and nothing says so. Every +// reused trace window therefore gets both ends written down. // -// Both bounds are written down, including the end. The old shape said "the last -// n hours" and was re-read on every run; the new one has no lookback to carry, -// so leaving the end open would let each reattach replay the same start against -// a later now and grade a wider span than the run before it, without limit and -// without saying so. Pinning both keeps the length of the window, which is the +// It is graded over the span it covers rather than the span it covered: the +// declaration is where a window that should move with each run comes from, and +// a run reached by id has no declaration to read. Freezing it once is the // closest a shape with no lookback can come to one. -func upgradeLegacyTraceSource(ds *eval_api.EvalRunDataSource) *eval_api.EvalRunDataSource { - if ds == nil || ds.Type != eval_api.EvalRunDataSourceTypeTraces { +// +// Pinning the end at now also excludes traces the service has not finished +// ingesting, which an open end would have picked up on the next run. +func pinReusedTraceWindow(ds *eval_api.EvalRunDataSource) *eval_api.EvalRunDataSource { + switch { + case ds == nil: + return ds + case ds.Type == eval_api.EvalRunDataSourceTypeTraces: + return upgradeLegacyTraceSource(ds) + case ds.Type == eval_api.EvalRunDataSourceTypeTracePreview: + if ds.TraceSource == nil || ds.TraceSource.EndTime != 0 { + return ds + } + ds.TraceSource.EndTime = time.Now().Unix() + return ds + default: return ds } +} + +// upgradeLegacyTraceSource carries a run recorded under the old trace shape +// onto the one that keeps what it is given. +// +// Without it, an eval whose last run predates the change would keep sending the +// version-blind source for good, and nothing would say so. +func upgradeLegacyTraceSource(ds *eval_api.EvalRunDataSource) *eval_api.EvalRunDataSource { end := time.Now() - if ds.EndTime != 0 { + if ds.EndTime > 0 { end = time.Unix(ds.EndTime, 0) } - // The recorded value is whatever an older build sent, from before the - // bound existed, so it is clamped rather than trusted: a lookback large + // The recorded values are whatever an older build sent, from before the + // bounds existed, so they are clamped rather than trusted: a lookback large // enough to overflow the duration puts the start in the future, and the // reattached run reads nothing. hours := ds.LookbackHours if hours <= 0 || hours > project.MaxLookbackHours { hours = legacyTraceLookbackHours } + // A negative cap is no cap at all, and leaving it off means the service's + // own default of a thousand traces -- a bigger, costlier run than the one + // being repeated. The cap `init` writes is bounded and can be raised in the + // declaration, which is the only place a considered value can come from. maxTraces := ds.MaxTraces if maxTraces < 0 { - maxTraces = 0 + maxTraces = project.DefaultScaffoldMaxTraces } // The old shape carried no version, so this pins nothing that was not // pinned before; it stops the service choosing differently run to run only @@ -520,7 +545,10 @@ func (ec *evalContext) buildRunDataSource( // run invokes nothing. func tracesDataSource(group *project.Eval) (*eval_api.EvalRunDataSource, error) { agent := group.Source.AgentName - if agent == "" && group.Target != nil { + // Only an agent target names an agent. A model target names a deployment, + // and filtering spans by a deployment name matches nothing: the run comes + // back empty with no reason given. + if agent == "" && group.Target != nil && group.Target.Type == project.TargetTypeAgent { agent = group.Target.Name } if agent == "" { @@ -543,10 +571,13 @@ func tracesDataSource(group *project.Eval) (*eval_api.EvalRunDataSource, error) // traceWindow resolves the bounds of the span a trace run reads. // // The rules live in the project package, with the check the configuration runs, -// so a window is judged the same way whether it is being validated or sent. The +// so a source is judged the same way whether it is being validated or sent. The // two used to be separate and had drifted on six inputs, each accepted by one // and refused by the other. func traceWindow(evalName string, source *project.SourceDecl) (start, end time.Time, err error) { + if err := project.ValidateSource(source); err != nil { + return time.Time{}, time.Time{}, messages.InEval(evalName, err) + } start, end, err = project.ResolveTraceWindow(source) if err != nil { return time.Time{}, time.Time{}, messages.InEval(evalName, err) @@ -559,6 +590,11 @@ func responsesDataSource(group *project.Eval) (*eval_api.EvalRunDataSource, erro if len(group.Source.ResponseIDs) == 0 { return nil, messages.ResponsesNeedIDs(group.Name) } + // The same check the configuration runs, so a field this source does not + // read is refused here too rather than only on the way to a deploy. + if err := project.ValidateSource(group.Source); err != nil { + return nil, messages.InEval(group.Name, err) + } return eval_api.NewResponsesDataSource(group.Source.ResponseIDs, group.Source.MaxTurns), nil } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_datasource_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_datasource_test.go index ecd8de620c3..e5a71ac2aca 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_datasource_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_datasource_test.go @@ -135,7 +135,7 @@ func TestBuildRunDataSource_TracesRefusesEveryWindowTheConfigWould(t *testing.T) {"start that is not a time", project.SourceDecl{StartTime: "yesterday"}, "not a time"}, {"start at year one", project.SourceDecl{StartTime: "0001-01-01T00:00:00Z"}, "traces were recorded at"}, {"start at the unix epoch", project.SourceDecl{StartTime: "1970-01-01T00:00:00Z"}, "traces were recorded at"}, - {"negative lookback", project.SourceDecl{LookbackHours: -24}, "cannot reach into the future"}, + {"negative lookback", project.SourceDecl{LookbackHours: -24}, "cannot be negative"}, {"lookback past the bound", project.SourceDecl{LookbackHours: project.MaxLookbackHours + 1}, "beyond the"}, {"negative cap", project.SourceDecl{MaxTraces: -5}, "source.max_traces"}, { diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go b/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go index 08179bf2b77..b2f3ca670ef 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go @@ -1675,25 +1675,25 @@ func TraceWindowEndsBeforeItStarts(start, end string) error { // lookback_hours measures back from where the window closes and start_time is // an absolute bound, so a file carrying both does not say which was meant. func TraceWindowOverSpecified() error { - return fmt.Errorf( + return errors.New( "source declares both start_time and lookback_hours, which are two ways " + "of saying where the window opens: keep one") } -// NegativeLookbackHours reports a window that reaches forwards. +// NegativeLookbackHours reports a lookback that is not a length. func NegativeLookbackHours(hours int) error { return fmt.Errorf( - "source.lookback_hours is %d, and a window cannot reach into the future: "+ - "give the hours to look back", + "source.lookback_hours is %d, and how far back to look cannot be "+ + "negative: give the hours to look back", hours) } // LookbackTooLarge reports a lookback beyond the span a window may cover. -func LookbackTooLarge(hours, max int) error { +func LookbackTooLarge(hours, limit int) error { return fmt.Errorf( "source.lookback_hours is %d, which is beyond the %d hours a window can "+ "reach back: give a shorter lookback, or replace it with a start_time", - hours, max) + hours, limit) } // MaxTracesUnusable reports a negative cap written into the file. @@ -1708,12 +1708,28 @@ func MaxTracesUnusable(maxTraces int) error { maxTraces) } -// WindowOnANonTraceSource reports window fields a source cannot use. -func WindowOnANonTraceSource(sourceType, traces string) error { +// SourceFieldsNotRead reports fields the declared source type ignores. +func SourceFieldsNotRead(sourceType string, fields []string) error { + return fmt.Errorf( + "source declares %s, which a %q source does not read: "+ + "remove them, or change the type to one that does", + strings.Join(fields, ", "), sourceType) +} + +// MaxTurnsUnusable reports a turn cap a run could not apply. +func MaxTurnsUnusable(maxTurns int) error { return fmt.Errorf( - "source declares a trace window, which a %q source does not read: "+ - "remove it, or set type to %q", - sourceType, traces) + "source.max_turns is %d: give a positive cap, or leave it out to use "+ + "the service's default", + maxTurns) +} + +// LookbackReachesTooFarBack reports a lookback that lands on an unusable start. +func LookbackReachesTooFarBack(hours int) error { + return fmt.Errorf( + "source.lookback_hours is %d, which opens the window before any trace "+ + "was recorded: give a shorter lookback", + hours) } // AmbiguousJudgeModel reports several deployments where only one can be used. diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go index 7bf46b03220..5d12d32636c 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go @@ -75,7 +75,8 @@ type SourceDecl struct { // version it picked. AgentVersion string `yaml:"agent_version,omitempty" json:"agent_version,omitempty"` // StartTime and EndTime bound the window explicitly. LookbackHours stays - // supported and is read as a start bound relative to now. + // supported, read as a start bound measured back from EndTime, or from now + // when nothing closes the window. StartTime string `yaml:"start_time,omitempty" json:"start_time,omitempty"` EndTime string `yaml:"end_time,omitempty" json:"end_time,omitempty"` } @@ -309,32 +310,18 @@ func (c *EvalConfig) validateCatalogs() error { return nil } -// validateTraceWindow refuses a window a run could not use. +// validateSource refuses a source declaration a run could not carry out. // -// Checked with the rest of the configuration rather than at run time, so a -// mistyped timestamp is caught before the eval is created rather than after. -// The rules themselves live with the resolver the run also uses, so the two -// cannot come to different conclusions about the same file. -func validateTraceWindow(i int, name string, source *SourceDecl) error { - if _, _, err := ResolveTraceWindow(source); err != nil { +// The rules live with the resolver the run also uses, so the two cannot come to +// different conclusions about the same file. Only the wrapper differs: here +// there is an index to name, and at run time there is not. +func validateSource(i int, name string, source *SourceDecl) error { + if err := ValidateSource(source); err != nil { return messages.InEvalAt(i, name, err) } return nil } -// validateNoTraceWindow refuses window fields on a source that cannot read one. -// -// Silently inert fields are how a file comes to say something it does not do: -// a `lookback_hours` under `type: responses` looks like it bounds the run and -// never has. -func validateNoTraceWindow(i int, name string, source *SourceDecl) error { - if source.StartTime == "" && source.EndTime == "" && - source.LookbackHours == 0 && source.MaxTraces == 0 { - return nil - } - return messages.InEvalAt(i, name, messages.WindowOnANonTraceSource(source.Type, SourceTypeTraces)) -} - func (c *EvalConfig) validateEval(i int, eval Eval) error { if eval.Dataset != "" && eval.Source != nil { return messages.DatasetAndSourceBothDeclared(i, eval.Name) @@ -358,14 +345,14 @@ func (c *EvalConfig) validateEval(i int, eval Eval) error { if eval.Source.AgentName == "" && (eval.Target == nil || eval.Target.Name == "") { return messages.TracesSourceNeedsAgentName(i, eval.Name) } - if err := validateTraceWindow(i, eval.Name, eval.Source); err != nil { + if err := validateSource(i, eval.Name, eval.Source); err != nil { return err } case SourceTypeResponses: if len(eval.Source.ResponseIDs) == 0 { return messages.ResponsesSourceNeedsIDs(i, eval.Name) } - if err := validateNoTraceWindow(i, eval.Name, eval.Source); err != nil { + if err := validateSource(i, eval.Name, eval.Source); err != nil { return err } case "": diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_test.go index 99f851ed622..4945b4529b4 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_test.go @@ -255,7 +255,7 @@ func TestValidate_Rejects(t *testing.T) { name: "lookback reaching forwards", body: "evals:\n - name: e\n source:\n type: traces\n agent_name: a\n" + " lookback_hours: -24\n evaluators:\n - evaluator: builtin.relevance\n", - wantErr: "cannot reach into the future", + wantErr: "how far back to look cannot be negative", }, { // The bound exists to keep a typo from becoming a query over every @@ -297,7 +297,7 @@ func TestValidate_Rejects(t *testing.T) { body: "evals:\n - name: e\n source:\n type: responses\n" + " response_ids: [resp_1]\n lookback_hours: 24\n" + " evaluators:\n - evaluator: builtin.relevance\n", - wantErr: "does not read", + wantErr: "source declares lookback_hours, which a \"responses\" source does not read", }, { name: "dataset without a name", diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/trace_window.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/trace_window.go index dc8898d3e30..2af1ca2df87 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/trace_window.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/trace_window.go @@ -17,6 +17,75 @@ import ( // rather than a window, and the run it produces is expensive and empty. const MaxLookbackHours = 24 * 365 * 10 +// ValidateSource refuses a source declaration a run could not carry out. +// +// The window rules and the question of which fields the declared type even +// reads, in one place, called by the configuration check and again when the +// request is built. Two copies drifted apart on every axis they were not both +// tested on, so which rules applied depended on which door the eval came +// through. +func ValidateSource(source *SourceDecl) error { + if source == nil { + return nil + } + if err := validateSourceFields(source); err != nil { + return err + } + _, _, err := ResolveTraceWindow(source) + return err +} + +// validateSourceFields refuses fields the declared source type does not read. +// +// A field that is quietly ignored is how a file comes to say something it does +// not do: a `lookback_hours` under `type: responses` looks like it bounds the +// run and never has, and nothing about the run it produces says otherwise. +func validateSourceFields(source *SourceDecl) error { + var inert []string + switch source.Type { + case SourceTypeTraces: + inert = setFields( + field{"response_ids", len(source.ResponseIDs) > 0}, + field{"max_turns", source.MaxTurns != 0}, + ) + case SourceTypeResponses: + if source.MaxTurns < 0 { + return messages.MaxTurnsUnusable(source.MaxTurns) + } + inert = setFields( + field{"start_time", source.StartTime != ""}, + field{"end_time", source.EndTime != ""}, + field{"lookback_hours", source.LookbackHours != 0}, + field{"max_traces", source.MaxTraces != 0}, + field{"agent_name", source.AgentName != ""}, + field{"agent_version", source.AgentVersion != ""}, + ) + default: + // An unsupported type is reported by the caller, which knows how to + // name the eval it came from and which types there are. + return nil + } + if len(inert) == 0 { + return nil + } + return messages.SourceFieldsNotRead(source.Type, inert) +} + +type field struct { + name string + set bool +} + +func setFields(fields ...field) []string { + var names []string + for _, f := range fields { + if f.set { + names = append(names, f.name) + } + } + return names +} + // ResolveTraceWindow reads the span of traces an eval grades. // // One definition, called by the configuration check and again when the request @@ -70,7 +139,15 @@ func ResolveTraceWindow(source *SourceDecl) (start, end time.Time, err error) { if from.IsZero() { from = time.Now() } - return from.Add(-time.Duration(source.LookbackHours) * time.Hour), end, nil + start = from.Add(-time.Duration(source.LookbackHours) * time.Hour) + // Held to the same rule as a written bound. A lookback long enough to + // reach past the epoch lands on a start the wire then drops, which is + // the silence the rule exists to break however the bound was arrived at. + if start.Unix() <= 0 { + return time.Time{}, time.Time{}, messages.LookbackReachesTooFarBack( + source.LookbackHours) + } + return start, end, nil } if !start.IsZero() && !end.IsZero() && !end.After(start) { @@ -82,11 +159,12 @@ func ResolveTraceWindow(source *SourceDecl) (start, end time.Time, err error) { // traceBound reads one end of the window. // -// A bound at or before the Unix epoch is refused rather than returned. Zero is -// what every layer below reads as "no bound" -- Go's zero time here, and an -// omitted field on the wire -- so a bound that resolves to it would be dropped -// from the request without a word, which is the silence this check exists to -// break rather than to join. +// A bound at or before the Unix epoch is refused. Exactly zero has to be +// refused because zero is what every layer below reads as "no bound" -- Go's +// zero time here, and an omitted field on the wire -- so a bound that lands on +// it would be dropped from the request without a word. The rest of the +// pre-1970 half-line is refused with it because no trace was recorded there, +// and one rule about the whole span is easier to state than a hole in it. func traceBound(field, value string) (time.Time, error) { if value == "" { return time.Time{}, nil diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/trace_window_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/trace_window_test.go index db127de97ec..a71afce3b4f 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/trace_window_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/trace_window_test.go @@ -13,11 +13,17 @@ import ( // The bound exists so a lookback cannot reach far enough back to overflow the // duration it becomes, which would wrap the start into the future and read -// nothing. Asserted rather than reasoned about, so raising the constant to a -// value that does overflow fails here instead of in a run that comes back empty. -func TestLookbackBoundCannotOverflowTheDurationItBecomes(t *testing.T) { - assert.Positive(t, time.Duration(MaxLookbackHours)*time.Hour) - assert.Equal(t, MaxLookbackHours, 24*365*10) +// nothing. Asserted through the resolver rather than against the constant: a +// comparison of the constant with its own definition cannot fail, and the +// multiplication is constant-folded, so an overflowing value would stop the +// package compiling rather than fail a test. +func TestResolveTraceWindow_TheLargestLookbackStillOpensInThePast(t *testing.T) { + start, _, err := ResolveTraceWindow(&SourceDecl{LookbackHours: MaxLookbackHours}) + + require.NoError(t, err) + assert.True(t, start.Before(time.Now()), "the window has to open in the past") + assert.True(t, start.After(time.Date(1990, 1, 1, 0, 0, 0, 0, time.UTC)), + "an overflowed duration lands centuries away, not ten years") } // A lookback beside an end_time used to be measured from now, which made the @@ -64,6 +70,69 @@ func TestResolveTraceWindow_OpenWindowIsFine(t *testing.T) { assert.True(t, end.IsZero()) } +// One end bounded and the other open is a window, not an error: "everything +// since" and "everything up to" are both things an eval can mean. +func TestResolveTraceWindow_OneEndOpenIsAWindow(t *testing.T) { + start, end, err := ResolveTraceWindow(&SourceDecl{StartTime: "2026-08-01T00:00:00Z"}) + require.NoError(t, err) + assert.Equal(t, int64(1785542400), start.Unix()) + assert.True(t, end.IsZero()) + + start, end, err = ResolveTraceWindow(&SourceDecl{EndTime: "2026-08-02T00:00:00Z"}) + require.NoError(t, err) + assert.True(t, start.IsZero()) + assert.Equal(t, int64(1785628800), end.Unix()) +} + +// A lookback long enough to reach past the epoch lands on a start the wire +// drops, which is the same silence a written bound at the epoch is refused for. +// The bound is arrived at differently and has to be held to the same rule. +func TestResolveTraceWindow_RefusesALookbackPastTheEpoch(t *testing.T) { + _, _, err := ResolveTraceWindow(&SourceDecl{ + EndTime: "1970-01-01T01:00:00Z", LookbackHours: 1, + }) + + require.Error(t, err) + assert.Contains(t, err.Error(), "before any trace was recorded") +} + +// A file wrong in two ways names the value that cannot be read at all, rather +// than a pair it also got wrong: fixing the pair would leave the unreadable +// value in place and send the reader round again. +func TestResolveTraceWindow_ReportsTheUnreadableValueFirst(t *testing.T) { + _, _, err := ResolveTraceWindow(&SourceDecl{StartTime: "yesterday", LookbackHours: -1}) + + require.Error(t, err) + assert.Contains(t, err.Error(), "which is not a time") + assert.NotContains(t, err.Error(), "lookback_hours") +} + +// Fields the declared type never reads are refused rather than ignored: a +// lookback under a responses source looks like it bounds the run and never has. +func TestValidateSource_RefusesFieldsTheTypeDoesNotRead(t *testing.T) { + err := ValidateSource(&SourceDecl{ + Type: SourceTypeResponses, ResponseIDs: []string{"resp_1"}, + LookbackHours: 24, AgentName: "a", + }) + require.Error(t, err) + // Named, because a reader with several set should not have to bisect. + assert.Contains(t, err.Error(), "lookback_hours, agent_name") + + err = ValidateSource(&SourceDecl{ + Type: SourceTypeTraces, AgentName: "a", MaxTurns: 3, + }) + require.Error(t, err) + assert.Contains(t, err.Error(), "max_turns") + + // max_traces is refused for its sign wherever it appears; max_turns is the + // same kind of value and was going out unchecked. + err = ValidateSource(&SourceDecl{ + Type: SourceTypeResponses, ResponseIDs: []string{"resp_1"}, MaxTurns: -3, + }) + require.Error(t, err) + assert.Contains(t, err.Error(), "source.max_turns is -3") +} + // Every rule, at the boundary rather than well past it, so a bound that is // moved by one still fails. func TestResolveTraceWindow_Refuses(t *testing.T) { @@ -97,7 +166,7 @@ func TestResolveTraceWindow_Refuses(t *testing.T) { { name: "negative lookback", source: SourceDecl{LookbackHours: -1}, - wantErr: "cannot reach into the future", + wantErr: "how far back to look cannot be negative", }, { name: "lookback one past the bound", @@ -142,5 +211,4 @@ func TestResolveTraceWindow_AcceptsTheLargestLookbackAllowed(t *testing.T) { require.NoError(t, err) assert.False(t, start.IsZero()) - assert.True(t, start.Before(time.Now()), "the window has to open in the past") } From a2587c2f38c781a29fb69cb8beb69ab6596a752c Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 18 Aug 2026 07:21:51 +0300 Subject: [PATCH 258/320] Stop the two doors disagreeing about an untyped target target.type is optional, and config validation accepts the traces fallback on target.name alone. Requiring an agent type at the run door therefore made a config that deploys cleanly fail every run, and the message still told the reader that declaring target.name was enough. The dataset branch twenty-five lines away already reads an untyped target as an agent; the traces branch now reads it the same way and refuses only a model target, which names a deployment and matches no spans. The rest of this round: - The source rules were being enforced on the lookup path, which is what ValidateForLookup exists to avoid: an inert `max_turns` in one entry would stop `run list --eval ` listing anything, and the way out was to hand-edit a file the error did not mention. They run on deploy, and at the run door on the entry the run is actually about. - `pinReusedTraceWindow` mutated the data source it was handed in one of its four branches and returned a fresh value in another. What the previous run sent is history; it clones now, and the contract says so. - A reused window with no start at all is left alone. It says "everything", which is what it said when it was recorded, and closing it would freeze a declaration that never asked to be bounded. Only a window with a start and an open end can widen, and that is the one that gets closed. - A recorded end early enough to put the reattached start at or before the epoch reached back from now instead, keeping the length of the window. It was the one place a start bound bypassed the rule a declaration is held to. - The run door reported `evals[0]` for an unsupported source.type, on a run that has no index. That was the distinction InEval was added for. ValidateSource returns the window it validated, so the run path no longer resolves it twice and no longer carries an unreachable second error branch. The value that is checked is the value that is sent. Tests: the idempotence check passed with the guard deleted, because two calls a microsecond apart write the same second; it asserts identity now. The run door had no coverage of the field rules it had just been given, and the responses door had none at all. --- .../internal/cmd/legacy_trace_test.go | 10 +-- .../internal/cmd/reuse_ownership_test.go | 67 ++++++++++++++++ .../azure.ai.evaluations/internal/cmd/run.go | 63 +++++++++------ .../internal/cmd/run_datasource_test.go | 72 ++++++++++++++--- .../internal/messages/messages.go | 17 +++- .../internal/project/eval_config.go | 24 ++++-- .../internal/project/trace_window.go | 80 ++++++++----------- .../internal/project/trace_window_test.go | 71 ++++++++-------- 8 files changed, 269 insertions(+), 135 deletions(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/reuse_ownership_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/legacy_trace_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/legacy_trace_test.go index 9c0c9b733e3..72e17abae81 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/legacy_trace_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/legacy_trace_test.go @@ -78,11 +78,11 @@ func TestPinReusedTraceWindow_ClosesAWindowSoItCannotWiden(t *testing.T) { assert.InDelta(t, time.Now().Unix(), ds.TraceSource.EndTime, 60) assert.InDelta(t, int64(24*3600), ds.TraceSource.EndTime-ds.TraceSource.StartTime, 60) - // Reusing the result again changes nothing, so the span cannot - // creep run after run. - pinned := *ds.TraceSource - again := pinReusedTraceWindow(ds) - assert.Equal(t, pinned, *again.TraceSource) + // An already-closed window is repeated rather than re-pinned, so + // the span cannot creep run after run. Asserted by identity: two + // calls a microsecond apart write the same second, so comparing + // the values would pass with the guard deleted. + assert.Same(t, ds, pinReusedTraceWindow(ds)) }) } } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reuse_ownership_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reuse_ownership_test.go new file mode 100644 index 00000000000..d56b62c0bf6 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reuse_ownership_test.go @@ -0,0 +1,67 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "testing" + "time" + + "azureaieval/internal/pkg/eval_api" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// What the previous run sent is history. A caller that logs it, or emits it +// under -o json, has to see what was recorded rather than what this run decided +// to send instead. +func TestPinReusedTraceWindow_DoesNotTouchWhatItWasGiven(t *testing.T) { + recorded := &eval_api.EvalRunDataSource{ + Type: eval_api.EvalRunDataSourceTypeTracePreview, + TraceSource: &eval_api.TraceSourceFilter{ + Type: "agent_filter", + AgentName: "support-agent", + StartTime: time.Now().Add(-24 * time.Hour).Unix(), + }, + } + before := *recorded.TraceSource + + pinned := pinReusedTraceWindow(recorded) + + require.NotSame(t, recorded, pinned) + assert.Equal(t, before, *recorded.TraceSource, "the recorded source is unchanged") + assert.NotZero(t, pinned.TraceSource.EndTime) +} + +// A window with no start says "everything", which is what it said when it was +// recorded. Closing it would freeze a declaration that never asked to be +// bounded, and each reattach would then grade a staler span than the last. +func TestPinReusedTraceWindow_LeavesAnUnboundedWindowUnbounded(t *testing.T) { + open := &eval_api.EvalRunDataSource{ + Type: eval_api.EvalRunDataSourceTypeTracePreview, + TraceSource: &eval_api.TraceSourceFilter{ + Type: "agent_filter", AgentName: "support-agent", + }, + } + + assert.Same(t, open, pinReusedTraceWindow(open)) + assert.Zero(t, open.TraceSource.EndTime) + assert.Zero(t, open.TraceSource.StartTime) +} + +// A recorded end early enough to put the start at or before the epoch would +// send a bound the wire drops, or a negative one, which is what a declaration +// is refused for. The length of the window is kept and reached back from now. +func TestPinReusedTraceWindow_KeepsAReattachedStartOutOfThePreEpoch(t *testing.T) { + ds := pinReusedTraceWindow(&eval_api.EvalRunDataSource{ + Type: eval_api.EvalRunDataSourceTypeTraces, + AgentName: "support-agent", + LookbackHours: 24, + EndTime: 3600, + }) + + require.NotNil(t, ds.TraceSource) + assert.Positive(t, ds.TraceSource.StartTime) + assert.Equal(t, int64(24*3600), ds.TraceSource.EndTime-ds.TraceSource.StartTime) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go index a199933f40e..569a6a80609 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go @@ -405,8 +405,11 @@ const legacyTraceLookbackHours = 24 * 7 // A run reached by id repeats whatever data source the last one sent, and a // trace window with a start and no end means "up to now". Replaying it a week // later grades a week more than the run it was copied from, and the run after -// that more again, so the span grows without limit and nothing says so. Every -// reused trace window therefore gets both ends written down. +// that more again, so the span grows without limit and nothing says so. +// +// A window with no start at all is repeated as it stands: it says "everything", +// which is what it said when it was recorded, and closing it would freeze a +// declaration that never asked to be bounded. // // It is graded over the span it covers rather than the span it covered: the // declaration is where a window that should move with each run comes from, and @@ -415,6 +418,9 @@ const legacyTraceLookbackHours = 24 * 7 // // Pinning the end at now also excludes traces the service has not finished // ingesting, which an open end would have picked up on the next run. +// +// The argument is never modified: what the previous run sent is history, and a +// caller that logs or emits it should see what was recorded. func pinReusedTraceWindow(ds *eval_api.EvalRunDataSource) *eval_api.EvalRunDataSource { switch { case ds == nil: @@ -422,11 +428,14 @@ func pinReusedTraceWindow(ds *eval_api.EvalRunDataSource) *eval_api.EvalRunDataS case ds.Type == eval_api.EvalRunDataSourceTypeTraces: return upgradeLegacyTraceSource(ds) case ds.Type == eval_api.EvalRunDataSourceTypeTracePreview: - if ds.TraceSource == nil || ds.TraceSource.EndTime != 0 { + if ds.TraceSource == nil || ds.TraceSource.EndTime != 0 || ds.TraceSource.StartTime == 0 { return ds } - ds.TraceSource.EndTime = time.Now().Unix() - return ds + pinned := *ds + filter := *ds.TraceSource + filter.EndTime = time.Now().Unix() + pinned.TraceSource = &filter + return &pinned default: return ds } @@ -443,13 +452,22 @@ func upgradeLegacyTraceSource(ds *eval_api.EvalRunDataSource) *eval_api.EvalRunD end = time.Unix(ds.EndTime, 0) } // The recorded values are whatever an older build sent, from before the - // bounds existed, so they are clamped rather than trusted: a lookback large - // enough to overflow the duration puts the start in the future, and the - // reattached run reads nothing. + // bounds existed, so they are clamped rather than trusted: a lookback beyond + // what a window may cover reaches back further than any trace was recorded, + // and the reattached run reads nothing. hours := ds.LookbackHours if hours <= 0 || hours > project.MaxLookbackHours { hours = legacyTraceLookbackHours } + start := end.Add(-time.Duration(hours) * time.Hour) + // A recorded end early enough to put the start at or before the epoch would + // send a bound the wire drops, or a negative one -- the same silence a + // declaration is refused for. Reaching back from now instead keeps the + // length of the window the run asked for. + if start.Unix() <= 0 { + end = time.Now() + start = end.Add(-time.Duration(hours) * time.Hour) + } // A negative cap is no cap at all, and leaving it off means the service's // own default of a thousand traces -- a bigger, costlier run than the one // being repeated. The cap `init` writes is bounded and can be raised in the @@ -461,8 +479,7 @@ func upgradeLegacyTraceSource(ds *eval_api.EvalRunDataSource) *eval_api.EvalRunD // The old shape carried no version, so this pins nothing that was not // pinned before; it stops the service choosing differently run to run only // once the declaration names one. - return eval_api.NewTracePreviewDataSource( - ds.AgentName, "", end.Add(-time.Duration(hours)*time.Hour), end, maxTraces) + return eval_api.NewTracePreviewDataSource(ds.AgentName, "", start, end, maxTraces) } // buildRunDataSource binds the eval's rows to the run. @@ -491,9 +508,8 @@ func (ec *evalContext) buildRunDataSource( // Config validation rejects this first, but a run reached by id has no // config to have been validated. Falling through would score the wrong // rows and say the eval declared no source. - return nil, messages.SourceTypeUnsupported( - 0, group.Name, group.Source.Type, - project.SourceTypeTraces, project.SourceTypeResponses) + return nil, messages.InEval(group.Name, messages.SourceTypeNotSupported( + group.Source.Type, project.SourceTypeTraces, project.SourceTypeResponses)) } } @@ -545,10 +561,12 @@ func (ec *evalContext) buildRunDataSource( // run invokes nothing. func tracesDataSource(group *project.Eval) (*eval_api.EvalRunDataSource, error) { agent := group.Source.AgentName - // Only an agent target names an agent. A model target names a deployment, - // and filtering spans by a deployment name matches nothing: the run comes - // back empty with no reason given. - if agent == "" && group.Target != nil && group.Target.Type == project.TargetTypeAgent { + // A model target names a deployment, and filtering spans by a deployment + // name matches nothing: the run comes back empty with no reason given. + // Anything else is read as an agent, which is how the dataset branch reads + // an untyped target too -- a config that deploys has to be one a run can + // send, and `target.type` is optional. + if agent == "" && group.Target != nil && group.Target.Type != project.TargetTypeModel { agent = group.Target.Name } if agent == "" { @@ -571,14 +589,9 @@ func tracesDataSource(group *project.Eval) (*eval_api.EvalRunDataSource, error) // traceWindow resolves the bounds of the span a trace run reads. // // The rules live in the project package, with the check the configuration runs, -// so a source is judged the same way whether it is being validated or sent. The -// two used to be separate and had drifted on six inputs, each accepted by one -// and refused by the other. +// so a source is judged the same way whichever door the eval came through. func traceWindow(evalName string, source *project.SourceDecl) (start, end time.Time, err error) { - if err := project.ValidateSource(source); err != nil { - return time.Time{}, time.Time{}, messages.InEval(evalName, err) - } - start, end, err = project.ResolveTraceWindow(source) + start, end, err = project.ValidateSource(source) if err != nil { return time.Time{}, time.Time{}, messages.InEval(evalName, err) } @@ -592,7 +605,7 @@ func responsesDataSource(group *project.Eval) (*eval_api.EvalRunDataSource, erro } // The same check the configuration runs, so a field this source does not // read is refused here too rather than only on the way to a deploy. - if err := project.ValidateSource(group.Source); err != nil { + if _, _, err := project.ValidateSource(group.Source); err != nil { return nil, messages.InEval(group.Name, err) } return eval_api.NewResponsesDataSource(group.Source.ResponseIDs, group.Source.MaxTurns), nil diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_datasource_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_datasource_test.go index e5a71ac2aca..3485bf32e87 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_datasource_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_datasource_test.go @@ -110,14 +110,11 @@ func TestBuildRunDataSource_TracesCarriesAnExplicitWindow(t *testing.T) { assert.Equal(t, int64(1785628800), ds.TraceSource.EndTime) } -// A window nobody can read, or one that holds nothing, is refused here rather -// than by a service that answers with no rows and no reason. -// // Every input the configuration refuses has to be refused here too. The two // used to be separate checks and had drifted on six inputs, each accepted by // one door and refused by the other, so which rules applied depended on how the // eval was reached. -func TestBuildRunDataSource_TracesRefusesEveryWindowTheConfigWould(t *testing.T) { +func TestBuildRunDataSource_TracesRefusesEverySourceTheConfigWould(t *testing.T) { ec := &evalContext{} build := func(source *project.SourceDecl) error { source.Type = project.SourceTypeTraces @@ -148,6 +145,14 @@ func TestBuildRunDataSource_TracesRefusesEveryWindowTheConfigWould(t *testing.T) project.SourceDecl{StartTime: "2026-08-02T00:00:00Z", EndTime: "2026-08-01T00:00:00Z"}, "holds no traces", }, + // Not a window rule. The run door used to accept and ignore these + // while the config refused them. + {"a turn cap traces do not read", project.SourceDecl{MaxTurns: 3}, "does not read"}, + { + "response ids traces do not read", + project.SourceDecl{ResponseIDs: []string{"resp_1"}}, + "does not read", + }, } for _, tc := range cases { @@ -162,21 +167,70 @@ func TestBuildRunDataSource_TracesRefusesEveryWindowTheConfigWould(t *testing.T) } } +// The responses door runs the same check. It reads no window, so a window on it +// bounds nothing and only looks as though it does. +func TestBuildRunDataSource_ResponsesRefusesFieldsItDoesNotRead(t *testing.T) { + ec := &evalContext{} + + _, err := ec.buildRunDataSource(context.Background(), &project.Eval{ + Name: "responses-eval", + Source: &project.SourceDecl{ + Type: project.SourceTypeResponses, + ResponseIDs: []string{"resp_1"}, + // Present, so the missing-ids guard does not answer first. + LookbackHours: 24, + }, + }, "", 0) + + require.Error(t, err) + assert.Contains(t, err.Error(), `eval "responses-eval"`) + assert.Contains(t, err.Error(), "lookback_hours") + assert.Contains(t, err.Error(), "does not read") +} + // agent_name under source: is a filter, but an eval that names a target and // leaves the filter off still means "this agent's traces". func TestBuildRunDataSource_TracesFallsBackToTargetName(t *testing.T) { ec := &evalContext{} + + for _, target := range []*project.Target{ + {Type: project.TargetTypeAgent, Name: "support-agent"}, + // `target.type` is optional, and config validation accepts the + // fallback on the name alone, so whatever it accepts a run has to be + // able to send. The dataset branch reads an untyped target as an agent + // too; requiring the type here made a config that deployed cleanly fail + // every run. + {Name: "support-agent"}, + } { + group := &project.Eval{ + Name: "trace-eval", + Source: &project.SourceDecl{Type: project.SourceTypeTraces}, + Target: target, + } + + ds, err := ec.buildRunDataSource(context.Background(), group, "", 0) + + require.NoError(t, err) + require.NotNil(t, ds.TraceSource) + assert.Equal(t, "support-agent", ds.TraceSource.AgentName) + } +} + +// A model target names a deployment, not an agent. Filtering spans by a +// deployment name matches nothing, so the run would come back empty with no +// reason given; saying so is more use. +func TestBuildRunDataSource_TracesWillNotReadAModelTarget(t *testing.T) { + ec := &evalContext{} group := &project.Eval{ Name: "trace-eval", Source: &project.SourceDecl{Type: project.SourceTypeTraces}, - Target: &project.Target{Type: project.TargetTypeAgent, Name: "support-agent"}, + Target: &project.Target{Type: project.TargetTypeModel, Name: "gpt-4o-mini"}, } - ds, err := ec.buildRunDataSource(context.Background(), group, "", 0) + _, err := ec.buildRunDataSource(context.Background(), group, "", 0) - require.NoError(t, err) - require.NotNil(t, ds.TraceSource) - assert.Equal(t, "support-agent", ds.TraceSource.AgentName) + require.Error(t, err) + assert.Contains(t, err.Error(), "agent_name") } // With neither, the run cannot say whose conversations to read, and saying so diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go b/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go index b2f3ca670ef..6c7dedb32c0 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go @@ -1631,7 +1631,7 @@ func GateNeedsATerminalRun(runID, status string) error { // InEvalAt says which declaration an error came from. // -// The window rules are checked in one place and reported from two, because the +// The source rules are checked in one place and reported from two, because the // same file is read when it is validated and again when a run is built. Only // the first of those has an index to name. func InEvalAt(i int, eval string, err error) error { @@ -1710,6 +1710,12 @@ func MaxTracesUnusable(maxTraces int) error { // SourceFieldsNotRead reports fields the declared source type ignores. func SourceFieldsNotRead(sourceType string, fields []string) error { + if len(fields) == 1 { + return fmt.Errorf( + "source declares %s, which a %q source does not read: "+ + "remove it, or change the type to one that does", + fields[0], sourceType) + } return fmt.Errorf( "source declares %s, which a %q source does not read: "+ "remove them, or change the type to one that does", @@ -1957,9 +1963,12 @@ func SourceTypeRequired(index int, eval string) error { // SourceTypeUnsupported reports a source.type the extension has no path for. func SourceTypeUnsupported(index int, eval, got, traces, responses string) error { - return fmt.Errorf( - "evals[%d] (%s): source.type %q is not supported; use %q or %q", - index, eval, got, traces, responses) + return InEvalAt(index, eval, SourceTypeNotSupported(got, traces, responses)) +} + +// SourceTypeNotSupported reports the same, where there is no index to name. +func SourceTypeNotSupported(got, traces, responses string) error { + return fmt.Errorf("source.type %q is not supported; use %q or %q", got, traces, responses) } // TracesSourceNeedsAgentName reports a trace source that does not say whose diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go index 5d12d32636c..094a63c7a5b 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go @@ -250,7 +250,7 @@ func (c *EvalConfig) validate(deploying bool) error { } seen[eval.Name] = true - if err := c.validateEval(i, eval); err != nil { + if err := c.validateEval(i, eval, deploying); err != nil { return err } @@ -315,14 +315,20 @@ func (c *EvalConfig) validateCatalogs() error { // The rules live with the resolver the run also uses, so the two cannot come to // different conclusions about the same file. Only the wrapper differs: here // there is an index to name, and at run time there is not. +// +// Deploy-time only. What a source says is not what resolving a declaration by +// name depends on, and enforcing it on the way to a lookup strands commands +// that had already been told which eval they meant: an inert `max_turns` in one +// entry would stop `run list --eval ` listing anything. The run door +// runs the same check on the entry it is actually about. func validateSource(i int, name string, source *SourceDecl) error { - if err := ValidateSource(source); err != nil { + if _, _, err := ValidateSource(source); err != nil { return messages.InEvalAt(i, name, err) } return nil } -func (c *EvalConfig) validateEval(i int, eval Eval) error { +func (c *EvalConfig) validateEval(i int, eval Eval, deploying bool) error { if eval.Dataset != "" && eval.Source != nil { return messages.DatasetAndSourceBothDeclared(i, eval.Name) } @@ -345,15 +351,19 @@ func (c *EvalConfig) validateEval(i int, eval Eval) error { if eval.Source.AgentName == "" && (eval.Target == nil || eval.Target.Name == "") { return messages.TracesSourceNeedsAgentName(i, eval.Name) } - if err := validateSource(i, eval.Name, eval.Source); err != nil { - return err + if deploying { + if err := validateSource(i, eval.Name, eval.Source); err != nil { + return err + } } case SourceTypeResponses: if len(eval.Source.ResponseIDs) == 0 { return messages.ResponsesSourceNeedsIDs(i, eval.Name) } - if err := validateSource(i, eval.Name, eval.Source); err != nil { - return err + if deploying { + if err := validateSource(i, eval.Name, eval.Source); err != nil { + return err + } } case "": return messages.SourceTypeRequired(i, eval.Name) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/trace_window.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/trace_window.go index 2af1ca2df87..791e5b103e3 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/trace_window.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/trace_window.go @@ -17,22 +17,26 @@ import ( // rather than a window, and the run it produces is expensive and empty. const MaxLookbackHours = 24 * 365 * 10 -// ValidateSource refuses a source declaration a run could not carry out. +// ValidateSource checks a source declaration and resolves the window it names. // -// The window rules and the question of which fields the declared type even -// reads, in one place, called by the configuration check and again when the -// request is built. Two copies drifted apart on every axis they were not both -// tested on, so which rules applied depended on which door the eval came -// through. -func ValidateSource(source *SourceDecl) error { +// One definition of what a source may say, called by the configuration check +// and again when the request is built. Two copies drifted apart on every axis +// they were not both tested on, so which rules applied depended on which door +// the eval came through. +// +// The window is returned as well as checked, because a rule about a window can +// only be stated once the window is known, and the caller that sends the +// request needs the same bounds the caller that validated it saw. +// +// A zero start or end means unbounded at that end. +func ValidateSource(source *SourceDecl) (start, end time.Time, err error) { if source == nil { - return nil + return time.Time{}, time.Time{}, nil } if err := validateSourceFields(source); err != nil { - return err + return time.Time{}, time.Time{}, err } - _, _, err := ResolveTraceWindow(source) - return err + return resolveTraceWindow(source) } // validateSourceFields refuses fields the declared source type does not read. @@ -44,21 +48,21 @@ func validateSourceFields(source *SourceDecl) error { var inert []string switch source.Type { case SourceTypeTraces: - inert = setFields( - field{"response_ids", len(source.ResponseIDs) > 0}, - field{"max_turns", source.MaxTurns != 0}, + inert = namesOfSet( + sourceField{"response_ids", len(source.ResponseIDs) > 0}, + sourceField{"max_turns", source.MaxTurns != 0}, ) case SourceTypeResponses: if source.MaxTurns < 0 { return messages.MaxTurnsUnusable(source.MaxTurns) } - inert = setFields( - field{"start_time", source.StartTime != ""}, - field{"end_time", source.EndTime != ""}, - field{"lookback_hours", source.LookbackHours != 0}, - field{"max_traces", source.MaxTraces != 0}, - field{"agent_name", source.AgentName != ""}, - field{"agent_version", source.AgentVersion != ""}, + inert = namesOfSet( + sourceField{"start_time", source.StartTime != ""}, + sourceField{"end_time", source.EndTime != ""}, + sourceField{"lookback_hours", source.LookbackHours != 0}, + sourceField{"max_traces", source.MaxTraces != 0}, + sourceField{"agent_name", source.AgentName != ""}, + sourceField{"agent_version", source.AgentVersion != ""}, ) default: // An unsupported type is reported by the caller, which knows how to @@ -71,12 +75,12 @@ func validateSourceFields(source *SourceDecl) error { return messages.SourceFieldsNotRead(source.Type, inert) } -type field struct { +type sourceField struct { name string set bool } -func setFields(fields ...field) []string { +func namesOfSet(fields ...sourceField) []string { var names []string for _, f := range fields { if f.set { @@ -86,23 +90,8 @@ func setFields(fields ...field) []string { return names } -// ResolveTraceWindow reads the span of traces an eval grades. -// -// One definition, called by the configuration check and again when the request -// is built. Two copies drifted apart on every axis they were not both tested -// on: the config refused a bound the request then dropped, and the request -// accepted values the config had already refused, so which rules applied -// depended on which door the eval came through. -// -// The window is resolved as well as checked, because a rule about a window can -// only be stated once the window is known, and both callers need the answer. -// -// A zero start or end means unbounded at that end. -func ResolveTraceWindow(source *SourceDecl) (start, end time.Time, err error) { - if source == nil { - return time.Time{}, time.Time{}, nil - } - +// resolveTraceWindow reads the span of traces an eval grades. +func resolveTraceWindow(source *SourceDecl) (start, end time.Time, err error) { // Parsed first, so a file that is wrong in two ways names the value that // cannot be read at all rather than the pair it also got wrong. start, err = traceBound("start_time", source.StartTime) @@ -142,10 +131,9 @@ func ResolveTraceWindow(source *SourceDecl) (start, end time.Time, err error) { start = from.Add(-time.Duration(source.LookbackHours) * time.Hour) // Held to the same rule as a written bound. A lookback long enough to // reach past the epoch lands on a start the wire then drops, which is - // the silence the rule exists to break however the bound was arrived at. + // the silence the rule exists to break however the bound was reached. if start.Unix() <= 0 { - return time.Time{}, time.Time{}, messages.LookbackReachesTooFarBack( - source.LookbackHours) + return time.Time{}, time.Time{}, messages.LookbackReachesTooFarBack(source.LookbackHours) } return start, end, nil } @@ -165,16 +153,16 @@ func ResolveTraceWindow(source *SourceDecl) (start, end time.Time, err error) { // it would be dropped from the request without a word. The rest of the // pre-1970 half-line is refused with it because no trace was recorded there, // and one rule about the whole span is easier to state than a hole in it. -func traceBound(field, value string) (time.Time, error) { +func traceBound(name, value string) (time.Time, error) { if value == "" { return time.Time{}, nil } parsed, err := time.Parse(time.RFC3339, value) if err != nil { - return time.Time{}, messages.TraceWindowNotATime(field, value) + return time.Time{}, messages.TraceWindowNotATime(name, value) } if parsed.Unix() <= 0 { - return time.Time{}, messages.TraceWindowBoundUnusable(field, value) + return time.Time{}, messages.TraceWindowBoundUnusable(name, value) } return parsed, nil } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/trace_window_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/trace_window_test.go index a71afce3b4f..a6840bb0a4d 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/trace_window_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/trace_window_test.go @@ -11,27 +11,12 @@ import ( "github.com/stretchr/testify/require" ) -// The bound exists so a lookback cannot reach far enough back to overflow the -// duration it becomes, which would wrap the start into the future and read -// nothing. Asserted through the resolver rather than against the constant: a -// comparison of the constant with its own definition cannot fail, and the -// multiplication is constant-folded, so an overflowing value would stop the -// package compiling rather than fail a test. -func TestResolveTraceWindow_TheLargestLookbackStillOpensInThePast(t *testing.T) { - start, _, err := ResolveTraceWindow(&SourceDecl{LookbackHours: MaxLookbackHours}) - - require.NoError(t, err) - assert.True(t, start.Before(time.Now()), "the window has to open in the past") - assert.True(t, start.After(time.Date(1990, 1, 1, 0, 0, 0, 0, time.UTC)), - "an overflowed duration lands centuries away, not ten years") -} - // A lookback beside an end_time used to be measured from now, which made the // same file valid today and invalid tomorrow with nothing edited: once now // minus the lookback drifted past the end, the window was empty for good. // Measuring back from where the window closes takes the clock out of it. -func TestResolveTraceWindow_LookbackMeasuresBackFromTheEnd(t *testing.T) { - start, end, err := ResolveTraceWindow(&SourceDecl{ +func TestValidateSource_LookbackMeasuresBackFromTheEnd(t *testing.T) { + start, end, err := ValidateSource(&SourceDecl{ Type: SourceTypeTraces, AgentName: "a", LookbackHours: 24, @@ -45,8 +30,8 @@ func TestResolveTraceWindow_LookbackMeasuresBackFromTheEnd(t *testing.T) { } // With nothing closing the window, the lookback measures back from now. -func TestResolveTraceWindow_LookbackWithNoEndMeasuresBackFromNow(t *testing.T) { - start, end, err := ResolveTraceWindow(&SourceDecl{ +func TestValidateSource_LookbackWithNoEndMeasuresBackFromNow(t *testing.T) { + start, end, err := ValidateSource(&SourceDecl{ Type: SourceTypeTraces, AgentName: "a", LookbackHours: 24, }) @@ -57,14 +42,14 @@ func TestResolveTraceWindow_LookbackWithNoEndMeasuresBackFromNow(t *testing.T) { // A source with no window at all is not an error: both ends open is what an // eval that never mentioned a window means. -func TestResolveTraceWindow_OpenWindowIsFine(t *testing.T) { - start, end, err := ResolveTraceWindow(&SourceDecl{Type: SourceTypeTraces, AgentName: "a"}) +func TestValidateSource_OpenWindowIsFine(t *testing.T) { + start, end, err := ValidateSource(&SourceDecl{Type: SourceTypeTraces, AgentName: "a"}) require.NoError(t, err) assert.True(t, start.IsZero()) assert.True(t, end.IsZero()) - start, end, err = ResolveTraceWindow(nil) + start, end, err = ValidateSource(nil) require.NoError(t, err) assert.True(t, start.IsZero()) assert.True(t, end.IsZero()) @@ -72,13 +57,13 @@ func TestResolveTraceWindow_OpenWindowIsFine(t *testing.T) { // One end bounded and the other open is a window, not an error: "everything // since" and "everything up to" are both things an eval can mean. -func TestResolveTraceWindow_OneEndOpenIsAWindow(t *testing.T) { - start, end, err := ResolveTraceWindow(&SourceDecl{StartTime: "2026-08-01T00:00:00Z"}) +func TestValidateSource_OneEndOpenIsAWindow(t *testing.T) { + start, end, err := ValidateSource(&SourceDecl{StartTime: "2026-08-01T00:00:00Z"}) require.NoError(t, err) assert.Equal(t, int64(1785542400), start.Unix()) assert.True(t, end.IsZero()) - start, end, err = ResolveTraceWindow(&SourceDecl{EndTime: "2026-08-02T00:00:00Z"}) + start, end, err = ValidateSource(&SourceDecl{EndTime: "2026-08-02T00:00:00Z"}) require.NoError(t, err) assert.True(t, start.IsZero()) assert.Equal(t, int64(1785628800), end.Unix()) @@ -87,8 +72,8 @@ func TestResolveTraceWindow_OneEndOpenIsAWindow(t *testing.T) { // A lookback long enough to reach past the epoch lands on a start the wire // drops, which is the same silence a written bound at the epoch is refused for. // The bound is arrived at differently and has to be held to the same rule. -func TestResolveTraceWindow_RefusesALookbackPastTheEpoch(t *testing.T) { - _, _, err := ResolveTraceWindow(&SourceDecl{ +func TestValidateSource_RefusesALookbackPastTheEpoch(t *testing.T) { + _, _, err := ValidateSource(&SourceDecl{ EndTime: "1970-01-01T01:00:00Z", LookbackHours: 1, }) @@ -99,8 +84,8 @@ func TestResolveTraceWindow_RefusesALookbackPastTheEpoch(t *testing.T) { // A file wrong in two ways names the value that cannot be read at all, rather // than a pair it also got wrong: fixing the pair would leave the unreadable // value in place and send the reader round again. -func TestResolveTraceWindow_ReportsTheUnreadableValueFirst(t *testing.T) { - _, _, err := ResolveTraceWindow(&SourceDecl{StartTime: "yesterday", LookbackHours: -1}) +func TestValidateSource_ReportsTheUnreadableValueFirst(t *testing.T) { + _, _, err := ValidateSource(&SourceDecl{StartTime: "yesterday", LookbackHours: -1}) require.Error(t, err) assert.Contains(t, err.Error(), "which is not a time") @@ -110,7 +95,7 @@ func TestResolveTraceWindow_ReportsTheUnreadableValueFirst(t *testing.T) { // Fields the declared type never reads are refused rather than ignored: a // lookback under a responses source looks like it bounds the run and never has. func TestValidateSource_RefusesFieldsTheTypeDoesNotRead(t *testing.T) { - err := ValidateSource(&SourceDecl{ + _, _, err := ValidateSource(&SourceDecl{ Type: SourceTypeResponses, ResponseIDs: []string{"resp_1"}, LookbackHours: 24, AgentName: "a", }) @@ -118,15 +103,17 @@ func TestValidateSource_RefusesFieldsTheTypeDoesNotRead(t *testing.T) { // Named, because a reader with several set should not have to bisect. assert.Contains(t, err.Error(), "lookback_hours, agent_name") - err = ValidateSource(&SourceDecl{ + _, _, err = ValidateSource(&SourceDecl{ Type: SourceTypeTraces, AgentName: "a", MaxTurns: 3, }) require.Error(t, err) - assert.Contains(t, err.Error(), "max_turns") + // One field reads "remove it", not "remove them". + assert.Contains(t, err.Error(), "source declares max_turns") + assert.Contains(t, err.Error(), "remove it") // max_traces is refused for its sign wherever it appears; max_turns is the // same kind of value and was going out unchecked. - err = ValidateSource(&SourceDecl{ + _, _, err = ValidateSource(&SourceDecl{ Type: SourceTypeResponses, ResponseIDs: []string{"resp_1"}, MaxTurns: -3, }) require.Error(t, err) @@ -135,7 +122,7 @@ func TestValidateSource_RefusesFieldsTheTypeDoesNotRead(t *testing.T) { // Every rule, at the boundary rather than well past it, so a bound that is // moved by one still fails. -func TestResolveTraceWindow_Refuses(t *testing.T) { +func TestValidateSource_Refuses(t *testing.T) { cases := []struct { name string source SourceDecl @@ -198,17 +185,23 @@ func TestResolveTraceWindow_Refuses(t *testing.T) { for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { - _, _, err := ResolveTraceWindow(&tc.source) + _, _, err := ValidateSource(&tc.source) require.Error(t, err) assert.Contains(t, err.Error(), tc.wantErr) }) } } -// The bound is exactly on the line, so the check is `>` and not `>=`. -func TestResolveTraceWindow_AcceptsTheLargestLookbackAllowed(t *testing.T) { - start, _, err := ResolveTraceWindow(&SourceDecl{LookbackHours: MaxLookbackHours}) +// The bound is exactly on the line, so the check is `>` and not `>=`. Asserted +// through the resolver rather than against the constant: a comparison of the +// constant with its own definition cannot fail, and the multiplication that +// would overflow is constant-folded, so an overflowing value would stop the +// package compiling rather than fail a test. +func TestValidateSource_AcceptsTheLargestLookbackAllowed(t *testing.T) { + start, _, err := ValidateSource(&SourceDecl{LookbackHours: MaxLookbackHours}) require.NoError(t, err) - assert.False(t, start.IsZero()) + assert.True(t, start.Before(time.Now()), "the window has to open in the past") + assert.True(t, start.After(time.Date(1990, 1, 1, 0, 0, 0, 0, time.UTC)), + "an overflowed duration lands centuries away, not ten years") } From b69170801356fdd24a4089d578bcfc912263d3a4 Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 18 Aug 2026 07:44:08 +0300 Subject: [PATCH 259/320] Ask which agent a trace eval reads in one place A trace eval whose only agent came from a model target deployed cleanly and failed every run. The last round taught the run door to read an untyped target as an agent and to refuse a model one, and nobody taught the config door, so the two disagreed again -- the same defect as the round before, one door over. It survived because the fix came with a run-side test and no config-side one. `project.TraceAgentName` answers it now and both doors call it. That was the last per-eval rule still written out twice by hand. The `deploying` carve-out moved to where its own comment already put it. It gated one rule in ten: a mistyped `source.type` in one entry still stopped `run list --eval ` listing anything, which is the dead end ValidateForLookup exists to remove, and the comment claimed the class was closed when it had only been narrowed. A lookup now checks what resolving a name depends on -- that it is present and unique -- and nothing else. The parameter is gone, and there are tests on both sides of the line, which there were not: deleting the gate broke nothing. `evalIDKeys` was not connected to anything. Its own doc described EVAL_ID as the documented way to point a config at an eval created elsewhere, and its own test said dropping that fallback had silently broken the behaviour once -- but the only reader of a recorded id never consulted it, so a single-eval project with EVAL_ID set by hand got "not deployed yet". `recordedEvalID` reads the keys now, and the test drives the helper the product calls. Smaller things: - The reuse door synthesised a trace source with no agent filter when the recorded run carried no agent name, which is an unfiltered query over every agent, broader than the run it was repeating. It repeats what was recorded. - A recorded end in the future closed the window after the last trace that exists. Clamped to now. - `run start --help` promised to create the eval if it did not exist. No create is reachable from it; it reports the eval is not deployed. - `messages.CreatingEval` and `CreatingEvalFailed` are the wording of that removed flow and had no callers. The refusal table in the window tests built sources with no `type`, which both doors reject before the source rules run, so it never exercised them: a change making `max_traces` inert under traces would not have failed it. --- .../internal/cmd/envkeys_test.go | 20 +++---- .../internal/cmd/evalref.go | 30 +++++++++-- .../azure.ai.evaluations/internal/cmd/run.go | 52 ++++++++----------- .../internal/cmd/run_ops.go | 5 +- .../internal/messages/messages.go | 10 ---- .../internal/project/eval_config.go | 40 +++++++------- .../internal/project/eval_config_test.go | 37 +++++++++++++ .../internal/project/trace_window.go | 25 +++++++++ .../internal/project/trace_window_test.go | 20 +++---- 9 files changed, 152 insertions(+), 87 deletions(-) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/envkeys_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/envkeys_test.go index de109689e41..a1775acc63a 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/envkeys_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/envkeys_test.go @@ -10,7 +10,6 @@ import ( "azureaieval/internal/project" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" ) // Ids are per declaration. A shared key works only while a config has one @@ -50,13 +49,12 @@ func TestIDKey_DoesNotCollideWithVersionKey(t *testing.T) { // Setting EVAL_ID by hand is the documented way to point a config at an eval // that already exists. It is also the key the extension writes itself, which is -// what let a second eval adopt the first one's id — so it stays readable only +// what let a second eval adopt the first one's id -- so it stays readable only // where it cannot be ambiguous. Fixing the aliasing dropped this fallback -// entirely once, silently breaking the documented behaviour. +// entirely once, silently breaking the documented behaviour, and it was later +// left in a helper that nothing called. func TestGroupIDKeys_SharedKeyReadOnlyWhenUnambiguous(t *testing.T) { - write := func(t *testing.T, names ...string) string { - t.Helper() - dir := t.TempDir() + configOf := func(names ...string) *project.EvalConfig { cfg := &project.EvalConfig{} for _, n := range names { cfg.Evals = append(cfg.Evals, project.Eval{ @@ -64,17 +62,19 @@ func TestGroupIDKeys_SharedKeyReadOnlyWhenUnambiguous(t *testing.T) { Evaluators: evalcore.EvaluatorList{{Evaluator: "builtin.relevance"}}, }) } - require.NoError(t, project.SaveEvalConfig(dir, cfg)) - return dir + return cfg } - sole := evalIDKeys("quality", write(t, "quality")) + sole := evalIDKeys(configOf("quality"), "quality") assert.Equal(t, idKey("eval", "quality"), sole[0], "an eval's own entry is preferred over the shared one") assert.Contains(t, sole, envKeyEvalID, "a project with one eval honours an id set by hand") assert.Equal(t, []string{idKey("eval", "quality")}, - evalIDKeys("quality", write(t, "quality", "nightly")), + evalIDKeys(configOf("quality", "nightly"), "quality"), "with several evals the shared entry cannot say which one it means") + + assert.Equal(t, []string{idKey("eval", "quality")}, evalIDKeys(nil, "quality"), + "an eval reached without a config has only its own entry") } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evalref.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evalref.go index 4aefaf35a6b..b75e44bc25d 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evalref.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evalref.go @@ -55,7 +55,7 @@ func (ec *evalContext) resolveEvalRef( eval, err := cfg.Eval(nameOrID) switch { case err == nil: - id := ec.recordedEvalID(ctx, eval.Name) + id := ec.recordedEvalID(ctx, cfg, eval.Name) if id == "" { // Nothing recorded is not the same as nothing published. The // id is kept in the azd environment, so a run against @@ -97,7 +97,7 @@ func (ec *evalContext) resolveEvalRef( // leaves it empty, and recordedEvalID answers "" without // asking when it is. Now that there is a name, ask properly // before reporting a deployed eval as missing. - if id = ec.recordedEvalID(ctx, eval.Name); id == "" { + if id = ec.recordedEvalID(ctx, cfg, eval.Name); id == "" { return evalRef{}, messages.EvalNotDeployedYet( eval.Name, ec.deployCommand(ctx)) } @@ -118,8 +118,30 @@ func (ec *evalContext) resolveEvalRef( } // recordedEvalID reads the id `azd up` stored for a declared eval. -func (ec *evalContext) recordedEvalID(ctx context.Context, evalName string) string { - return ec.getEnvValue(ctx, idKey("eval", evalName)) +func (ec *evalContext) recordedEvalID(ctx context.Context, cfg *project.EvalConfig, evalName string) string { + for _, key := range evalIDKeys(cfg, evalName) { + if id := ec.getEnvValue(ctx, key); id != "" { + return id + } + } + return "" +} + +// evalIDKeys lists the env entries that may hold this eval's id, most specific +// first. +// +// The per-name entry is what the extension writes. EVAL_ID is the documented +// way to point a config at an eval that already exists, created in the portal +// or by another tool, so it stays readable -- but only when the configuration +// declares a single eval. With more than one there is no way to tell which eval +// a shared entry refers to, and reading it anyway is what let a second eval +// adopt the first one's id. +func evalIDKeys(cfg *project.EvalConfig, name string) []string { + keys := []string{idKey("eval", name)} + if cfg != nil && len(cfg.Evals) == 1 { + keys = append(keys, envKeyEvalID) + } + return keys } // evalIDNamed finds the id of the eval the service lists under this name. diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go index 569a6a80609..56b8726c825 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go @@ -82,7 +82,7 @@ func newRunCommand() *cobra.Command { } addRunSubcommands(cmd) cmd.AddCommand(buildRunCommand( - "start", "Start a run, creating the eval if it does not exist yet.")) + "start", "Start a run of an eval that has been deployed.")) return cmd } @@ -301,24 +301,6 @@ func buildRunCommand(use, short string) *cobra.Command { return cmd } -// evalIDKeys lists the env entries that may hold this eval's id, most -// specific first. -// -// The per-name entry is what the extension writes. EVAL_ID is also the -// documented way to point a config at an eval that already exists, created in -// the portal or by another tool, so it stays readable — but only when the -// configuration declares a single eval. With more than one there is no way to -// tell which eval a shared entry refers to, and reading it anyway is what let a -// second eval adopt the first one's id. -func evalIDKeys(name, evalDir string) []string { - keys := []string{idKey("eval", name)} - if cfg, err := project.OpenEvalConfig(evalDir); err == nil && - cfg != nil && len(cfg.Evals) == 1 { - keys = append(keys, envKeyEvalID) - } - return keys -} - // checkDatasetRegistered fails when the group's local dataset has edits that // were never deployed. // @@ -409,7 +391,8 @@ const legacyTraceLookbackHours = 24 * 7 // // A window with no start at all is repeated as it stands: it says "everything", // which is what it said when it was recorded, and closing it would freeze a -// declaration that never asked to be bounded. +// declaration that never asked to be bounded. So is a window that already has +// an end, which cannot widen and is not this function's to move. // // It is graded over the span it covers rather than the span it covered: the // declaration is where a window that should move with each run comes from, and @@ -447,9 +430,21 @@ func pinReusedTraceWindow(ds *eval_api.EvalRunDataSource) *eval_api.EvalRunDataS // Without it, an eval whose last run predates the change would keep sending the // version-blind source for good, and nothing would say so. func upgradeLegacyTraceSource(ds *eval_api.EvalRunDataSource) *eval_api.EvalRunDataSource { + // Without an agent the preview shape carries no filter at all, and + // omitempty drops it: the reattached run would read every agent's spans, + // a broader and costlier query than the one it is repeating. Repeating + // what was recorded is the lesser wrong. + if ds.AgentName == "" { + return ds + } end := time.Now() if ds.EndTime > 0 { end = time.Unix(ds.EndTime, 0) + // A recorded end in the future would close the window after the last + // trace that exists, which reads nothing past now and says nothing. + if end.After(time.Now()) { + end = time.Now() + } } // The recorded values are whatever an older build sent, from before the // bounds existed, so they are clamped rather than trusted: a lookback beyond @@ -468,6 +463,13 @@ func upgradeLegacyTraceSource(ds *eval_api.EvalRunDataSource) *eval_api.EvalRunD end = time.Now() start = end.Add(-time.Duration(hours) * time.Hour) } + // Only reachable on a machine whose clock is set before about 1980, where + // even now minus the longest window a declaration may name lands in the + // pre-epoch. Dropping the bound says "everything", which is at least what + // the legacy shape said when it carried no start. + if start.Unix() <= 0 { + start = time.Time{} + } // A negative cap is no cap at all, and leaving it off means the service's // own default of a thousand traces -- a bigger, costlier run than the one // being repeated. The cap `init` writes is bounded and can be raised in the @@ -560,15 +562,7 @@ func (ec *evalContext) buildRunDataSource( // found. `agent_name` filters the traces; it is not a target, because a trace // run invokes nothing. func tracesDataSource(group *project.Eval) (*eval_api.EvalRunDataSource, error) { - agent := group.Source.AgentName - // A model target names a deployment, and filtering spans by a deployment - // name matches nothing: the run comes back empty with no reason given. - // Anything else is read as an agent, which is how the dataset branch reads - // an untyped target too -- a config that deploys has to be one a run can - // send, and `target.type` is optional. - if agent == "" && group.Target != nil && group.Target.Type != project.TargetTypeModel { - agent = group.Target.Name - } + agent := project.TraceAgentName(group.Source, group.Target) if agent == "" { return nil, messages.TracesNeedAgentName(group.Name) } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go index 36db38b6da9..ced8d6bac06 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go @@ -17,9 +17,8 @@ import ( // addRunSubcommands attaches the atomic run operations. // -// `azd ai eval run` stays the composite that creates the group if needed and -// starts a run; these expose the individual operations so every one is -// reachable without the config file. +// `azd ai eval run` is a group rather than a verb; these are the operations it +// groups, each reachable without the config file. func addRunSubcommands(cmd *cobra.Command) { cmd.AddCommand( newRunListCommand(), diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go b/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go index 6c7dedb32c0..1c43c578973 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go @@ -154,16 +154,6 @@ func DatasetHasUnregisteredEdits(dataset, deployCmd string) error { dataset, deployCmd) } -// CreatingEval reports an eval being created because it has never been deployed. -func CreatingEval(eval string) string { - return fmt.Sprintf("Creating eval %q...\n", eval) -} - -// CreatingEvalFailed reports the service refusing to create the eval. -func CreatingEvalFailed(eval string, err error) error { - return fmt.Errorf("creating eval %q: %w", eval, err) -} - // StartingRun reports the service refusing to start the run. func StartingRun(err error) error { return fmt.Errorf("starting the evaluation run: %w", err) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go index 094a63c7a5b..9cc709dd7a1 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go @@ -250,14 +250,22 @@ func (c *EvalConfig) validate(deploying bool) error { } seen[eval.Name] = true - if err := c.validateEval(i, eval, deploying); err != nil { - return err - } - if !deploying { + // Only what resolving a declaration by name depends on, which is a + // name that is present and not shared. Everything an eval says about + // itself is checked on the way to deploying it, and again at the run + // door on the entry the run is actually about. Enforcing it here + // stranded commands that had already been told which eval they + // meant: one malformed entry stopped `run list --eval ` + // listing anything, and the way out was to hand-edit a file the + // error did not mention. continue } + if err := c.validateEval(i, eval); err != nil { + return err + } + // Two evals that differ only by name are indistinguishable once // deployed: the environment records an id against each eval's substance // so a renamed declaration can find what it already deployed, and a @@ -315,12 +323,6 @@ func (c *EvalConfig) validateCatalogs() error { // The rules live with the resolver the run also uses, so the two cannot come to // different conclusions about the same file. Only the wrapper differs: here // there is an index to name, and at run time there is not. -// -// Deploy-time only. What a source says is not what resolving a declaration by -// name depends on, and enforcing it on the way to a lookup strands commands -// that had already been told which eval they meant: an inert `max_turns` in one -// entry would stop `run list --eval ` listing anything. The run door -// runs the same check on the entry it is actually about. func validateSource(i int, name string, source *SourceDecl) error { if _, _, err := ValidateSource(source); err != nil { return messages.InEvalAt(i, name, err) @@ -328,7 +330,7 @@ func validateSource(i int, name string, source *SourceDecl) error { return nil } -func (c *EvalConfig) validateEval(i int, eval Eval, deploying bool) error { +func (c *EvalConfig) validateEval(i int, eval Eval) error { if eval.Dataset != "" && eval.Source != nil { return messages.DatasetAndSourceBothDeclared(i, eval.Name) } @@ -347,23 +349,19 @@ func (c *EvalConfig) validateEval(i int, eval Eval, deploying bool) error { case SourceTypeTraces: // The run needs one of these to say whose traces to read. Refusing // here rather than at run time keeps a config that cannot run from - // deploying. - if eval.Source.AgentName == "" && (eval.Target == nil || eval.Target.Name == "") { + // deploying, so it has to be the same question the run asks. + if TraceAgentName(eval.Source, eval.Target) == "" { return messages.TracesSourceNeedsAgentName(i, eval.Name) } - if deploying { - if err := validateSource(i, eval.Name, eval.Source); err != nil { - return err - } + if err := validateSource(i, eval.Name, eval.Source); err != nil { + return err } case SourceTypeResponses: if len(eval.Source.ResponseIDs) == 0 { return messages.ResponsesSourceNeedsIDs(i, eval.Name) } - if deploying { - if err := validateSource(i, eval.Name, eval.Source); err != nil { - return err - } + if err := validateSource(i, eval.Name, eval.Source); err != nil { + return err } case "": return messages.SourceTypeRequired(i, eval.Name) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_test.go index 4945b4529b4..ba81f5fcdf3 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_test.go @@ -223,6 +223,16 @@ func TestValidate_Rejects(t *testing.T) { " evaluators:\n - evaluator: builtin.relevance\n", wantErr: "source.agent_name is required", }, + { + // A model target names a deployment. The run refuses to filter + // spans by one, so accepting it here would deploy a config that + // cannot run -- which is what this check exists to prevent. + name: "trace source pointed at a model target", + body: "evals:\n - name: e\n source:\n type: traces\n" + + " target:\n type: model\n name: gpt-4o-mini\n" + + " evaluators:\n - evaluator: builtin.relevance\n", + wantErr: "source.agent_name is required", + }, { name: "responses source listing no ids", body: "evals:\n - name: e\n source:\n type: responses\n" + @@ -417,6 +427,33 @@ func TestValidateForLookupStillRefusesADuplicateName(t *testing.T) { require.Contains(t, err.Error(), "duplicate") } +// What an eval says about itself is not what resolving it by name depends on. +// One malformed entry used to stop `run list --eval ` listing +// anything, and the way out was to hand-edit a file the error did not mention. +// The run door checks the entry the run is actually about. +func TestValidateForLookupLeavesAnEvalsOwnDeclarationToDeploying(t *testing.T) { + cases := map[string]string{ + "a field the source does not read": "evals:\n - name: a\n source:\n" + + " type: traces\n agent_name: x\n max_turns: 3\n" + + " evaluators:\n - evaluator: builtin.relevance\n", + "a mistyped source type": "evals:\n - name: a\n source:\n type: tracs\n" + + " evaluators:\n - evaluator: builtin.relevance\n", + "an unusable window": "evals:\n - name: a\n source:\n type: traces\n" + + " agent_name: x\n lookback_hours: -1\n" + + " evaluators:\n - evaluator: builtin.relevance\n", + } + + for name, body := range cases { + t.Run(name, func(t *testing.T) { + cfg := loadFromString(t, body) + + require.NoError(t, cfg.ValidateForLookup(), + "a lookup only needs the name to be present and unique") + require.Error(t, cfg.Validate(), "deploying it is another matter") + }) + } +} + // outputDir accepts a directory or an explicit file path. func TestArtifactPath(t *testing.T) { cases := []struct { diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/trace_window.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/trace_window.go index 791e5b103e3..f243b8a3980 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/trace_window.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/trace_window.go @@ -17,6 +17,27 @@ import ( // rather than a window, and the run it produces is expensive and empty. const MaxLookbackHours = 24 * 365 * 10 +// TraceAgentName says whose conversations a trace eval reads. +// +// `agent_name` is the filter; an eval that leaves it off and names a target +// still means "this agent's traces". A model target is not an agent: it names a +// deployment, and filtering spans by a deployment name matches nothing, so the +// run comes back empty with no reason given. `target.type` is optional, and an +// untyped target is read as an agent, which is how the dataset path reads one. +// +// One definition, because the configuration check and the request builder both +// have to answer this and used to answer it separately: the config accepted a +// model target and every run of it then failed. +func TraceAgentName(source *SourceDecl, target *Target) string { + if source != nil && source.AgentName != "" { + return source.AgentName + } + if target == nil || target.Type == TargetTypeModel { + return "" + } + return target.Name +} + // ValidateSource checks a source declaration and resolves the window it names. // // One definition of what a source may say, called by the configuration check @@ -91,6 +112,10 @@ func namesOfSet(fields ...sourceField) []string { } // resolveTraceWindow reads the span of traces an eval grades. +// +// The sole enforcement point for the sign and size of a lookback, the sign of +// max_traces, a window declared twice over, and the pre-epoch rule for both a +// written bound and a derived one. A new rule about the window belongs here. func resolveTraceWindow(source *SourceDecl) (start, end time.Time, err error) { // Parsed first, so a file that is wrong in two ways names the value that // cannot be read at all rather than the pair it also got wrong. diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/trace_window_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/trace_window_test.go index a6840bb0a4d..122dec11960 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/trace_window_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/trace_window_test.go @@ -130,55 +130,55 @@ func TestValidateSource_Refuses(t *testing.T) { }{ { name: "start that is not a time", - source: SourceDecl{StartTime: "yesterday"}, + source: SourceDecl{Type: SourceTypeTraces, AgentName: "a", StartTime: "yesterday"}, wantErr: "source.start_time is \"yesterday\", which is not a time", }, { name: "end that is not a time", - source: SourceDecl{EndTime: "tomorrow"}, + source: SourceDecl{Type: SourceTypeTraces, AgentName: "a", EndTime: "tomorrow"}, wantErr: "source.end_time is \"tomorrow\", which is not a time", }, { name: "start at year one", - source: SourceDecl{StartTime: "0001-01-01T00:00:00Z"}, + source: SourceDecl{Type: SourceTypeTraces, AgentName: "a", StartTime: "0001-01-01T00:00:00Z"}, wantErr: "not a time any traces were recorded at", }, { // Parses, is not Go's zero time, and still serializes to a unix // zero that omitempty drops from the request. name: "start at the unix epoch", - source: SourceDecl{StartTime: "1970-01-01T00:00:00Z"}, + source: SourceDecl{Type: SourceTypeTraces, AgentName: "a", StartTime: "1970-01-01T00:00:00Z"}, wantErr: "not a time any traces were recorded at", }, { name: "negative lookback", - source: SourceDecl{LookbackHours: -1}, + source: SourceDecl{Type: SourceTypeTraces, AgentName: "a", LookbackHours: -1}, wantErr: "how far back to look cannot be negative", }, { name: "lookback one past the bound", - source: SourceDecl{LookbackHours: MaxLookbackHours + 1}, + source: SourceDecl{Type: SourceTypeTraces, AgentName: "a", LookbackHours: MaxLookbackHours + 1}, wantErr: "beyond the 87600 hours", }, { name: "negative cap", - source: SourceDecl{MaxTraces: -1}, + source: SourceDecl{Type: SourceTypeTraces, AgentName: "a", MaxTraces: -1}, wantErr: "source.max_traces is -1", }, { name: "window declared twice over", - source: SourceDecl{StartTime: "2026-08-01T00:00:00Z", LookbackHours: 1}, + source: SourceDecl{Type: SourceTypeTraces, AgentName: "a", StartTime: "2026-08-01T00:00:00Z", LookbackHours: 1}, wantErr: "keep one", }, { name: "end before start", - source: SourceDecl{StartTime: "2026-08-02T00:00:00Z", EndTime: "2026-08-01T00:00:00Z"}, + source: SourceDecl{Type: SourceTypeTraces, AgentName: "a", StartTime: "2026-08-02T00:00:00Z", EndTime: "2026-08-01T00:00:00Z"}, wantErr: "holds no traces", }, { // An instant is not a window, and a run over it reads nothing. name: "end equal to start", - source: SourceDecl{StartTime: "2026-08-01T00:00:00Z", EndTime: "2026-08-01T00:00:00Z"}, + source: SourceDecl{Type: SourceTypeTraces, AgentName: "a", StartTime: "2026-08-01T00:00:00Z", EndTime: "2026-08-01T00:00:00Z"}, wantErr: "holds no traces", }, } From a55de72288a6eb7802d63007c18fc2d610b5230d Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 18 Aug 2026 08:08:25 +0300 Subject: [PATCH 260/320] Give a declaration one set of rules and two reporters The last round moved the deploy gate to cover the whole of validateEval, which fixed the stranding and quietly handed five per-eval invariants to a run door that re-implemented two of them. One it implemented the wrong way, three lines from where the config door calls it wrong: a target with no name became a dataset-only run, which is a different evaluation reported as success. A negative max_samples read as "no cap" and sent the whole dataset to a run billed per row. `dataset` and `source` together stopped being refused and started being settled by which field is read first. `project.ValidateRunnable` is the whole of what an eval says about itself, in one place. Both doors call it and wrap the error with what they know: the configuration has an index to name, a run does not. validateEval keeps only what needs the rest of the file -- catalog membership, duplicate criteria, substance uniqueness. That also settles the last of the wiring from the round before: - Reading EVAL_ID for a single-eval config was a regression, not a fix. Every deploy writes that key, so nothing tells a value meant for this declaration from one left behind by the eval it replaced. A file whose one entry had been swapped ran the previous eval's criteria over the new one's rows and reported success. An eval's id comes from the entry under its own name; a miss falls through to the service listing, which answers the question the id was standing in for. - A dataset that is not in the catalog is said so at the run door too, rather than coming back as a 404 for a name nobody registered. ValidateForLookup's doc claimed it checked only the name. It also checks the catalogs, and should: a duplicate dataset name makes the lookup it exists to serve ambiguous. Said plainly now instead of overclaimed. --- .../internal/cmd/envkeys_test.go | 42 ++-------- .../internal/cmd/evalref.go | 36 +++----- .../azure.ai.evaluations/internal/cmd/run.go | 57 ++++++++++--- .../internal/cmd/run_datasource_test.go | 75 +++++++++++++++-- .../internal/messages/messages.go | 82 ++++++++++++++----- .../internal/project/eval_config.go | 68 ++++----------- .../internal/project/runnable.go | 72 ++++++++++++++++ 7 files changed, 281 insertions(+), 151 deletions(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/project/runnable.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/envkeys_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/envkeys_test.go index a1775acc63a..4db7c6da2a7 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/envkeys_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/envkeys_test.go @@ -6,9 +6,6 @@ package cmd import ( "testing" - "azureaieval/internal/pkg/evalcore" - "azureaieval/internal/project" - "github.com/stretchr/testify/assert" ) @@ -47,34 +44,13 @@ func TestIDKey_DoesNotCollideWithVersionKey(t *testing.T) { assert.NotEqual(t, idKey("dataset", "golden"), versionKey("dataset", "golden")) } -// Setting EVAL_ID by hand is the documented way to point a config at an eval -// that already exists. It is also the key the extension writes itself, which is -// what let a second eval adopt the first one's id -- so it stays readable only -// where it cannot be ambiguous. Fixing the aliasing dropped this fallback -// entirely once, silently breaking the documented behaviour, and it was later -// left in a helper that nothing called. -func TestGroupIDKeys_SharedKeyReadOnlyWhenUnambiguous(t *testing.T) { - configOf := func(names ...string) *project.EvalConfig { - cfg := &project.EvalConfig{} - for _, n := range names { - cfg.Evals = append(cfg.Evals, project.Eval{ - Name: n, - Evaluators: evalcore.EvaluatorList{{Evaluator: "builtin.relevance"}}, - }) - } - return cfg - } - - sole := evalIDKeys(configOf("quality"), "quality") - assert.Equal(t, idKey("eval", "quality"), sole[0], - "an eval's own entry is preferred over the shared one") - assert.Contains(t, sole, envKeyEvalID, - "a project with one eval honours an id set by hand") - - assert.Equal(t, []string{idKey("eval", "quality")}, - evalIDKeys(configOf("quality", "nightly"), "quality"), - "with several evals the shared entry cannot say which one it means") - - assert.Equal(t, []string{idKey("eval", "quality")}, evalIDKeys(nil, "quality"), - "an eval reached without a config has only its own entry") +// The shared EVAL_ID entry is written by every deploy, so it cannot say which +// declaration it belongs to. Reading it for a config that names a single eval +// let a file whose one entry had been replaced run the previous eval's criteria +// over the new one's rows, reported as success. An eval's id is read from the +// entry recorded under its own name and nowhere else. +func TestEvalIDIsReadFromTheEvalsOwnEntry(t *testing.T) { + assert.NotEqual(t, envKeyEvalID, idKey("eval", "quality")) + assert.NotEqual(t, idKey("eval", "quality"), idKey("eval", "nightly"), + "two declarations cannot share an entry") } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evalref.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evalref.go index b75e44bc25d..f3e09668493 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evalref.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evalref.go @@ -55,7 +55,7 @@ func (ec *evalContext) resolveEvalRef( eval, err := cfg.Eval(nameOrID) switch { case err == nil: - id := ec.recordedEvalID(ctx, cfg, eval.Name) + id := ec.recordedEvalID(ctx, eval.Name) if id == "" { // Nothing recorded is not the same as nothing published. The // id is kept in the azd environment, so a run against @@ -97,7 +97,7 @@ func (ec *evalContext) resolveEvalRef( // leaves it empty, and recordedEvalID answers "" without // asking when it is. Now that there is a name, ask properly // before reporting a deployed eval as missing. - if id = ec.recordedEvalID(ctx, cfg, eval.Name); id == "" { + if id = ec.recordedEvalID(ctx, eval.Name); id == "" { return evalRef{}, messages.EvalNotDeployedYet( eval.Name, ec.deployCommand(ctx)) } @@ -118,30 +118,16 @@ func (ec *evalContext) resolveEvalRef( } // recordedEvalID reads the id `azd up` stored for a declared eval. -func (ec *evalContext) recordedEvalID(ctx context.Context, cfg *project.EvalConfig, evalName string) string { - for _, key := range evalIDKeys(cfg, evalName) { - if id := ec.getEnvValue(ctx, key); id != "" { - return id - } - } - return "" -} - -// evalIDKeys lists the env entries that may hold this eval's id, most specific -// first. // -// The per-name entry is what the extension writes. EVAL_ID is the documented -// way to point a config at an eval that already exists, created in the portal -// or by another tool, so it stays readable -- but only when the configuration -// declares a single eval. With more than one there is no way to tell which eval -// a shared entry refers to, and reading it anyway is what let a second eval -// adopt the first one's id. -func evalIDKeys(cfg *project.EvalConfig, name string) []string { - keys := []string{idKey("eval", name)} - if cfg != nil && len(cfg.Evals) == 1 { - keys = append(keys, envKeyEvalID) - } - return keys +// Only the entry recorded under this eval's own name. EVAL_ID is written by +// every deploy as well as being settable by hand, so nothing tells a value that +// was meant for this declaration from one left behind by the eval it replaced. +// Reading it for a config that names a single eval meant a file whose one entry +// had been swapped for a different one ran the previous eval's criteria over +// the new one's rows, reported as success. A miss falls through to the service +// listing by name, which answers the question the id was standing in for. +func (ec *evalContext) recordedEvalID(ctx context.Context, evalName string) string { + return ec.getEnvValue(ctx, idKey("eval", evalName)) } // evalIDNamed finds the id of the eval the service lists under this name. diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go index 56b8726c825..353fa3bf136 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go @@ -484,12 +484,30 @@ func upgradeLegacyTraceSource(ds *eval_api.EvalRunDataSource) *eval_api.EvalRunD return eval_api.NewTracePreviewDataSource(ds.AgentName, "", start, end, maxTraces) } +// runnableEval refuses a declaration this run could not carry out. +// +// The rules live with the check the configuration runs, so the two cannot come +// to different conclusions about the same eval. Only the wrapper differs: the +// configuration has an index to name and a run does not. +func runnableEval(group *project.Eval) error { + if err := project.ValidateRunnable(group); err != nil { + return messages.InEval(group.Name, err) + } + return nil +} + // buildRunDataSource binds the eval's rows to the run. // // Three shapes, in the order the configuration decides them. A `source:` block // hands the gathering to the service and sends nothing local. Otherwise the -// rows come from a dataset, and `target:` says what to invoke for each one — +// rows come from a dataset, and `target:` says what to invoke for each one -- // including nothing at all, when the rows already hold both sides. +// +// This is where a declaration is refused, not merely where it is read. Resolving +// an eval by name does not validate what it says about itself, and a run reached +// by id has no declaration to validate, so every contradiction the configuration +// names has to be answered here as well. Settling one by evaluation order sends +// a request that succeeds and grades something the file did not ask for. func (ec *evalContext) buildRunDataSource( ctx context.Context, group *project.Eval, @@ -499,19 +517,19 @@ func (ec *evalContext) buildRunDataSource( if group == nil { return nil, messages.NoEvalToRun() } + if err := runnableEval(group); err != nil { + return nil, err + } + if group.Dataset != "" && configPath != "" && !datasetIsDeclared(configPath, group) { + return nil, messages.InEval(group.Name, messages.DatasetNotDeclared(group.Dataset)) + } if group.Source != nil { switch group.Source.Type { case project.SourceTypeTraces: return tracesDataSource(group) - case project.SourceTypeResponses: - return responsesDataSource(group) default: - // Config validation rejects this first, but a run reached by id has no - // config to have been validated. Falling through would score the wrong - // rows and say the eval declared no source. - return nil, messages.InEval(group.Name, messages.SourceTypeNotSupported( - group.Source.Type, project.SourceTypeTraces, project.SourceTypeResponses)) + return responsesDataSource(group) } } @@ -562,6 +580,8 @@ func (ec *evalContext) buildRunDataSource( // found. `agent_name` filters the traces; it is not a target, because a trace // run invokes nothing. func tracesDataSource(group *project.Eval) (*eval_api.EvalRunDataSource, error) { + // Checked by runnableEval before this is reached; read again rather than + // assumed, because an agent name is what the whole request is about. agent := project.TraceAgentName(group.Source, group.Target) if agent == "" { return nil, messages.TracesNeedAgentName(group.Name) @@ -597,11 +617,6 @@ func responsesDataSource(group *project.Eval) (*eval_api.EvalRunDataSource, erro if len(group.Source.ResponseIDs) == 0 { return nil, messages.ResponsesNeedIDs(group.Name) } - // The same check the configuration runs, so a field this source does not - // read is refused here too rather than only on the way to a deploy. - if _, _, err := project.ValidateSource(group.Source); err != nil { - return nil, messages.InEval(group.Name, err) - } return eval_api.NewResponsesDataSource(group.Source.ResponseIDs, group.Source.MaxTurns), nil } @@ -683,6 +698,22 @@ func localDatasetPath(configPath string, group *project.Eval) string { return filepath.Join(filepath.Dir(configPath), decl.Source) } +// datasetIsDeclared says whether the configuration's catalog holds the dataset +// this eval names. +// +// Without it a mistyped name falls through to a registry read and comes back as +// a 404 for a dataset nobody ever registered, which sends the reader to the +// service rather than to the line they mistyped. Answered yes when there is no +// configuration to ask: an eval reached by id has no catalog. +func datasetIsDeclared(configPath string, group *project.Eval) bool { + cfg, err := project.LoadEvalConfig(configPath) + if err != nil || cfg == nil { + return true + } + _, ok := cfg.DatasetDeclaration(group.Dataset) + return ok +} + // readJSONL reads newline-delimited JSON, optionally truncating to limit rows. func readJSONL(path string, limit int) ([]map[string]any, error) { f, err := os.Open(path) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_datasource_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_datasource_test.go index 3485bf32e87..76948cb9fb8 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_datasource_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_datasource_test.go @@ -355,24 +355,87 @@ func TestBuildRunDataSource_NoRowsFromAnywhere(t *testing.T) { // A misspelled source.type used to fall through to the dataset path, which // scored the wrong rows and then blamed the eval for declaring no source. -// Config validation catches it first, but a run reached by id has no config to -// have been validated. +// +// The run door is the first refusal for a declared eval as well as for one +// reached by id: resolving an eval by name checks only the name. func TestBuildRunDataSource_UnknownSourceTypeIsRefused(t *testing.T) { ec := &evalContext{} group := &project.Eval{ - Name: "typo", - Dataset: "d", - Source: &project.SourceDecl{Type: "trace"}, + Name: "typo", + Source: &project.SourceDecl{Type: "trace"}, } _, err := ec.buildRunDataSource(context.Background(), group, "", 0) require.Error(t, err) - assert.Contains(t, err.Error(), "trace") + assert.Contains(t, err.Error(), `source.type "trace" is not supported`) assert.NotContains(t, err.Error(), "references no dataset", "a declared source must not be reported as no source at all") } +// The run door refuses every contradiction the configuration names. Resolving +// an eval by name does not check what it says about itself, and a run reached +// by id has no declaration to check, so settling one of these by evaluation +// order sends a request that succeeds and grades something else. +func TestBuildRunDataSource_RefusesADeclarationNoRunCouldCarryOut(t *testing.T) { + ec := &evalContext{} + + cases := []struct { + name string + eval project.Eval + wantErr string + }{ + { + "rows from two places", + project.Eval{ + Dataset: "d", + Source: &project.SourceDecl{Type: project.SourceTypeTraces, AgentName: "a"}, + }, + "declare one", + }, + { + // Read as "no cap", so the whole dataset went to a run billed per + // row when the file asked for fewer rows than that. + "a negative cap", + project.Eval{Dataset: "d", MaxSamples: -1}, + "max_samples cannot be negative", + }, + { + // Scored as though nothing were invoked, which is a different + // evaluation from the one that was written down. + "a target naming nothing", + project.Eval{Dataset: "d", Target: &project.Target{Type: project.TargetTypeAgent}}, + "target.name is required", + }, + { + "a target nothing can invoke", + project.Eval{Dataset: "d", Target: &project.Target{Type: "prompt", Name: "x"}}, + "is not supported", + }, + { + "a source that does not say what it reads", + project.Eval{Source: &project.SourceDecl{}}, + "source.type is required", + }, + { + "a responses source listing nothing", + project.Eval{Source: &project.SourceDecl{Type: project.SourceTypeResponses}}, + "source.response_ids is required", + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + tc.eval.Name = "e" + _, err := ec.buildRunDataSource(context.Background(), &tc.eval, "", 0) + + require.Error(t, err) + assert.Contains(t, err.Error(), `eval "e"`) + assert.Contains(t, err.Error(), tc.wantErr) + }) + } +} + // --max-samples has to mean the same thing wherever the rows come from. func TestBuildRunDataSource_MaxSamplesCapsLocalRows(t *testing.T) { ec := &evalContext{} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go b/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go index 1c43c578973..770fa9cbd41 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go @@ -1934,21 +1934,37 @@ func EvaluatorVersionWithSource(index int, evaluator string) error { // DatasetAndSourceBothDeclared reports two answers to where rows come from. func DatasetAndSourceBothDeclared(index int, eval string) error { - return fmt.Errorf( - "evals[%d] (%s): `dataset` and `source` both say where rows come from; "+ - "declare one", index, eval) + return InEvalAt(index, eval, DatasetAndSourceDeclareTheSameThing()) +} + +// DatasetAndSourceDeclareTheSameThing reports it where there is no index. +func DatasetAndSourceDeclareTheSameThing() error { + return errors.New("`dataset` and `source` both say where rows come from; declare one") +} + +// NoEvalToValidate reports a declaration that is not there at all. +func NoEvalToValidate() error { + return errors.New("no eval declaration to check") } // DatasetNotInDatasetsCatalog reports an eval naming a dataset nobody declared. func DatasetNotInDatasetsCatalog(index int, eval, dataset string) error { - return fmt.Errorf( - "evals[%d] (%s): dataset %q is not in the datasets catalog", - index, eval, dataset) + return InEvalAt(index, eval, DatasetNotDeclared(dataset)) +} + +// DatasetNotDeclared reports it where there is no index. +func DatasetNotDeclared(dataset string) error { + return fmt.Errorf("dataset %q is not in the datasets catalog", dataset) } // SourceTypeRequired reports a source: block that does not say what it reads. func SourceTypeRequired(index int, eval string) error { - return fmt.Errorf("evals[%d] (%s): source.type is required", index, eval) + return InEvalAt(index, eval, SourceTypeMissing()) +} + +// SourceTypeMissing reports it where there is no index. +func SourceTypeMissing() error { + return errors.New("source.type is required") } // SourceTypeUnsupported reports a source.type the extension has no path for. @@ -1964,17 +1980,28 @@ func SourceTypeNotSupported(got, traces, responses string) error { // TracesSourceNeedsAgentName reports a trace source that does not say whose // conversations to read. func TracesSourceNeedsAgentName(index int, eval string) error { - return fmt.Errorf( - "evals[%d] (%s): source.agent_name is required for a trace source, "+ - "or declare target.name", index, eval) + return InEvalAt(index, eval, TraceSourceNeedsAnAgent()) +} + +// TraceSourceNeedsAnAgent reports it where there is no index. +// +// A target names one too, unless it names a model: a deployment name matches +// no spans, so it is not an answer to whose conversations to read. +func TraceSourceNeedsAnAgent() error { + return errors.New( + "source.agent_name is required for a trace source, " + + "or declare an agent target.name") } // ResponsesSourceNeedsIDs reports a stored-response source with nothing to // retrieve. func ResponsesSourceNeedsIDs(index int, eval string) error { - return fmt.Errorf( - "evals[%d] (%s): source.response_ids is required for a responses source", - index, eval) + return InEvalAt(index, eval, ResponsesSourceNeedsResponseIDs()) +} + +// ResponsesSourceNeedsResponseIDs reports it where there is no index. +func ResponsesSourceNeedsResponseIDs() error { + return errors.New("source.response_ids is required for a responses source") } // AtLeastOneEvaluatorRequired reports an eval that scores nothing. @@ -2003,9 +2030,12 @@ func EvaluatorNotInCatalog(evalIndex, refIndex int, evaluator string) error { // TargetTypeUnsupported reports a target.type the extension cannot invoke. func TargetTypeUnsupported(index int, eval, got, agent, model string) error { - return fmt.Errorf( - "evals[%d] (%s): target.type %q is not supported; use %q or %q", - index, eval, got, agent, model) + return InEvalAt(index, eval, TargetTypeNotSupported(got, agent, model)) +} + +// TargetTypeNotSupported reports it where there is no index. +func TargetTypeNotSupported(got, agent, model string) error { + return fmt.Errorf("target.type %q is not supported; use %q or %q", got, agent, model) } // EvaluationLevelInvalid reports a scoring granularity the service does not accept. @@ -2017,9 +2047,13 @@ func EvaluationLevelInvalid(index int, eval, got, turn, conversation string) err // TargetNameRequired reports a declared target that names nothing to invoke. func TargetNameRequired(index int, eval string) error { - return fmt.Errorf( - "evals[%d] (%s): target.name is required; remove the target: to score the "+ - "dataset as it stands", index, eval) + return InEvalAt(index, eval, TargetNameMissing()) +} + +// TargetNameMissing reports it where there is no index. +func TargetNameMissing() error { + return errors.New( + "target.name is required; remove the target: to score the dataset as it stands") } // AmbiguousEvalConfig reports a directory holding both configuration names. @@ -2148,10 +2182,14 @@ func SampleSizeOutOfRange(min, max, got int) error { // dataset to a run that is billed per row -- the opposite of what a cap asks // for, and silent. func NegativeMaxSamples(index int, name string, got int) error { + return InEvalAt(index, name, MaxSamplesNegative(got)) +} + +// MaxSamplesNegative reports it where there is no index. +func MaxSamplesNegative(got int) error { return fmt.Errorf( - "evals[%d] %q: max_samples cannot be negative, got %d. "+ - "Remove it to send every row, or set the number of rows to send", - index, name, got) + "max_samples cannot be negative, got %d. "+ + "Remove it to send every row, or set the number of rows to send", got) } // NegativeMaxSamplesFlag reports the same thing given on the command line. diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go index 9cc709dd7a1..85654f331c1 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go @@ -219,14 +219,19 @@ func (c *EvalConfig) Validate() error { return c.validate(true) } -// ValidateForLookup checks only what resolving a declaration by name depends -// on. +// ValidateForLookup checks what resolving a declaration by name depends on: a +// readable set of catalogs, and a name that is present and not shared. // -// Two evals that differ only in substance still resolve unambiguously by name, -// and that clash only matters to something about to deploy. Enforcing it on the -// way to a lookup stranded commands that had already been told which eval they -// meant -- `run list --eval ` refused to list anything, and the way out -// was to hand-edit the config, which the error did not say. +// What an eval says about itself is left to deploying it, and to the run door, +// which applies the same rules to the entry the run is actually about. Checking +// it here stranded commands that had already been told which eval they meant -- +// `run list --eval ` refused to list anything because a different entry +// was malformed, and the way out was to hand-edit a file the error did not +// mention. +// +// The catalogs stay, because they are the file's shared half: a duplicate +// dataset name makes the lookup this method exists to serve ambiguous, and no +// declaration can be read against a catalog that does not parse into one. func (c *EvalConfig) ValidateForLookup() error { return c.validate(false) } @@ -331,45 +336,16 @@ func validateSource(i int, name string, source *SourceDecl) error { } func (c *EvalConfig) validateEval(i int, eval Eval) error { - if eval.Dataset != "" && eval.Source != nil { - return messages.DatasetAndSourceBothDeclared(i, eval.Name) - } - // A negative cap read as "no cap", so the whole dataset went to a billed - // run when the config asked for fewer rows than that. - if eval.MaxSamples < 0 { - return messages.NegativeMaxSamples(i, eval.Name, eval.MaxSamples) + // Everything an eval says about itself, from the definition the run door + // also uses. What follows is only what needs the rest of the file. + if err := ValidateRunnable(&eval); err != nil { + return messages.InEvalAt(i, eval.Name, err) } if eval.Dataset != "" { if _, ok := c.DatasetDeclaration(eval.Dataset); !ok { return messages.DatasetNotInDatasetsCatalog(i, eval.Name, eval.Dataset) } } - if eval.Source != nil { - switch eval.Source.Type { - case SourceTypeTraces: - // The run needs one of these to say whose traces to read. Refusing - // here rather than at run time keeps a config that cannot run from - // deploying, so it has to be the same question the run asks. - if TraceAgentName(eval.Source, eval.Target) == "" { - return messages.TracesSourceNeedsAgentName(i, eval.Name) - } - if err := validateSource(i, eval.Name, eval.Source); err != nil { - return err - } - case SourceTypeResponses: - if len(eval.Source.ResponseIDs) == 0 { - return messages.ResponsesSourceNeedsIDs(i, eval.Name) - } - if err := validateSource(i, eval.Name, eval.Source); err != nil { - return err - } - case "": - return messages.SourceTypeRequired(i, eval.Name) - default: - return messages.SourceTypeUnsupported( - i, eval.Name, eval.Source.Type, SourceTypeTraces, SourceTypeResponses) - } - } if len(eval.Evaluators) == 0 { return messages.AtLeastOneEvaluatorRequired(i, eval.Name) @@ -395,18 +371,6 @@ func (c *EvalConfig) validateEval(i int, eval Eval) error { } } - if eval.Target != nil { - if eval.Target.Type != "" && - eval.Target.Type != TargetTypeAgent && eval.Target.Type != TargetTypeModel { - return messages.TargetTypeUnsupported( - i, eval.Name, eval.Target.Type, TargetTypeAgent, TargetTypeModel) - } - // A target with no name is scored as though nothing were invoked, which - // is a different evaluation from the one that was written down. - if eval.Target.Name == "" { - return messages.TargetNameRequired(i, eval.Name) - } - } switch eval.EvaluationLevel { case "", EvaluationLevelTurn, EvaluationLevelConversation: default: diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/runnable.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/runnable.go new file mode 100644 index 00000000000..3d7fa83b6bc --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/runnable.go @@ -0,0 +1,72 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package project + +import "azureaieval/internal/messages" + +// ValidateRunnable refuses a declaration no run could carry out. +// +// One definition of what an eval has to say about itself, called on the way to +// deploying it and again when a run is built. Resolving an eval by name does +// not check any of this -- a lookup depends only on the name -- and a run +// reached by id has no declaration to check, so the run door is the first +// refusal as often as the config door is. Two hand-written copies drifted apart +// every time, on whichever axis was not tested at both ends. +// +// The errors carry no prefix. The caller says whether it has an index to name. +// +// What is not here needs the rest of the file to decide: whether a dataset or +// an evaluator is in its catalog, and whether two evals are the same in +// substance. +func ValidateRunnable(eval *Eval) error { + if eval == nil { + return messages.NoEvalToValidate() + } + // Two answers to where rows come from, and the file does not say which was + // meant. Refused rather than ranked: settling it by which field is read + // first sends a request that succeeds and grades the other one. + if eval.Dataset != "" && eval.Source != nil { + return messages.DatasetAndSourceDeclareTheSameThing() + } + if eval.MaxSamples < 0 { + return messages.MaxSamplesNegative(eval.MaxSamples) + } + + if eval.Source != nil { + switch eval.Source.Type { + case SourceTypeTraces: + if TraceAgentName(eval.Source, eval.Target) == "" { + return messages.TraceSourceNeedsAnAgent() + } + case SourceTypeResponses: + if len(eval.Source.ResponseIDs) == 0 { + return messages.ResponsesSourceNeedsResponseIDs() + } + case "": + return messages.SourceTypeMissing() + default: + return messages.SourceTypeNotSupported( + eval.Source.Type, SourceTypeTraces, SourceTypeResponses) + } + if _, _, err := ValidateSource(eval.Source); err != nil { + return err + } + } + + if eval.Target != nil { + // The type is checked first because it is the thing that was written: + // a target with an unsupported type and no name should be told about + // the type rather than sent to add a name it cannot use. + if eval.Target.Type != "" && + eval.Target.Type != TargetTypeAgent && eval.Target.Type != TargetTypeModel { + return messages.TargetTypeNotSupported(eval.Target.Type, TargetTypeAgent, TargetTypeModel) + } + // A target with no name is scored as though nothing were invoked, + // which is a different evaluation from the one that was written down. + if eval.Target.Name == "" { + return messages.TargetNameMissing() + } + } + return nil +} From 16eed6fdb2c7ff4325a28364447f2d78df8a5005 Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 18 Aug 2026 08:30:44 +0300 Subject: [PATCH 261/320] Delete the copies the shared rule replaced The consolidation left its own leftovers. `validateSource` had no callers and a doc comment describing the two-door arrangement it had just been collapsed into -- an unexported unreferenced function, which the module's linter refuses. Eight indexed message constructors had no callers either, and the messages package exists so the whole voice of the CLI can be read in one sitting; half the config-validation section was voice nothing spoke. Two run-door branches kept second wordings of invariants the shared rule now owns, unreachable and ready to drift again. Removing the EVAL_ID fallback from `recordedEvalID` last round only removed half of it. `resolveEvalID` still read that key, so `run start` and `run show`/`list`/`cancel`/`delete`/`output` resolved different evals for the same bare invocation whenever the per-name entry was absent -- which is the swapped-declaration case the removal was for. `run cancel` with no arguments would then cancel a run of the eval that had been replaced: a destructive verb on the wrong resource, chosen silently. Both doors go through the declaration now. `evaluation_level` was the last self-contained per-eval rule left at one door. It reads only the eval and two constants, and the run door sends it as run metadata without checking it, where anything that is not "conversation" is read as turn-shaped -- so an unrecognised value graded the run at a granularity the file did not ask for. It is in the shared rule. Two ordering fixes in that rule: - The target is checked before the source, because the trace rule reads the target. An eval with an unusable target was told to name an agent on it, then told on the next run that the target was a kind nothing can invoke. - A trace eval pointed at a model deployment gets its own sentence. The general advice, "or declare an agent target.name", read as an invitation to relabel the deployment, which produces a filter matching no spans and a run that reports nothing. A configuration with catalogs and no evals is what `generate` leaves behind until `init` wires one. Refusing it on the way to a lookup stranded `run --eval ` over the absence of a declaration the id did not need. Tests: the shared rule had no test of its own -- its shared-ness was asserted only by two other tables happening to pass -- and the replacement for the deleted EVAL_ID test asserted that two key names differ, which was true before the change and would stay true if it were reverted. --- .../internal/cmd/reconciler.go | 6 +- .../azure.ai.evaluations/internal/cmd/run.go | 8 +- .../internal/cmd/run_output.go | 22 ++- .../internal/messages/messages.go | 53 ++---- .../internal/project/eval_config.go | 27 +-- .../internal/project/runnable.go | 52 ++++-- .../internal/project/runnable_test.go | 154 ++++++++++++++++++ 7 files changed, 227 insertions(+), 95 deletions(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/project/runnable_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go index 77ce20628f9..630289cc78e 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go @@ -530,8 +530,10 @@ func (r *evalReconciler) EnsureEval( r.ec.remember(ctx, key, digest) r.ec.remember(ctx, idKey("eval", group.Name), created.ID) r.ec.remember(ctx, digestIDKey(digest), created.ID) - // EVAL_ID stays the last-deployed eval, which is what the commands - // fall back to when a config names only one. + // EVAL_ID stays the last-deployed eval. Nothing reads it to decide which + // eval a command means, because every deploy writes it and it cannot say + // which declaration it belongs to; it is here for anything outside this + // extension that wants the id of what was just deployed. r.ec.remember(ctx, envKeyEvalID, created.ID) return created.ID, true, nil } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go index 353fa3bf136..fd82d9f422b 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go @@ -580,11 +580,11 @@ func (ec *evalContext) buildRunDataSource( // found. `agent_name` filters the traces; it is not a target, because a trace // run invokes nothing. func tracesDataSource(group *project.Eval) (*eval_api.EvalRunDataSource, error) { - // Checked by runnableEval before this is reached; read again rather than - // assumed, because an agent name is what the whole request is about. + // runnableEval has already refused an empty one; read rather than assumed, + // because the agent name is what the whole request is about. agent := project.TraceAgentName(group.Source, group.Target) if agent == "" { - return nil, messages.TracesNeedAgentName(group.Name) + return nil, messages.InEval(group.Name, messages.TraceSourceNeedsAnAgent()) } start, end, err := traceWindow(group.Name, group.Source) @@ -615,7 +615,7 @@ func traceWindow(evalName string, source *project.SourceDecl) (start, end time.T // responsesDataSource evaluates responses the project already stored. func responsesDataSource(group *project.Eval) (*eval_api.EvalRunDataSource, error) { if len(group.Source.ResponseIDs) == 0 { - return nil, messages.ResponsesNeedIDs(group.Name) + return nil, messages.InEval(group.Name, messages.ResponsesSourceNeedsResponseIDs()) } return eval_api.NewResponsesDataSource(group.Source.ResponseIDs, group.Source.MaxTurns), nil } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output.go index ae1a961c8b7..4a057628ee1 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output.go @@ -292,19 +292,17 @@ func resolveEvalID( return args[0], nil } - if groupName != "" { - ref, err := ec.resolveEvalRef( - cmd.Context(), ec.evalDir(cmd.Context(), evalPathFlag(cmd)), groupName) - if err != nil { - return "", err - } - return ref.ID, nil - } - - if cached := ec.getEnvValue(cmd.Context(), envKeyEvalID); cached != "" { - return cached, nil + // With no name given, the eval is the one the configuration declares, + // which is how `run start` decides it. Reading EVAL_ID here instead made + // the two disagree: every deploy writes that key, so a file whose one entry + // had been replaced sent `run cancel` at the eval it used to be -- a + // destructive verb on the wrong resource, chosen silently. + ref, err := ec.resolveEvalRef( + cmd.Context(), ec.evalDir(cmd.Context(), evalPathFlag(cmd)), groupName) + if err != nil { + return "", err } - return "", messages.NoEvalGiven() + return ref.ID, nil } // addEvalFlag registers the flag that says which eval a command acts on. It diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go b/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go index 770fa9cbd41..a2a4f483618 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go @@ -1932,11 +1932,6 @@ func EvaluatorVersionWithSource(index int, evaluator string) error { "on the project", index, evaluator) } -// DatasetAndSourceBothDeclared reports two answers to where rows come from. -func DatasetAndSourceBothDeclared(index int, eval string) error { - return InEvalAt(index, eval, DatasetAndSourceDeclareTheSameThing()) -} - // DatasetAndSourceDeclareTheSameThing reports it where there is no index. func DatasetAndSourceDeclareTheSameThing() error { return errors.New("`dataset` and `source` both say where rows come from; declare one") @@ -1957,32 +1952,16 @@ func DatasetNotDeclared(dataset string) error { return fmt.Errorf("dataset %q is not in the datasets catalog", dataset) } -// SourceTypeRequired reports a source: block that does not say what it reads. -func SourceTypeRequired(index int, eval string) error { - return InEvalAt(index, eval, SourceTypeMissing()) -} - // SourceTypeMissing reports it where there is no index. func SourceTypeMissing() error { return errors.New("source.type is required") } -// SourceTypeUnsupported reports a source.type the extension has no path for. -func SourceTypeUnsupported(index int, eval, got, traces, responses string) error { - return InEvalAt(index, eval, SourceTypeNotSupported(got, traces, responses)) -} - // SourceTypeNotSupported reports the same, where there is no index to name. func SourceTypeNotSupported(got, traces, responses string) error { return fmt.Errorf("source.type %q is not supported; use %q or %q", got, traces, responses) } -// TracesSourceNeedsAgentName reports a trace source that does not say whose -// conversations to read. -func TracesSourceNeedsAgentName(index int, eval string) error { - return InEvalAt(index, eval, TraceSourceNeedsAnAgent()) -} - // TraceSourceNeedsAnAgent reports it where there is no index. // // A target names one too, unless it names a model: a deployment name matches @@ -1993,12 +1972,6 @@ func TraceSourceNeedsAnAgent() error { "or declare an agent target.name") } -// ResponsesSourceNeedsIDs reports a stored-response source with nothing to -// retrieve. -func ResponsesSourceNeedsIDs(index int, eval string) error { - return InEvalAt(index, eval, ResponsesSourceNeedsResponseIDs()) -} - // ResponsesSourceNeedsResponseIDs reports it where there is no index. func ResponsesSourceNeedsResponseIDs() error { return errors.New("source.response_ids is required for a responses source") @@ -2028,26 +2001,26 @@ func EvaluatorNotInCatalog(evalIndex, refIndex int, evaluator string) error { evalIndex, refIndex, evaluator) } -// TargetTypeUnsupported reports a target.type the extension cannot invoke. -func TargetTypeUnsupported(index int, eval, got, agent, model string) error { - return InEvalAt(index, eval, TargetTypeNotSupported(got, agent, model)) -} - // TargetTypeNotSupported reports it where there is no index. func TargetTypeNotSupported(got, agent, model string) error { return fmt.Errorf("target.type %q is not supported; use %q or %q", got, agent, model) } -// EvaluationLevelInvalid reports a scoring granularity the service does not accept. -func EvaluationLevelInvalid(index int, eval, got, turn, conversation string) error { - return fmt.Errorf( - "evals[%d] (%s): evaluation_level %q is invalid; expected %q or %q", - index, eval, got, turn, conversation) +// EvaluationLevelNotSupported reports it where there is no index. +func EvaluationLevelNotSupported(got, turn, conversation string) error { + return fmt.Errorf("evaluation_level %q is invalid; expected %q or %q", got, turn, conversation) } -// TargetNameRequired reports a declared target that names nothing to invoke. -func TargetNameRequired(index int, eval string) error { - return InEvalAt(index, eval, TargetNameMissing()) +// TraceSourceCannotReadAModelTarget reports a trace eval pointed at a deployment. +// +// Its own sentence, because the general advice is "declare an agent target", +// which here reads as an invitation to relabel the deployment -- producing a +// filter that matches no spans and a run that reports nothing. +func TraceSourceCannotReadAModelTarget(name string) error { + return fmt.Errorf( + "source.agent_name is required for a trace source: target %q is a model "+ + "deployment, and traces are recorded against an agent, not a deployment", + name) } // TargetNameMissing reports it where there is no index. diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go index 85654f331c1..643f84c6ceb 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go @@ -240,7 +240,12 @@ func (c *EvalConfig) validate(deploying bool) error { if err := c.validateCatalogs(); err != nil { return err } - if len(c.Evals) == 0 { + // A catalog with no eval is what `generate` leaves behind, and it stays + // that way until `init` wires one. Refusing it on the way to a lookup + // stranded `run --eval ` in a project where `generate` ran first, over + // the absence of a declaration the id did not need. `Eval` answers for the + // case that does need one. + if deploying && len(c.Evals) == 0 { return messages.AtLeastOneEvalRequired() } @@ -323,21 +328,7 @@ func (c *EvalConfig) validateCatalogs() error { return nil } -// validateSource refuses a source declaration a run could not carry out. -// -// The rules live with the resolver the run also uses, so the two cannot come to -// different conclusions about the same file. Only the wrapper differs: here -// there is an index to name, and at run time there is not. -func validateSource(i int, name string, source *SourceDecl) error { - if _, _, err := ValidateSource(source); err != nil { - return messages.InEvalAt(i, name, err) - } - return nil -} - func (c *EvalConfig) validateEval(i int, eval Eval) error { - // Everything an eval says about itself, from the definition the run door - // also uses. What follows is only what needs the rest of the file. if err := ValidateRunnable(&eval); err != nil { return messages.InEvalAt(i, eval.Name, err) } @@ -371,11 +362,5 @@ func (c *EvalConfig) validateEval(i int, eval Eval) error { } } - switch eval.EvaluationLevel { - case "", EvaluationLevelTurn, EvaluationLevelConversation: - default: - return messages.EvaluationLevelInvalid( - i, eval.Name, eval.EvaluationLevel, EvaluationLevelTurn, EvaluationLevelConversation) - } return nil } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/runnable.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/runnable.go index 3d7fa83b6bc..95dda12f269 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/runnable.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/runnable.go @@ -16,9 +16,11 @@ import "azureaieval/internal/messages" // // The errors carry no prefix. The caller says whether it has an index to name. // -// What is not here needs the rest of the file to decide: whether a dataset or -// an evaluator is in its catalog, and whether two evals are the same in -// substance. +// What is not here is what the rest of the file, or the service, has to decide: +// whether a dataset or an evaluator is in its catalog, whether two evals are +// the same in substance, and what a published evaluator requires. The evaluator +// checks stay at the create door because they constrain what is created rather +// than what a run sends. func ValidateRunnable(eval *Eval) error { if eval == nil { return messages.NoEvalToValidate() @@ -33,10 +35,33 @@ func ValidateRunnable(eval *Eval) error { return messages.MaxSamplesNegative(eval.MaxSamples) } + // The target is checked before the source, because the trace rule below + // reads the target: without this, an eval with an unusable target is told + // to name an agent on it, and told on the next run that the target it was + // sent to name is a kind nothing can invoke. + if eval.Target != nil { + if eval.Target.Type != "" && + eval.Target.Type != TargetTypeAgent && eval.Target.Type != TargetTypeModel { + return messages.TargetTypeNotSupported(eval.Target.Type, TargetTypeAgent, TargetTypeModel) + } + // A target with no name is scored as though nothing were invoked, + // which is a different evaluation from the one that was written down. + if eval.Target.Name == "" { + return messages.TargetNameMissing() + } + } + if eval.Source != nil { switch eval.Source.Type { case SourceTypeTraces: if TraceAgentName(eval.Source, eval.Target) == "" { + // A model target is the one case where a target is present and + // still no answer. Saying "or declare an agent target.name" + // there reads as an invitation to relabel the deployment, which + // produces a filter that matches no spans and reports nothing. + if eval.Target != nil && eval.Target.Type == TargetTypeModel { + return messages.TraceSourceCannotReadAModelTarget(eval.Target.Name) + } return messages.TraceSourceNeedsAnAgent() } case SourceTypeResponses: @@ -54,19 +79,14 @@ func ValidateRunnable(eval *Eval) error { } } - if eval.Target != nil { - // The type is checked first because it is the thing that was written: - // a target with an unsupported type and no name should be told about - // the type rather than sent to add a name it cannot use. - if eval.Target.Type != "" && - eval.Target.Type != TargetTypeAgent && eval.Target.Type != TargetTypeModel { - return messages.TargetTypeNotSupported(eval.Target.Type, TargetTypeAgent, TargetTypeModel) - } - // A target with no name is scored as though nothing were invoked, - // which is a different evaluation from the one that was written down. - if eval.Target.Name == "" { - return messages.TargetNameMissing() - } + switch eval.EvaluationLevel { + case "", EvaluationLevelTurn, EvaluationLevelConversation: + default: + // Sent as run metadata, and anything that is not "conversation" is + // read as turn-shaped, so a value nobody recognises grades the run at + // a granularity the file did not ask for. + return messages.EvaluationLevelNotSupported( + eval.EvaluationLevel, EvaluationLevelTurn, EvaluationLevelConversation) } return nil } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/runnable_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/runnable_test.go new file mode 100644 index 00000000000..2b91e0e02f0 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/runnable_test.go @@ -0,0 +1,154 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package project + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// One definition, two reporters. The configuration and the request builder both +// have to decide whether a declaration can run, and every round they decided it +// separately they drifted: one accepted what the other refused, and which rules +// applied depended on which door the eval came through. +// +// The errors carry no prefix of their own -- the caller adds what it knows -- +// so the rule can be stated once and reported from either side. +func TestValidateRunnable_RefusesWhatNoRunCouldCarryOut(t *testing.T) { + dataset := func(e Eval) Eval { e.Name, e.Dataset = "e", "d"; return e } + + cases := []struct { + name string + eval Eval + wantErr string + }{ + { + "rows from two places", + Eval{Dataset: "d", Source: &SourceDecl{Type: SourceTypeTraces, AgentName: "a"}}, + "declare one", + }, + {"a negative cap", dataset(Eval{MaxSamples: -1}), "max_samples cannot be negative"}, + { + "a target naming nothing", + dataset(Eval{Target: &Target{Type: TargetTypeAgent}}), + "target.name is required", + }, + { + "a target nothing can invoke", + dataset(Eval{Target: &Target{Type: "prompt", Name: "x"}}), + `target.type "prompt" is not supported`, + }, + { + "a source that does not say what it reads", + Eval{Name: "e", Source: &SourceDecl{}}, + "source.type is required", + }, + { + "a source nothing can read", + Eval{Name: "e", Source: &SourceDecl{Type: "trace"}}, + `source.type "trace" is not supported`, + }, + { + "a trace source naming no agent", + Eval{Name: "e", Source: &SourceDecl{Type: SourceTypeTraces}}, + "source.agent_name is required", + }, + { + "a responses source listing nothing", + Eval{Name: "e", Source: &SourceDecl{Type: SourceTypeResponses}}, + "source.response_ids is required", + }, + { + "a window the source cannot use", + Eval{Name: "e", Source: &SourceDecl{ + Type: SourceTypeTraces, AgentName: "a", LookbackHours: -1, + }}, + "cannot be negative", + }, + { + // Sent as run metadata, and anything that is not "conversation" is + // read as turn-shaped, so an unrecognised value grades the run at a + // granularity the file did not ask for. + "a granularity nothing scores at", + dataset(Eval{EvaluationLevel: "sentence"}), + `evaluation_level "sentence" is invalid`, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + err := ValidateRunnable(&tc.eval) + + require.Error(t, err) + assert.Contains(t, err.Error(), tc.wantErr) + // The caller says where the error came from. A prefix here would + // be repeated by one door and wrong at the other. + assert.NotContains(t, err.Error(), "evals[") + assert.NotContains(t, err.Error(), `eval "`) + }) + } +} + +// A trace eval pointed at a model deployment is the one case where a target is +// present and still answers nothing. The general advice reads as an invitation +// to relabel the deployment as an agent, which produces a filter matching no +// spans and a run that reports nothing. +func TestValidateRunnable_SaysWhyAModelTargetIsNotAnAgent(t *testing.T) { + err := ValidateRunnable(&Eval{ + Name: "e", + Source: &SourceDecl{Type: SourceTypeTraces}, + Target: &Target{Type: TargetTypeModel, Name: "gpt-4o-mini"}, + }) + + require.Error(t, err) + assert.Contains(t, err.Error(), "is a model deployment") + assert.NotContains(t, err.Error(), "declare an agent target.name", + "advice that leaves the eval wrong in a way nothing reports") +} + +// An eval wrong in two ways is told about the one that cannot be worked around, +// so following the advice does not lead straight back here. +func TestValidateRunnable_ReportsTheUnusableTargetBeforeTheRuleThatReadsIt(t *testing.T) { + err := ValidateRunnable(&Eval{ + Name: "e", + Source: &SourceDecl{Type: SourceTypeTraces}, + Target: &Target{Type: "prompt"}, + }) + + require.Error(t, err) + assert.Contains(t, err.Error(), "is not supported") + assert.NotContains(t, err.Error(), "agent_name", + "naming an agent on a target nothing can invoke fixes nothing") +} + +// A declaration that says nothing contradictory passes, whichever shape it is. +func TestValidateRunnable_Accepts(t *testing.T) { + for name, eval := range map[string]Eval{ + "a dataset scored as it stands": {Name: "e", Dataset: "d"}, + "a dataset with an agent target": { + Name: "e", Dataset: "d", Target: &Target{Type: TargetTypeAgent, Name: "a"}, + }, + "a dataset with an untyped target": { + Name: "e", Dataset: "d", Target: &Target{Name: "a"}, + }, + "traces filtered by name": { + Name: "e", Source: &SourceDecl{Type: SourceTypeTraces, AgentName: "a"}, + }, + "traces named by the target": { + Name: "e", + Source: &SourceDecl{Type: SourceTypeTraces}, + Target: &Target{Type: TargetTypeAgent, Name: "a"}, + }, + "stored responses": { + Name: "e", + Source: &SourceDecl{Type: SourceTypeResponses, ResponseIDs: []string{"resp_1"}}, + }, + } { + t.Run(name, func(t *testing.T) { + require.NoError(t, ValidateRunnable(&eval)) + }) + } +} From a1929ae4b692ffd88ca82cfa826ec6e12e64316f Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 18 Aug 2026 08:33:15 +0300 Subject: [PATCH 262/320] Spell it the way the dictionary does cspell reads US English, and "recognises" is not in it. Reworded rather than added to the word list: the sentence reads the same either way, and the list is for terms the project actually needs. --- .../azure.ai.evaluations/internal/project/runnable.go | 2 +- .../azure.ai.evaluations/internal/project/runnable_test.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/runnable.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/runnable.go index 95dda12f269..5d63072957d 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/runnable.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/runnable.go @@ -83,7 +83,7 @@ func ValidateRunnable(eval *Eval) error { case "", EvaluationLevelTurn, EvaluationLevelConversation: default: // Sent as run metadata, and anything that is not "conversation" is - // read as turn-shaped, so a value nobody recognises grades the run at + // read as turn-shaped, so a value nothing knows about grades the run at // a granularity the file did not ask for. return messages.EvaluationLevelNotSupported( eval.EvaluationLevel, EvaluationLevelTurn, EvaluationLevelConversation) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/runnable_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/runnable_test.go index 2b91e0e02f0..8003771f91f 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/runnable_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/runnable_test.go @@ -70,8 +70,8 @@ func TestValidateRunnable_RefusesWhatNoRunCouldCarryOut(t *testing.T) { }, { // Sent as run metadata, and anything that is not "conversation" is - // read as turn-shaped, so an unrecognised value grades the run at a - // granularity the file did not ask for. + // read as turn-shaped, so a value nothing knows about grades the run + // at a granularity the file did not ask for. "a granularity nothing scores at", dataset(Eval{EvaluationLevel: "sentence"}), `evaluation_level "sentence" is invalid`, From 1d29e535f3f5e0612a09fef2128a4b1b93ec99ae Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 18 Aug 2026 08:40:57 +0300 Subject: [PATCH 263/320] Make an unspoken message fail the build Consolidating two ways of saying one thing keeps leaving the losing wording behind: still compiling, still reviewed, ready to be picked up by someone who finds it and assumes it is live. `unused` cannot catch them, because they are exported. Three rounds of review found them by reading, one at a time. A test now walks the module and asserts every exported constructor in the messages package is referenced from outside it. It found eight on its first run, six of which predate this work: - `ParsingServiceConfig`, `ReadingJob`, `EvalIs`, `NothingGenerated` and the indexed `NegativeMaxSamples` had no caller anywhere. Deleted. - `CreatedHeading`, `CreatedConfigLine` and `NoEvalConfig` are reached only through a live constructor in the same file, so they were never part of the CLI's voice. Unexported, which puts them back under `unused`. - `TracesNeedAgentName`, `ResponsesNeedIDs` and `NoEvalGiven` were left behind by this branch's own consolidation. Verified by planting an orphan and watching the test fail, then removing it. The rule is blunt on purpose: referenced outside the package, or deleted. Anything written for a caller that does not exist yet should arrive with it. --- .../internal/messages/messages.go | 66 ++------- .../internal/messages/spoken_test.go | 128 ++++++++++++++++++ 2 files changed, 138 insertions(+), 56 deletions(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/messages/spoken_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go b/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go index a2a4f483618..6dd05d9071f 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go @@ -61,22 +61,6 @@ func DatasetFileEmpty(path string) error { return fmt.Errorf("dataset file %q has no rows", filepath.ToSlash(path)) } -// TracesNeedAgentName reports a trace-backed eval that does not say whose -// traces to read. -func TracesNeedAgentName(eval string) error { - return fmt.Errorf( - "eval %q reads traces but does not say whose. Set source.agent_name to the "+ - "agent whose conversations should be evaluated", - eval) -} - -// ResponsesNeedIDs reports a stored-response eval with nothing to retrieve. -func ResponsesNeedIDs(eval string) error { - return fmt.Errorf( - "eval %q evaluates stored responses but lists none. Set source.response_ids", - eval) -} - // DatasetOverrideNeedsDeclaredEval reports --dataset passed against a bare id. func DatasetOverrideNeedsDeclaredEval() error { return errors.New( @@ -279,11 +263,6 @@ func NoEvalNamedOrDeclared(configPath string) error { filepath.ToSlash(configPath)) } -// NoEvalGiven reports a command run outside a project with no eval id. -func NoEvalGiven() error { - return errors.New("no eval given; pass its id as an argument, or name one with --eval") -} - // ListingRuns reports a failure to list an eval's runs. func ListingRuns(evalID string, err error) error { return fmt.Errorf("listing runs for %q: %w", evalID, err) @@ -766,11 +745,6 @@ func ArtifactExists(path string) error { path) } -// NothingGenerated reports a generation that produced no artifact. -func NothingGenerated() string { - return "Nothing was generated.\n" -} - // JobKindRequired reports a job command that does not say which collection. func JobKindRequired() error { return errors.New("pass --dataset or --evaluator to say which generation jobs to act on") @@ -816,11 +790,6 @@ func JobNotFound(kind, jobID, other string) error { kind, jobID, other) } -// ReadingJob reports a failure to read a generation job. -func ReadingJob(kind, jobID string, err error) error { - return fmt.Errorf("reading %s generation job %s: %w", kind, jobID, err) -} - // JobActionFailed reports a job operation that was not a read, so the sentence // names what was attempted. A delete that reports "reading" sends the reader // looking for a read that never happened. @@ -1345,11 +1314,6 @@ func ReadingServiceConfig(err error) error { return fmt.Errorf("reading the eval service configuration: %w", err) } -// ParsingServiceConfig reports the service entry failing to parse. -func ParsingServiceConfig(err error) error { - return fmt.Errorf("parsing the eval service configuration: %w", err) -} - // ReconcilingDataset reports the dataset a deploy has reached. func ReconcilingDataset(dataset string) string { return fmt.Sprintf("Reconciling dataset %s", dataset) @@ -1381,11 +1345,6 @@ func EvalProblem(eval string, err error) error { return fmt.Errorf("eval %q: %w", eval, err) } -// EvalIs reports the id a deployed eval resolved to. -func EvalIs(eval, id string) string { - return fmt.Sprintf("Eval %s is %s", eval, id) -} - // EvalCreatedProgress and EvalUnchangedProgress are the deploy-time equivalents // of EvalCreated and EvalUnchanged, without the status marks azd adds itself. func EvalCreatedProgress(eval, id string) string { @@ -1778,8 +1737,8 @@ func GradingWith(evaluators []string) string { return fmt.Sprintf("%s Grading with: %s\n", DoneMark, strings.Join(evaluators, ", ")) } -// CreatedHeading opens the list of what a scaffold wrote. -func CreatedHeading() string { +// createdHeading opens the list of what a scaffold wrote. +func createdHeading() string { return "\nCreated\n" } @@ -1790,11 +1749,11 @@ func ScaffoldHeading(existed bool) string { if existed { return "\nUpdated\n" } - return CreatedHeading() + return createdHeading() } -// CreatedConfigLine names the configuration a scaffold wrote. -func CreatedConfigLine(configPath string) string { +// createdConfigLine names the configuration a scaffold wrote. +func createdConfigLine(configPath string) string { return fmt.Sprintf(" %-33s evaluation configuration\n", configPath) } @@ -1803,7 +1762,7 @@ func ScaffoldConfigLine(configPath string, existed bool) string { if existed { return fmt.Sprintf(" %-33s evaluation configuration (eval added)\n", configPath) } - return CreatedConfigLine(configPath) + return createdConfigLine(configPath) } // AddedServiceLine reports the eval service being added to the root config. @@ -2040,12 +1999,12 @@ func AmbiguousEvalConfig(current, legacy string) error { // ReadingEvalConfig reports a configuration file that would not read. func ReadingEvalConfig(path string, err error) error { if errors.Is(err, fs.ErrNotExist) { - return NoEvalConfig(path) + return noEvalConfig(path) } return fmt.Errorf("reading eval config %q: %w", filepath.ToSlash(path), err) } -// NoEvalConfig reports a command run before anything scaffolded a config. +// noEvalConfig reports a command run before anything scaffolded a config. // // The bare read failure underneath is a Windows syscall phrase about a path, // which describes the symptom of running `create` before `init` without naming @@ -2055,7 +2014,7 @@ func ReadingEvalConfig(path string, err error) error { // configuration — OpenEvalConfig, and the reference resolution above it — decide // that by asking, and a nicer sentence that stopped answering would turn every // one of those into a failure. -func NoEvalConfig(path string) error { +func noEvalConfig(path string) error { return &missingFileError{ msg: fmt.Sprintf( "no eval configuration at %s; run `azd ai eval init` to scaffold one", @@ -2149,16 +2108,11 @@ func SampleSizeOutOfRange(min, max, got int) error { return fmt.Errorf("sample size must be between %d and %d, got %d", min, max, got) } -// NegativeMaxSamples reports a declared row cap below zero. +// MaxSamplesNegative reports a declared row cap below zero. // // Anything not above zero reads as "no cap", so this used to send the whole // dataset to a run that is billed per row -- the opposite of what a cap asks // for, and silent. -func NegativeMaxSamples(index int, name string, got int) error { - return InEvalAt(index, name, MaxSamplesNegative(got)) -} - -// MaxSamplesNegative reports it where there is no index. func MaxSamplesNegative(got int) error { return fmt.Errorf( "max_samples cannot be negative, got %d. "+ diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/messages/spoken_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/messages/spoken_test.go new file mode 100644 index 00000000000..e9d3ae8ccba --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/messages/spoken_test.go @@ -0,0 +1,128 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package messages_test + +import ( + "go/ast" + "go/parser" + "go/token" + "io/fs" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +// Every sentence in this package has to be spoken by something. +// +// The package exists so the whole voice of the CLI can be read in one sitting. +// A constructor nobody calls is a sentence in that reading which no user will +// ever see, and it is worse than clutter: twice now, consolidating two ways of +// saying one thing has left the losing wording behind, still compiling, still +// reviewed, ready to be picked up again by someone who finds it and assumes it +// is live. `unused` cannot catch them, because they are exported. +// +// The rule is deliberately blunt -- referenced somewhere outside the package, +// or deleted. Anything genuinely written for a caller that does not exist yet +// should arrive with that caller. +func TestEveryMessageIsSpokenBySomething(t *testing.T) { + root := moduleRoot(t) + declared := exportedFuncs(t, filepath.Join(root, "internal", "messages")) + require.NotEmpty(t, declared, "the package should declare messages") + + spoken := referencedNames(t, root) + + var silent []string + for name := range declared { + if !spoken[name] { + silent = append(silent, name) + } + } + + require.Empty(t, silent, + "messages nothing calls: delete them, or call them.\n%s", + strings.Join(silent, "\n")) +} + +// moduleRoot walks up from this package to the directory holding go.mod. +func moduleRoot(t *testing.T) string { + t.Helper() + dir, err := filepath.Abs(".") + require.NoError(t, err) + for range 8 { + if _, err := filepath.Glob(filepath.Join(dir, "go.mod")); err == nil { + if matches, _ := filepath.Glob(filepath.Join(dir, "go.mod")); len(matches) > 0 { + return dir + } + } + parent := filepath.Dir(dir) + require.NotEqual(t, dir, parent, "walked past the filesystem root") + dir = parent + } + t.Fatal("no go.mod above the messages package") + return "" +} + +// exportedFuncs collects the package-level exported functions declared in dir. +func exportedFuncs(t *testing.T, dir string) map[string]bool { + t.Helper() + names := map[string]bool{} + fset := token.NewFileSet() + pkgs, err := parser.ParseDir(fset, dir, nil, 0) + require.NoError(t, err) + + for _, pkg := range pkgs { + for path, file := range pkg.Files { + // A test file's helpers are not part of the voice. + if strings.HasSuffix(path, "_test.go") { + continue + } + for _, decl := range file.Decls { + fn, ok := decl.(*ast.FuncDecl) + // Methods hang off a type and are reached through it. + if !ok || fn.Recv != nil || !fn.Name.IsExported() { + continue + } + names[fn.Name.Name] = true + } + } + } + return names +} + +// referencedNames collects every `messages.X` selector used outside the package. +func referencedNames(t *testing.T, root string) map[string]bool { + t.Helper() + used := map[string]bool{} + fset := token.NewFileSet() + + err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() || !strings.HasSuffix(path, ".go") { + return nil + } + file, err := parser.ParseFile(fset, path, nil, 0) + if err != nil { + // A file this package cannot parse is not this test's business; + // the build already fails on it. + return nil //nolint:nilerr // parse failures are the compiler's to report + } + ast.Inspect(file, func(n ast.Node) bool { + sel, ok := n.(*ast.SelectorExpr) + if !ok { + return true + } + if ident, ok := sel.X.(*ast.Ident); ok && ident.Name == "messages" { + used[sel.Sel.Name] = true + } + return true + }) + return nil + }) + require.NoError(t, err) + return used +} From 6bd737b56958c7cc2c754d6c3850e3359970491c Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 18 Aug 2026 09:07:03 +0300 Subject: [PATCH 264/320] Defend the change that removed the wrong-resource hazard Removing the EVAL_ID fallback was the riskiest change on this branch and the least defended: no test would have failed if it came back, and `resolveEvalID` still carried a dead `args` parameter with a doc comment describing behaviour it no longer had. Four of its seven callers have a positional that is a *run* id, so a maintainer wiring `args` through on the strength of that comment would turn `run cancel ` into a cancel against eval `` -- the exact failure class the change removed, left loaded in the signature. The parameter is gone and there is a test, verified by restoring the fallback and watching it fail. The seven run commands now prompt for the eval the way `run start` does. Losing the fallback otherwise left a two-eval project able to start a run by answering a question and then unable to list, show or cancel it without repeating the answer as a flag. A configuration with catalogs and no evals said "no evals are declared" on the run path and named the way out on the deploy path. That state is what `generate` leaves behind, and the run path is where a user lands, so it had the worse of the two sentences. One sentence now. An eval name that is not declared no longer trails off mid-clause when nothing is declared. The reachability test added last round was weaker than its own doc claimed: - It counted references from `_test.go` files, so the wording deleted in a consolidation could be kept alive by the test written for it -- which is how `evalIDKeys` survived. Production and test references are separated, and a message only a test calls now fails with its own message. - It matched the identifier `messages` without reading the file's imports, so an aliased import would have reported a live sentence as dead and a variable of that name would have marked dead ones live. - It walked its own package, so an external test file dropped beside it would have made the check satisfy itself. - `moduleRoot` used filepath.Glob as an existence check behind a tautological error guard. A checkout path containing a bracket answers ErrBadPattern and the walk would blame the repo layout. Widening it to exported vars and consts found three progress markers used only inside the package. Unexported, which puts them back under `unused`. And the "un start" typo in seven comments: `r in a PowerShell double-quoted string is a carriage return, which is how it was written in the first place and how it was rewritten on the first attempt to fix it. --- .../internal/cmd/eval_id_key_test.go | 37 ++++ .../internal/cmd/run_ops.go | 16 +- .../internal/cmd/run_output.go | 50 ++--- .../internal/messages/messages.go | 53 +++-- .../internal/messages/spoken_test.go | 188 +++++++++++++----- 5 files changed, 238 insertions(+), 106 deletions(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/eval_id_key_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/eval_id_key_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/eval_id_key_test.go new file mode 100644 index 00000000000..b44d34be590 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/eval_id_key_test.go @@ -0,0 +1,37 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// EVAL_ID is written by every deploy, so nothing tells a value meant for this +// declaration from one left behind by the eval it replaced. +// +// Reading it as a fallback meant a file whose one entry had been swapped ran +// the previous eval's criteria over the new one's rows and reported success, +// and `run cancel` with no arguments cancelled a run of the eval the file no +// longer described -- a destructive verb on a resource picked by accident. +// +// This is the only enforcement of that. The reasoning lives in a comment on +// the reconciler and in another on `resolveEvalID`, and a comment cannot fail. +func TestRecordedEvalIDIgnoresTheSharedKey(t *testing.T) { + env := &testEnvServer{values: map[string]string{ + envKeyEvalID: "evalgroup_the_one_this_replaced", + }} + ec := &evalContext{azdClient: newTestAzdClient(t, env), envName: "test"} + + assert.Empty(t, ec.recordedEvalID(context.Background(), "nightly"), + "a shared key cannot say which declaration it belongs to") + + // The entry recorded under the eval's own name does answer, which is what + // makes the miss above a deliberate refusal rather than a broken read. + env.values[idKey("eval", "nightly")] = "evalgroup_nightly" + require.Equal(t, "evalgroup_nightly", ec.recordedEvalID(context.Background(), "nightly")) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go index ced8d6bac06..ab9b08325e5 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go @@ -53,7 +53,7 @@ func newRunListCommand() *cobra.Command { } defer ec.Close() - evalID, err := resolveEvalID(cmd, ec, nil, groupName) + evalID, err := resolveEvalID(cmd, ec, groupName) if err != nil { return err } @@ -98,7 +98,7 @@ func newRunListCommand() *cobra.Command { } addEvalFlag(cmd, &groupName) // Registered wherever a declared name is resolved, so a configuration - // outside ./evals can be addressed by every command, not just un start. + // outside ./evals can be addressed by every command, not just `run start`. addEvalPathFlag(cmd, new(string)) cmd.Flags().IntVar(&limit, "limit", 0, "Return at most this many runs. Omit for the service default.") @@ -132,7 +132,7 @@ func newRunShowCommand() *cobra.Command { } defer ec.Close() - evalID, err := resolveEvalID(cmd, ec, nil, groupName) + evalID, err := resolveEvalID(cmd, ec, groupName) if err != nil { return err } @@ -203,7 +203,7 @@ func newRunShowCommand() *cobra.Command { addFailOnFlag(cmd, &failOn) addEvalFlag(cmd, &groupName) // Registered wherever a declared name is resolved, so a configuration - // outside ./evals can be addressed by every command, not just un start. + // outside ./evals can be addressed by every command, not just `run start`. addEvalPathFlag(cmd, new(string)) cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") return cmd @@ -235,7 +235,7 @@ func newRunCancelCommand() *cobra.Command { } defer ec.Close() - evalID, err := resolveEvalID(cmd, ec, nil, groupName) + evalID, err := resolveEvalID(cmd, ec, groupName) if err != nil { return err } @@ -269,7 +269,7 @@ func newRunCancelCommand() *cobra.Command { } addEvalFlag(cmd, &groupName) // Registered wherever a declared name is resolved, so a configuration - // outside ./evals can be addressed by every command, not just un start. + // outside ./evals can be addressed by every command, not just `run start`. addEvalPathFlag(cmd, new(string)) cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") return cmd @@ -300,7 +300,7 @@ func newRunDeleteCommand() *cobra.Command { } defer ec.Close() - evalID, err := resolveEvalID(cmd, ec, nil, groupName) + evalID, err := resolveEvalID(cmd, ec, groupName) if err != nil { return err } @@ -323,7 +323,7 @@ func newRunDeleteCommand() *cobra.Command { } addEvalFlag(cmd, &groupName) // Registered wherever a declared name is resolved, so a configuration - // outside ./evals can be addressed by every command, not just un start. + // outside ./evals can be addressed by every command, not just `run start`. addEvalPathFlag(cmd, new(string)) cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") return cmd diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output.go index 4a057628ee1..651e3990244 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output.go @@ -56,7 +56,7 @@ func newRunOutputListCommand() *cobra.Command { } defer ec.Close() - evalID, err := resolveEvalID(cmd, ec, nil, groupName) + evalID, err := resolveEvalID(cmd, ec, groupName) if err != nil { return err } @@ -117,7 +117,7 @@ func newRunOutputListCommand() *cobra.Command { cmd.Flags().StringVar(&outFile, "output-file", "", "Write JSON results to this path.") addEvalFlag(cmd, &groupName) // Registered wherever a declared name is resolved, so a configuration - // outside ./evals can be addressed by every command, not just un start. + // outside ./evals can be addressed by every command, not just `run start`. addEvalPathFlag(cmd, new(string)) cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") return cmd @@ -148,7 +148,7 @@ func newRunOutputShowCommand() *cobra.Command { } defer ec.Close() - evalID, err := resolveEvalID(cmd, ec, nil, groupName) + evalID, err := resolveEvalID(cmd, ec, groupName) if err != nil { return err } @@ -175,7 +175,7 @@ func newRunOutputShowCommand() *cobra.Command { cmd.Flags().StringVar(&runID, "run", "", "Run the item belongs to. Defaults to the most recent run.") addEvalFlag(cmd, &groupName) // Registered wherever a declared name is resolved, so a configuration - // outside ./evals can be addressed by every command, not just un start. + // outside ./evals can be addressed by every command, not just `run start`. addEvalPathFlag(cmd, new(string)) cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") return cmd @@ -231,7 +231,7 @@ func newRunOutputExportCommand() *cobra.Command { } defer ec.Close() - evalID, err := resolveEvalID(cmd, ec, nil, groupName) + evalID, err := resolveEvalID(cmd, ec, groupName) if err != nil { return err } @@ -269,36 +269,36 @@ func newRunOutputExportCommand() *cobra.Command { cmd.Flags().StringVar(&outFile, "output-file", "", "Write to this path instead of stdout.") addEvalFlag(cmd, &groupName) // Registered wherever a declared name is resolved, so a configuration - // outside ./evals can be addressed by every command, not just un start. + // outside ./evals can be addressed by every command, not just `run start`. addEvalPathFlag(cmd, new(string)) cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") return cmd } -// resolveEvalID takes the eval id from the argument, from --eval, or from the -// id cached in the azd environment. +// resolveEvalID resolves the eval a run command is about, from --eval or from +// the declaration the configuration holds. // // --eval accepts a name or a raw id on the one flag: an eval created outside a // project has no declaration to name, and the environment records one id per // name, so editing a declaration leaves every run of the previous eval // reachable only by id. -func resolveEvalID( - cmd *cobra.Command, - ec *evalContext, - args []string, - groupName string, -) (string, error) { - if len(args) > 0 && args[0] != "" { - return args[0], nil - } - - // With no name given, the eval is the one the configuration declares, - // which is how `run start` decides it. Reading EVAL_ID here instead made - // the two disagree: every deploy writes that key, so a file whose one entry - // had been replaced sent `run cancel` at the eval it used to be -- a - // destructive verb on the wrong resource, chosen silently. - ref, err := ec.resolveEvalRef( - cmd.Context(), ec.evalDir(cmd.Context(), evalPathFlag(cmd)), groupName) +// +// It takes no positional argument, deliberately. The positional on `run show`, +// `run cancel` and `run output *` is a *run* id, and a signature that accepted +// either would let one be resolved as the other -- a destructive verb aimed at +// a resource picked by accident. +// +// It reads no EVAL_ID either. Every deploy writes that key, so nothing tells a +// value meant for this declaration from one left behind by the eval it +// replaced; `run cancel` used to cancel a run of an eval the file no longer +// described. The declaration is asked instead, which is how `run start` +// decides it, so the two doors cannot pick different evals. +func resolveEvalID(cmd *cobra.Command, ec *evalContext, groupName string) (string, error) { + evalDir := ec.evalDir(cmd.Context(), evalPathFlag(cmd)) + // The same prompt `run start` gets. Without it a project declaring two + // evals could start a run by answering a question, and then not list, + // show or cancel it without repeating the answer as a flag. + ref, err := ec.resolveEvalRef(cmd.Context(), evalDir, chooseEvalIn(cmd, evalDir, groupName)) if err != nil { return "", err } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go b/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go index 6dd05d9071f..3805ef58b4b 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go @@ -488,7 +488,7 @@ func GatePassRateBelow(actual, required float64) string { // GateBreached is the block a breached gate leaves in a pipeline's log. func GateBreached(reason string) string { return fmt.Sprintf("%s Evaluation gate: %s\n\nERROR: evaluation quality gate not met.\n", - FailedMark, reason) + failedMark, reason) } // --------------------------------------------------------------------------- @@ -735,7 +735,7 @@ func ReattachToJob(selector, jobID string) string { // WroteArtifact reports where a generated artifact landed. func WroteArtifact(path string) string { - return fmt.Sprintf("%s Downloaded %s\n", DoneMark, filepath.ToSlash(path)) + return fmt.Sprintf("%s Downloaded %s\n", doneMark, filepath.ToSlash(path)) } // ArtifactExists reports a generation that would overwrite a checked-in file. @@ -1357,7 +1357,7 @@ func EvalUnchangedProgress(eval, id string) string { // EvalCreated confirms a single eval created outside a full deploy. func EvalCreated(eval, id string) string { - return fmt.Sprintf("%s Created eval: %s (%s)\n", DoneMark, eval, id) + return fmt.Sprintf("%s Created eval: %s (%s)\n", doneMark, eval, id) } // EvalUnchanged reports an eval a create found already in place. @@ -1367,7 +1367,7 @@ func EvalCreated(eval, id string) string { // hides the one thing worth checking: that the id, and so the run history // hanging off it, survived. func EvalUnchanged(eval, id string) string { - return fmt.Sprintf("%s Eval %s is unchanged (%s)\n", SkippedMark, eval, id) + return fmt.Sprintf("%s Eval %s is unchanged (%s)\n", skippedMark, eval, id) } // ListingEvals reports a failure to list the project's evals. @@ -1513,7 +1513,7 @@ func CreatingEvaluatorsDir(err error) error { // DetectedTarget reports the agent the scaffolded eval will evaluate. func DetectedTarget(target string) string { - return fmt.Sprintf("%s Detected agent target: %s\n", DoneMark, target) + return fmt.Sprintf("%s Detected agent target: %s\n", doneMark, target) } // NoAgentToEvaluate reports a project declaring no agent service. @@ -1716,17 +1716,17 @@ func SelectingJudgeModel(err error) error { // cannot verify one it did not see. func UsingTraceSource(connected bool) string { if connected { - return fmt.Sprintf("%s Using data source: traces (Application Insights)\n", DoneMark) + return fmt.Sprintf("%s Using data source: traces (Application Insights)\n", doneMark) } return fmt.Sprintf( "%s Using data source: traces. No Application Insights connection is recorded "+ "in this environment, so the run finds rows only if the project has one\n", - DoneMark) + doneMark) } // JudgeModelDeployment reports the deployment the graders will judge with. func JudgeModelDeployment(model string) string { - return fmt.Sprintf("%s Judge model deployment: %s\n", DoneMark, model) + return fmt.Sprintf("%s Judge model deployment: %s\n", doneMark, model) } // GradingWith reports the evaluators the scaffold settled on. @@ -1734,7 +1734,7 @@ func JudgeModelDeployment(model string) string { // Omitting --evaluator picks them, so without this the one thing `init` decided // on the reader's behalf is the one thing it does not mention. func GradingWith(evaluators []string) string { - return fmt.Sprintf("%s Grading with: %s\n", DoneMark, strings.Join(evaluators, ", ")) + return fmt.Sprintf("%s Grading with: %s\n", doneMark, strings.Join(evaluators, ", ")) } // createdHeading opens the list of what a scaffold wrote. @@ -1787,12 +1787,12 @@ func FurtherNextStep(step string) string { // CreatedCatalogFile reports a configuration created to hold a catalog entry. func CreatedCatalogFile(configPath string) string { - return fmt.Sprintf("%s Created %s with the catalog entry\n", DoneMark, filepath.ToSlash(configPath)) + return fmt.Sprintf("%s Created %s with the catalog entry\n", doneMark, filepath.ToSlash(configPath)) } // AddedToCatalog reports a generated artifact recorded in the configuration. func AddedToCatalog(kind, artifact, configPath string) string { - return fmt.Sprintf("%s Added %s %s to %s\n", DoneMark, kind, artifact, filepath.ToSlash(configPath)) + return fmt.Sprintf("%s Added %s %s to %s\n", doneMark, kind, artifact, filepath.ToSlash(configPath)) } // ArtifactDescription names a catalogued artifact, with its version when there is one. @@ -1804,8 +1804,15 @@ func ArtifactDescription(name, version string) string { } // NoEvalsDeclared reports a configuration with nothing to act on. +// +// The same sentence wherever it is reached. `generate` writes the dataset and +// evaluator it made into the catalog but declares no eval, so a `create` or a +// run straight afterwards lands here, and both need to be told the same way +// out. func NoEvalsDeclared() error { - return errors.New("no evals are declared") + return errors.New( + "no eval is declared; `azd ai eval init` declares one. " + + "`generate` only adds the dataset and evaluator it made") } // SeveralEvalsDeclared reports an unnamed eval where guessing would be wrong. @@ -1821,20 +1828,22 @@ func SeveralEvalsDeclared(count int, names []string) error { // EvalNotDeclared reports a name the configuration does not carry. func EvalNotDeclared(eval string, names []string) error { + // "this configuration has" with nothing after it is a sentence that stops + // mid-clause, which is what an empty list produces. + if len(names) == 0 { + return fmt.Errorf("eval %q is not declared, and this configuration declares none", eval) + } return fmt.Errorf( "eval %q is not declared; this configuration has %s", eval, strings.Join(names, ", ")) } -// AtLeastOneEvalRequired reports a configuration that declares no eval. +// AtLeastOneEvalRequired reports it on the way to deploying. // -// `generate` writes the dataset and evaluator it made into the catalog but -// declares no eval, so a `create` run straight afterwards lands here with -// nothing to act on. +// One sentence for one fact: the deploy door and the run door reach this from +// different directions and both need the same way out. func AtLeastOneEvalRequired() error { - return errors.New( - "no eval is declared; `azd ai eval init` declares one. " + - "`generate` only adds the dataset and evaluator it made") + return NoEvalsDeclared() } // EvalNameRequired reports an eval entry with no name. @@ -2241,9 +2250,9 @@ func ProjectContextRead(err error) error { // Progress markers from the azd style guide, so the extension's lines sit // alongside core's without a second vocabulary. const ( - DoneMark = "(✓) Done:" // finished successfully - SkippedMark = "(-) Skipped:" // intentionally not done, not a failure - FailedMark = "(x) Failed:" // the step did not complete + doneMark = "(✓) Done:" // finished successfully + skippedMark = "(-) Skipped:" // intentionally not done, not a failure + failedMark = "(x) Failed:" // the step did not complete ) // Warning reports a problem that is not worth failing the command over. diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/messages/spoken_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/messages/spoken_test.go index e9d3ae8ccba..b9c392f0dba 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/messages/spoken_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/messages/spoken_test.go @@ -8,6 +8,7 @@ import ( "go/parser" "go/token" "io/fs" + "os" "path/filepath" "strings" "testing" @@ -15,35 +16,47 @@ import ( "github.com/stretchr/testify/require" ) -// Every sentence in this package has to be spoken by something. +// importPath is what a file has to import for its `x.Name` selectors to count. +const importPath = "azureaieval/internal/messages" + +// Every sentence in this package has to be spoken by something a user runs. // // The package exists so the whole voice of the CLI can be read in one sitting. // A constructor nobody calls is a sentence in that reading which no user will -// ever see, and it is worse than clutter: twice now, consolidating two ways of -// saying one thing has left the losing wording behind, still compiling, still -// reviewed, ready to be picked up again by someone who finds it and assumes it -// is live. `unused` cannot catch them, because they are exported. +// ever see, and it is worse than clutter: three times now, consolidating two +// ways of saying one thing has left the losing wording behind -- still +// compiling, still reviewed, ready to be picked up by someone who finds it and +// assumes it is live. `unused` cannot catch them, because they are exported. // -// The rule is deliberately blunt -- referenced somewhere outside the package, -// or deleted. Anything genuinely written for a caller that does not exist yet -// should arrive with that caller. -func TestEveryMessageIsSpokenBySomething(t *testing.T) { +// A reference from a test does not count. A message that only a test calls is +// still a sentence no user sees, and counting tests would let the deleted +// wording be kept alive by the test written for it, which is exactly what +// happened to `evalIDKeys`. +func TestEveryMessageIsSpokenBySomethingAUserRuns(t *testing.T) { root := moduleRoot(t) - declared := exportedFuncs(t, filepath.Join(root, "internal", "messages")) + pkgDir := filepath.Join(root, "internal", "messages") + + declared := exported(t, pkgDir) require.NotEmpty(t, declared, "the package should declare messages") - spoken := referencedNames(t, root) + spoken := referenced(t, root, pkgDir, false) + fromTests := referenced(t, root, pkgDir, true) - var silent []string + var silent, testOnly []string for name := range declared { - if !spoken[name] { + switch { + case spoken[name]: + case fromTests[name]: + testOnly = append(testOnly, name) + default: silent = append(silent, name) } } require.Empty(t, silent, - "messages nothing calls: delete them, or call them.\n%s", - strings.Join(silent, "\n")) + "messages nothing calls: delete them, or call them.\n%s", strings.Join(silent, "\n")) + require.Empty(t, testOnly, + "messages only a test calls, so no user ever sees them:\n%s", strings.Join(testOnly, "\n")) } // moduleRoot walks up from this package to the directory holding go.mod. @@ -51,72 +64,101 @@ func moduleRoot(t *testing.T) string { t.Helper() dir, err := filepath.Abs(".") require.NoError(t, err) - for range 8 { - if _, err := filepath.Glob(filepath.Join(dir, "go.mod")); err == nil { - if matches, _ := filepath.Glob(filepath.Join(dir, "go.mod")); len(matches) > 0 { - return dir - } + for { + // Stat rather than Glob: Glob reads its argument as a pattern, so a + // checkout path containing a bracket answers ErrBadPattern and the + // walk blames the repo layout for a path this test chose to build. + if _, err := os.Stat(filepath.Join(dir, "go.mod")); err == nil { + return dir } parent := filepath.Dir(dir) - require.NotEqual(t, dir, parent, "walked past the filesystem root") + require.NotEqual(t, dir, parent, "no go.mod above the messages package") dir = parent } - t.Fatal("no go.mod above the messages package") - return "" } -// exportedFuncs collects the package-level exported functions declared in dir. -func exportedFuncs(t *testing.T, dir string) map[string]bool { +// exported collects the package-level exported names declared in dir. +// +// Functions, and the vars and consts beside them: a marker nobody prints +// is as unspoken as a constructor nobody calls. +func exported(t *testing.T, dir string) map[string]bool { t.Helper() names := map[string]bool{} - fset := token.NewFileSet() - pkgs, err := parser.ParseDir(fset, dir, nil, 0) - require.NoError(t, err) - for _, pkg := range pkgs { - for path, file := range pkg.Files { - // A test file's helpers are not part of the voice. - if strings.HasSuffix(path, "_test.go") { - continue - } - for _, decl := range file.Decls { - fn, ok := decl.(*ast.FuncDecl) - // Methods hang off a type and are reached through it. - if !ok || fn.Recv != nil || !fn.Name.IsExported() { - continue + for _, file := range parseDir(t, dir) { + if strings.HasSuffix(file.path, "_test.go") { + continue + } + for _, decl := range file.ast.Decls { + switch d := decl.(type) { + case *ast.FuncDecl: + // A method is reached through its type, not by name here. + if d.Recv == nil && d.Name.IsExported() { + names[d.Name.Name] = true + } + case *ast.GenDecl: + for _, spec := range d.Specs { + value, ok := spec.(*ast.ValueSpec) + if !ok { + continue + } + for _, ident := range value.Names { + if ident.IsExported() { + names[ident.Name] = true + } + } } - names[fn.Name.Name] = true } } } return names } -// referencedNames collects every `messages.X` selector used outside the package. -func referencedNames(t *testing.T, root string) map[string]bool { +// referenced collects the names selected off this package elsewhere in the +// module, from test files or from the rest of it. +func referenced(t *testing.T, root, pkgDir string, inTests bool) map[string]bool { t.Helper() used := map[string]bool{} - fset := token.NewFileSet() err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error { if err != nil { - return err + // An entry this test cannot read says nothing about messages, and + // failing the invariant over it would report the wrong problem. + return nil //nolint:nilerr // unreadable entries are not this test's business + } + if d.IsDir() { + // The package's own internal tests call these bare, so they + // contribute no selectors; an external one beside this file would, + // and would make the check satisfy itself. + if path == pkgDir { + return fs.SkipDir + } + return nil + } + if !strings.HasSuffix(path, ".go") || strings.HasSuffix(path, "_test.go") != inTests { + return nil + } + file, err := parser.ParseFile(token.NewFileSet(), path, nil, parser.ImportsOnly|parser.SkipObjectResolution) + if err != nil { + return nil //nolint:nilerr // a file that will not parse is the compiler's to report } - if d.IsDir() || !strings.HasSuffix(path, ".go") { + local, ok := localName(file) + if !ok { return nil } - file, err := parser.ParseFile(fset, path, nil, 0) + full, err := parser.ParseFile(token.NewFileSet(), path, nil, parser.SkipObjectResolution) if err != nil { - // A file this package cannot parse is not this test's business; - // the build already fails on it. - return nil //nolint:nilerr // parse failures are the compiler's to report + return nil //nolint:nilerr // a file that will not parse is the compiler's to report } - ast.Inspect(file, func(n ast.Node) bool { + ast.Inspect(full, func(n ast.Node) bool { sel, ok := n.(*ast.SelectorExpr) if !ok { return true } - if ident, ok := sel.X.(*ast.Ident); ok && ident.Name == "messages" { + // Matched against the name this file bound the import to, so an + // aliased import counts and an unrelated `messages` identifier + // does not. + if ident, ok := sel.X.(*ast.Ident); ok && ident.Name == local { used[sel.Sel.Name] = true } return true @@ -126,3 +168,47 @@ func referencedNames(t *testing.T, root string) map[string]bool { require.NoError(t, err) return used } + +// localName is the name a file binds this package to, if it imports it. +func localName(file *ast.File) (string, bool) { + for _, imp := range file.Imports { + if strings.Trim(imp.Path.Value, `"`) != importPath { + continue + } + if imp.Name != nil { + // A dot import puts the names in scope unqualified, which this + // test cannot follow; nothing in the module does it. + if imp.Name.Name == "." || imp.Name.Name == "_" { + return "", false + } + return imp.Name.Name, true + } + return "messages", true + } + return "", false +} + +type parsedFile struct { + path string + ast *ast.File +} + +func parseDir(t *testing.T, dir string) []parsedFile { + t.Helper() + entries, err := os.ReadDir(dir) + require.NoError(t, err) + + var files []parsedFile + for _, entry := range entries { + if entry.IsDir() || !strings.HasSuffix(entry.Name(), ".go") { + continue + } + path := filepath.Join(dir, entry.Name()) + // ParseDir is deprecated as of Go 1.22, and reading the directory + // here costs nothing. + file, err := parser.ParseFile(token.NewFileSet(), path, nil, parser.SkipObjectResolution) + require.NoError(t, err) + files = append(files, parsedFile{path: path, ast: file}) + } + return files +} From b8614766e6853cbe8b22013de82f3e128b87678f Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 18 Aug 2026 09:22:03 +0300 Subject: [PATCH 265/320] Stop a shared temp directory choosing where the debug log goes MkdirAll accepts a directory that is already there and does not make it private. The temp directory is shared on Linux, so another user could leave `azd-ai-eval` in place -- world-readable, to read HTTP traces that carry request headers, or holding a symbolic link at the predictable daily filename, to have the traces appended to a file of their choosing. MkdirTemp picks the name and creates it private in one step, and the file is opened O_EXCL inside it, so neither is reachable. The path was already echoed to stderr, which is what made a fixed name worth having in the first place. The gRPC absence rule stops reading every Unknown as "nothing here". azd declares its absences with errors.New and its interceptor passes anything carrying no suggestion and no auth failure through untouched, so "default environment not found" and a failure to load project state arrive under the same code. Only the two absences are admitted, matched by the message that is all that survives the wire. Matching text is a coupling, and it fails closed: if azd renames a sentinel the command reports the daemon error instead of resolving quietly to a lower-priority endpoint that can belong to a different project. Both are byte-identical in the dataset extension and are fixed there too. --- .../internal/cmd/debug.go | 37 ++++++++++------- .../foundry/projectctx/hosted_absence_test.go | 13 +++++- .../internal/foundry/projectctx/resolver.go | 41 +++++++++++++++---- 3 files changed, 66 insertions(+), 25 deletions(-) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/debug.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/debug.go index edb3c0ef6c7..f781cb00734 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/debug.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/debug.go @@ -31,20 +31,29 @@ func setupDebugLogging(flags *pflag.FlagSet) func() { // Written outside the working directory: that is the user's repository, the // scaffolded .gitignore does not cover this name, and a routine `git add -A` - // committed one. The private subdirectory matters because the temp directory - // is shared on Linux -- at a predictable path another user could leave a - // symbolic link and have the HTTP trace appended to a file of their choosing. - logDir := filepath.Join(os.TempDir(), "azd-ai-eval") - if err := os.MkdirAll(logDir, 0o700); err != nil { - logDir = os.TempDir() + // committed one. + // + // A fresh random directory rather than a named one. The temp directory is + // shared on Linux, and MkdirAll accepts a directory that is already there + // without making it private, so at a predictable path another user could + // leave one they own -- world-readable, to read the HTTP traces, or holding + // a symbolic link at the predictable daily name, to have them appended to a + // file of their choosing. MkdirTemp picks the name and creates it in one + // step, so neither is possible. The path is echoed below, which is what + // made the fixed name worth having. + logDir, dirErr := os.MkdirTemp("", "azd-ai-eval-") + var logFile *os.File + var err error + if dirErr != nil { + err = dirErr + } else { + logFileName := filepath.Join( + logDir, + fmt.Sprintf("azd-ai-eval-%s.log", time.Now().Format("2006-01-02")), + ) + //nolint:gosec // the directory was just created private, and the name is a date + logFile, err = os.OpenFile(logFileName, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0600) } - logFileName := filepath.Join( - logDir, - fmt.Sprintf("azd-ai-eval-%s.log", time.Now().Format("2006-01-02")), - ) - - //nolint:gosec // the name is generated locally from the date, not user input - logFile, err := os.OpenFile(logFileName, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0600) var w io.Writer var closeFile func() @@ -56,7 +65,7 @@ func setupDebugLogging(flags *pflag.FlagSet) func() { closeFile = func() { logFile.Close() } //nolint:gosec // best-effort cleanup // A log nobody can find is not a log. Debugging was asked for // explicitly, so naming the file costs nothing. - fmt.Fprintf(os.Stderr, "Debug log: %s\n", filepath.ToSlash(logFileName)) + fmt.Fprintf(os.Stderr, "Debug log: %s\n", filepath.ToSlash(logFile.Name())) } log.SetOutput(w) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/foundry/projectctx/hosted_absence_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/foundry/projectctx/hosted_absence_test.go index 6c20983c26d..ed5333055c9 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/foundry/projectctx/hosted_absence_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/foundry/projectctx/hosted_absence_test.go @@ -13,8 +13,8 @@ import ( "google.golang.org/grpc/status" ) -// azd answers "no default environment" and "no such key" with plain Go errors. -// Its interceptor only rewrites errors carrying a suggestion or an auth +// azd answers "no default environment" and "no such environment" with plain Go +// errors. Its interceptor only rewrites errors carrying a suggestion or an auth // failure, so everything else reaches the client as Unknown. Reading Unknown as // a failure would stop a project with no environment selected from ever // reaching the global config or the host variable. @@ -23,6 +23,7 @@ func TestUnansweredHostedSourcesLetTheCascadeCarryOn(t *testing.T) { "no daemon at all": status.Error(codes.Unavailable, "connection refused"), "nothing under that key": status.Error(codes.NotFound, "key not found"), "no default environment": status.Error(codes.Unknown, "default environment not found"), + "no such environment": status.Error(codes.Unknown, "'dev': environment not found"), "wrapped in context": fmt.Errorf("reading the environment: %w", status.Error(codes.Unknown, "default environment not found")), } { @@ -48,6 +49,14 @@ func TestAFailureToAnswerIsReportedRatherThanSkipped(t *testing.T) { // A bare error carries no status at all, so it never travelled the wire // as an absence the daemon reported. "not a status at all": errors.New("something local went wrong"), + // Unknown is not absence on its own. azd passes any error carrying no + // suggestion and no auth failure through untouched, so a failure to + // load project state or the environment manager arrives under the same + // code as "no default environment". + "project state would not load": status.Error(codes.Unknown, + "loading project state: open azure.yaml: permission denied"), + "the environment manager broke": status.Error(codes.Unknown, + "creating environment manager: no such host"), "wrapped expiry": fmt.Errorf("reading the environment: %w", status.Error(codes.Unauthenticated, "expired")), "nested twice over": fmt.Errorf("outer: %w", diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/foundry/projectctx/resolver.go b/cli/azd/extensions/azure.ai.evaluations/internal/foundry/projectctx/resolver.go index db8208679ba..cd385127dba 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/foundry/projectctx/resolver.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/foundry/projectctx/resolver.go @@ -7,6 +7,7 @@ import ( "context" "errors" "os" + "strings" "github.com/azure/azure-dev/cli/azd/pkg/azdext" "google.golang.org/grpc/codes" @@ -78,25 +79,47 @@ func readAzdHostedSources(ctx context.Context) (AzdHostedSources, error) { return out, nil } +// azd's sentinels for the two absences that reach us without a status. +// +// `pkg/environment` declares these with errors.New, and the daemon's +// error-wrapping interceptor passes an error carrying no suggestion and no auth +// failure through untouched, so grpc encodes both as Unknown -- the same code a +// failure to load project state or the environment manager arrives under. The +// message is the only thing left to tell them apart. +// +// Matched rather than imported: taking a dependency on the environment manager +// for two strings costs more than it settles, and a rename fails closed. The +// command would report the daemon error instead of resolving quietly to a +// lower-priority endpoint, which is the direction to fail in. +const ( + azdNoDefaultEnvironment = "default environment not found" + azdNoSuchEnvironment = "environment not found" +) + // hostedSourceAbsent reports whether an error from the azd daemon leaves the // cascade free to carry on to the next level. // -// Three codes mean "nothing here". Unavailable is no daemon at all. NotFound is -// a daemon with nothing under that name. Unknown is the one that is not -// obvious: azd answers the ordinary absences -- no default environment, no such -// key -- with plain Go errors, its interceptor passes those through untouched, -// and grpc encodes an error carrying no status as Unknown. Without it, a +// Unavailable is no daemon at all. NotFound is a daemon with nothing under that +// name. Unknown is the one that is not obvious: azd answers the ordinary +// absences -- no default environment, no such environment -- with plain Go +// errors that reach us with no status, and without letting those through, a // project that simply has no environment selected could never reach the global -// config or the host variable. +// config or the host variable. It is admitted only for those two, because +// Unknown is equally what a failure to load project state arrives as. // // Everything else is a failure to answer rather than an answer of "nothing": // an expired login, a denial, a cancellation, or a server fault. Falling // through on any of those would resolve quietly to a lower-priority endpoint // that can belong to a different project. func hostedSourceAbsent(err error) bool { - return containsGRPCCode(err, codes.Unavailable) || - containsGRPCCode(err, codes.NotFound) || - containsGRPCCode(err, codes.Unknown) + if containsGRPCCode(err, codes.Unavailable) || containsGRPCCode(err, codes.NotFound) { + return true + } + if !containsGRPCCode(err, codes.Unknown) { + return false + } + msg := status.Convert(err).Message() + return strings.Contains(msg, azdNoDefaultEnvironment) || strings.Contains(msg, azdNoSuchEnvironment) } // containsGRPCCode walks the error chain looking for a gRPC status with the From 17599fafb970569e1ae5c22d3385b97d7e0deb17 Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 18 Aug 2026 09:54:36 +0300 Subject: [PATCH 266/320] Let the filename carry the safety, not a directory A private directory per invocation was more machinery than the problem needed and left litter behind: nothing removed the directory, and on a failed open it leaked an empty one. CreateTemp finds an unused name and creates the file 0600 in one step, which is what actually defeats the pre-created file and the planted symlink -- the directory was never the load-bearing part. The date stays in the name because it is what makes a directory of these readable. That also matters more than it looks: setupDebugLogging runs from PersistentPreRunE, so AZD_EXT_DEBUG in a profile reaches it on every command azd runs, including the hidden ones it uses to discover the command tree. Windows never sweeps %TEMP% on its own. The absence match was wider than its own comment claimed. "default environment not found" contains "environment not found", so the first constant could never decide anything, and the substring test admitted any message mentioning an environment -- including a failure. Both sentinels are matched whole now, in the two shapes azd emits them. It also reads the innermost Unknown status rather than status.FromError, which flattens a wrapper's own prose into the message it reports. Nothing wraps these today, but a wrapper worded like an absence would otherwise have decided the classification -- the silent wrong-project outcome this exists to prevent. Both are pinned by tests that fail against the looser match. The doc said callers should defer the cleanup. Neither of them does, and both run for the length of the process, so it says what it is for instead. The nolint on Close named gosec, which does not report it; the assignment says it. --- .../internal/cmd/debug.go | 39 +++++-------- .../foundry/projectctx/hosted_absence_test.go | 9 +++ .../internal/foundry/projectctx/resolver.go | 56 ++++++++++++++----- 3 files changed, 66 insertions(+), 38 deletions(-) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/debug.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/debug.go index f781cb00734..6934bc01997 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/debug.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/debug.go @@ -20,8 +20,11 @@ import ( // // The data-plane clients trace every request through log.Printf, which Go // writes to stderr by default. Without this the CLI interleaves raw HTTP traces -// with its own output on every command. Returns a cleanup function the caller -// should defer. +// with its own output on every command. +// +// The returned function puts the logger back and closes the file. Callers run +// for the length of the process and let the OS close it, so it is returned for +// tests and for any caller that wants to stop logging early. func setupDebugLogging(flags *pflag.FlagSet) func() { if !isDebug(flags) { log.SetOutput(io.Discard) @@ -33,27 +36,15 @@ func setupDebugLogging(flags *pflag.FlagSet) func() { // scaffolded .gitignore does not cover this name, and a routine `git add -A` // committed one. // - // A fresh random directory rather than a named one. The temp directory is - // shared on Linux, and MkdirAll accepts a directory that is already there - // without making it private, so at a predictable path another user could - // leave one they own -- world-readable, to read the HTTP traces, or holding - // a symbolic link at the predictable daily name, to have them appended to a - // file of their choosing. MkdirTemp picks the name and creates it in one - // step, so neither is possible. The path is echoed below, which is what - // made the fixed name worth having. - logDir, dirErr := os.MkdirTemp("", "azd-ai-eval-") - var logFile *os.File - var err error - if dirErr != nil { - err = dirErr - } else { - logFileName := filepath.Join( - logDir, - fmt.Sprintf("azd-ai-eval-%s.log", time.Now().Format("2006-01-02")), - ) - //nolint:gosec // the directory was just created private, and the name is a date - logFile, err = os.OpenFile(logFileName, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0600) - } + // The name is picked by CreateTemp rather than built from the date alone. + // The temp directory is shared on Linux, and at a predictable path another + // user can leave a file of their own -- readable, to collect HTTP traces + // that carry request headers, or a symbolic link, to have them written to a + // file of their choosing. CreateTemp finds an unused name and creates it + // 0600 in one step, so neither is reachable. The date stays in the name + // because it is what makes a directory of these readable, and the full path + // is echoed below. + logFile, err := os.CreateTemp("", fmt.Sprintf("azd-ai-eval-%s-*.log", time.Now().Format("2006-01-02"))) var w io.Writer var closeFile func() @@ -62,7 +53,7 @@ func setupDebugLogging(flags *pflag.FlagSet) func() { closeFile = func() {} } else { w = logFile - closeFile = func() { logFile.Close() } //nolint:gosec // best-effort cleanup + closeFile = func() { _ = logFile.Close() } // A log nobody can find is not a log. Debugging was asked for // explicitly, so naming the file costs nothing. fmt.Fprintf(os.Stderr, "Debug log: %s\n", filepath.ToSlash(logFile.Name())) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/foundry/projectctx/hosted_absence_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/foundry/projectctx/hosted_absence_test.go index ed5333055c9..7d5d7fd16c0 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/foundry/projectctx/hosted_absence_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/foundry/projectctx/hosted_absence_test.go @@ -57,6 +57,15 @@ func TestAFailureToAnswerIsReportedRatherThanSkipped(t *testing.T) { "loading project state: open azure.yaml: permission denied"), "the environment manager broke": status.Error(codes.Unknown, "creating environment manager: no such host"), + // The message is the only evidence, so it is matched whole. A failure + // whose prose happens to mention one must not read as an absence. + "a failure that mentions an environment": status.Error(codes.Unknown, + "listing deployments: the environment not found in the subscription cache"), + // status.FromError flattens a wrapper's own prose into the message it + // reports, so a wrapper worded like an absence must not decide this. + "a failure wrapped in absence-sounding prose": fmt.Errorf( + "default environment not found in the cache: %w", + status.Error(codes.Unknown, "loading project state: permission denied")), "wrapped expiry": fmt.Errorf("reading the environment: %w", status.Error(codes.Unauthenticated, "expired")), "nested twice over": fmt.Errorf("outer: %w", diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/foundry/projectctx/resolver.go b/cli/azd/extensions/azure.ai.evaluations/internal/foundry/projectctx/resolver.go index cd385127dba..fd0f9b1e2da 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/foundry/projectctx/resolver.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/foundry/projectctx/resolver.go @@ -79,13 +79,19 @@ func readAzdHostedSources(ctx context.Context) (AzdHostedSources, error) { return out, nil } -// azd's sentinels for the two absences that reach us without a status. +// azd's two absence sentinels, as they reach us. // -// `pkg/environment` declares these with errors.New, and the daemon's +// `pkg/environment` declares them with errors.New, and the daemon's // error-wrapping interceptor passes an error carrying no suggestion and no auth -// failure through untouched, so grpc encodes both as Unknown -- the same code a -// failure to load project state or the environment manager arrives under. The -// message is the only thing left to tell them apart. +// failure through untouched, so both arrive as Unknown -- the same code a +// failure to load project state arrives under. The message is the only thing +// left to tell them apart. +// +// Matched whole rather than by substring. The message is the only evidence +// there is, so a failure whose prose happens to mention an environment must not +// read as one of these. The default-environment sentinel arrives on its own; +// the named one arrives from the data store as `'': environment not +// found`. // // Matched rather than imported: taking a dependency on the environment manager // for two strings costs more than it settles, and a rename fails closed. The @@ -100,12 +106,13 @@ const ( // cascade free to carry on to the next level. // // Unavailable is no daemon at all. NotFound is a daemon with nothing under that -// name. Unknown is the one that is not obvious: azd answers the ordinary -// absences -- no default environment, no such environment -- with plain Go -// errors that reach us with no status, and without letting those through, a -// project that simply has no environment selected could never reach the global -// config or the host variable. It is admitted only for those two, because -// Unknown is equally what a failure to load project state arrives as. +// name -- kept as a defence, though azd's environment service does not use it +// today. Unknown is the one that is not obvious: azd answers the ordinary +// absences with plain Go errors that reach us with no status, and without +// letting those through, a project that simply has no environment selected +// could never reach the global config or the host variable. It is admitted only +// for the two messages above, because Unknown is equally what a failure to load +// project state arrives as. // // Everything else is a failure to answer rather than an answer of "nothing": // an expired login, a denial, a cancellation, or a server fault. Falling @@ -115,11 +122,32 @@ func hostedSourceAbsent(err error) bool { if containsGRPCCode(err, codes.Unavailable) || containsGRPCCode(err, codes.NotFound) { return true } - if !containsGRPCCode(err, codes.Unknown) { + msg, ok := unknownStatusMessage(err) + if !ok { return false } - msg := status.Convert(err).Message() - return strings.Contains(msg, azdNoDefaultEnvironment) || strings.Contains(msg, azdNoSuchEnvironment) + return msg == azdNoDefaultEnvironment || strings.HasSuffix(msg, "': "+azdNoSuchEnvironment) +} + +// unknownStatusMessage is the message carried by the innermost Unknown status +// in the chain, if there is one. +// +// Read link by link rather than through status.FromError, which flattens: given +// a wrapped error it replaces the status message with the whole of err.Error(), +// so the wrapper's own prose would take part in the comparison above. Only what +// the daemon said should decide whether this is an absence. +func unknownStatusMessage(err error) (string, bool) { + type grpcStatuser interface{ GRPCStatus() *status.Status } + for ; err != nil; err = errors.Unwrap(err) { + gs, ok := err.(grpcStatuser) + if !ok { + continue + } + if st := gs.GRPCStatus(); st != nil && st.Code() == codes.Unknown { + return st.Message(), true + } + } + return "", false } // containsGRPCCode walks the error chain looking for a gRPC status with the From a6ea39e7d7b4efe06931eef21b4d349682e87377 Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 18 Aug 2026 10:29:10 +0300 Subject: [PATCH 267/320] Let a command run outside a project again Narrowing Unknown to the two environment sentinels broke the standalone case. Outside a project azd answers GetCurrent with azdcontext.ErrNoProject -- "no project exists; to create a new project, run `azd init`" -- as another plain error, so it arrives as Unknown too. It stopped reading as absence, the cascade reported it instead of carrying on, and FOUNDRY_PROJECT_ENDPOINT on the host became unreachable outside a project. The atomic commands exist to work that way. It is the third sentinel now, with the case in the table. The walk over the error chain is azdext.GRPCStatusFromError, which was already there in a package this file imports. It traverses errors.Join, avoids a type assertion on an error under errorlint, and returns the status the daemon sent rather than the flattened text -- the property the hand-rolled version was written for, without the second implementation to keep in step. The duplicate predicate in context.go stays. It looked like the same question and is not: the cascade asks whether it may carry on to the next level, where an unreachable daemon means yes, and confirmedNoAzdEnvironment asks whether the reason is that no environment is selected, where an unreachable daemon means no -- it is not an answer about environments at all. Merging them told anyone whose azd hiccupped to create an environment they already have. The comment now says which question each one answers. --- .../internal/cmd/context.go | 7 +++ .../foundry/projectctx/hosted_absence_test.go | 5 ++ .../internal/foundry/projectctx/resolver.go | 63 ++++++++----------- 3 files changed, 37 insertions(+), 38 deletions(-) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/context.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/context.go index 3c0b7060aa0..ddd72785ac3 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/context.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/context.go @@ -239,6 +239,13 @@ func (ec *evalContext) confirmedNoAzdEnvironment(ctx context.Context) bool { // already have. Matching on text because that is what survives the trip -- the // sentinel is wrapped in a gRPC status on the way out of azd, so errors.Is has // nothing to compare against on this side. +// +// Deliberately not projectctx.AbsentFromAzd, which answers a different +// question. That one asks whether the cascade may carry on to the next level, +// and an unreachable daemon means yes. This one asks whether the reason is that +// no environment is selected, and an unreachable daemon means no -- it is not +// an answer about environments at all. The two agree on everything except that, +// and that is the case each exists for. func isNoDefaultEnvironmentError(err error) bool { if err == nil { return false diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/foundry/projectctx/hosted_absence_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/foundry/projectctx/hosted_absence_test.go index 7d5d7fd16c0..7220ab1496f 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/foundry/projectctx/hosted_absence_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/foundry/projectctx/hosted_absence_test.go @@ -24,6 +24,11 @@ func TestUnansweredHostedSourcesLetTheCascadeCarryOn(t *testing.T) { "nothing under that key": status.Error(codes.NotFound, "key not found"), "no default environment": status.Error(codes.Unknown, "default environment not found"), "no such environment": status.Error(codes.Unknown, "'dev': environment not found"), + // The atomic commands are meant to work standalone against the data + // plane with FOUNDRY_PROJECT_ENDPOINT exported, so running outside a + // project has to reach the host variable rather than stop here. + "outside a project": status.Error(codes.Unknown, + "no project exists; to create a new project, run `azd init`"), "wrapped in context": fmt.Errorf("reading the environment: %w", status.Error(codes.Unknown, "default environment not found")), } { diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/foundry/projectctx/resolver.go b/cli/azd/extensions/azure.ai.evaluations/internal/foundry/projectctx/resolver.go index fd0f9b1e2da..5d17dcb16d9 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/foundry/projectctx/resolver.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/foundry/projectctx/resolver.go @@ -79,27 +79,28 @@ func readAzdHostedSources(ctx context.Context) (AzdHostedSources, error) { return out, nil } -// azd's two absence sentinels, as they reach us. +// azd's absence sentinels, as they reach us. // -// `pkg/environment` declares them with errors.New, and the daemon's -// error-wrapping interceptor passes an error carrying no suggestion and no auth -// failure through untouched, so both arrive as Unknown -- the same code a -// failure to load project state arrives under. The message is the only thing -// left to tell them apart. +// `pkg/environment` and `pkg/environment/azdcontext` declare these with +// errors.New, and the daemon's error-wrapping interceptor passes an error +// carrying no suggestion and no auth failure through untouched, so all three +// arrive as Unknown -- the same code a failure to load project state arrives +// under. The message is the only thing left to tell them apart. // // Matched whole rather than by substring. The message is the only evidence // there is, so a failure whose prose happens to mention an environment must not -// read as one of these. The default-environment sentinel arrives on its own; -// the named one arrives from the data store as `'': environment not -// found`. +// read as one of these. The default-environment and no-project sentinels arrive +// on their own; the named-environment one arrives from the data store as +// `'': environment not found`. // // Matched rather than imported: taking a dependency on the environment manager -// for two strings costs more than it settles, and a rename fails closed. The +// for three strings costs more than it settles, and a rename fails closed. The // command would report the daemon error instead of resolving quietly to a // lower-priority endpoint, which is the direction to fail in. const ( azdNoDefaultEnvironment = "default environment not found" azdNoSuchEnvironment = "environment not found" + azdNoProject = "no project exists; to create a new project, run `azd init`" ) // hostedSourceAbsent reports whether an error from the azd daemon leaves the @@ -109,10 +110,11 @@ const ( // name -- kept as a defence, though azd's environment service does not use it // today. Unknown is the one that is not obvious: azd answers the ordinary // absences with plain Go errors that reach us with no status, and without -// letting those through, a project that simply has no environment selected -// could never reach the global config or the host variable. It is admitted only -// for the two messages above, because Unknown is equally what a failure to load -// project state arrives as. +// letting those through, a project with no environment selected -- or a command +// run outside a project at all, which the atomic commands are meant to support +// -- could never reach the global config or the host variable. It is admitted +// only for the three messages above, because Unknown is equally what a failure +// to load project state arrives as. // // Everything else is a failure to answer rather than an answer of "nothing": // an expired login, a denial, a cancellation, or a server fault. Falling @@ -122,32 +124,17 @@ func hostedSourceAbsent(err error) bool { if containsGRPCCode(err, codes.Unavailable) || containsGRPCCode(err, codes.NotFound) { return true } - msg, ok := unknownStatusMessage(err) - if !ok { + // The status the daemon sent, not the flattened text: status.FromError + // replaces a wrapped error's message with the whole of err.Error(), so the + // wrapper's own prose would take part in the comparison below. + st, ok := azdext.GRPCStatusFromError(err) + if !ok || st.Code() != codes.Unknown { return false } - return msg == azdNoDefaultEnvironment || strings.HasSuffix(msg, "': "+azdNoSuchEnvironment) -} - -// unknownStatusMessage is the message carried by the innermost Unknown status -// in the chain, if there is one. -// -// Read link by link rather than through status.FromError, which flattens: given -// a wrapped error it replaces the status message with the whole of err.Error(), -// so the wrapper's own prose would take part in the comparison above. Only what -// the daemon said should decide whether this is an absence. -func unknownStatusMessage(err error) (string, bool) { - type grpcStatuser interface{ GRPCStatus() *status.Status } - for ; err != nil; err = errors.Unwrap(err) { - gs, ok := err.(grpcStatuser) - if !ok { - continue - } - if st := gs.GRPCStatus(); st != nil && st.Code() == codes.Unknown { - return st.Message(), true - } - } - return "", false + msg := st.Message() + return msg == azdNoDefaultEnvironment || + msg == azdNoProject || + strings.HasSuffix(msg, "': "+azdNoSuchEnvironment) } // containsGRPCCode walks the error chain looking for a gRPC status with the From 203385ff52af638de2e75fbad1f5df00d720b8cf Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 18 Aug 2026 10:56:09 +0300 Subject: [PATCH 268/320] Put the rule that keeps regressing under test Twice now the guard that tells "the daemon answered nothing" from "the daemon could not answer" has been broken with the whole suite green. Nothing reached it: the absence predicate was tested in isolation, and the only seam over the function that uses it replaced the function wholesale. Deleting the guard outright left both extensions passing. The environment reads move behind a two-method interface, and a table covers the axis that matters: no environment selected, no project at all, an environment named in config whose directory is gone, no daemon, an expired login, a daemon that broke while looking, each key present or absent, and a key whose read failed. Deleting the guard now fails two of them. isNoDefaultEnvironmentError kept its own list of azd's sentinels and so had missed `no project exists` -- the third time that sentinel was handled in one place and missed in another. Running outside a project it answered "this is not about environments", and the caller told the reader to publish an eval that may already exist. It is now the cascade's own rule minus the one case the two genuinely disagree on, an unreachable daemon, so the comment's claim holds by construction and the next sentinel is added once. Its comment also pointed at a function name that does not exist. The config read used Unavailable where its two neighbours used the full rule. Identical in behaviour today, but three classifications in one function with two rules and no note is how the three come to disagree. --- .../internal/cmd/context.go | 20 +-- .../internal/cmd/no_environment_test.go | 22 ++- .../foundry/projectctx/env_source_test.go | 163 ++++++++++++++++++ .../internal/foundry/projectctx/resolver.go | 106 ++++++++---- 4 files changed, 263 insertions(+), 48 deletions(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/foundry/projectctx/env_source_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/context.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/context.go index ddd72785ac3..a7cf5b5b138 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/context.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/context.go @@ -231,26 +231,22 @@ func (ec *evalContext) confirmedNoAzdEnvironment(ctx context.Context) bool { return false } -// isNoDefaultEnvironmentError picks azd's "there is no environment" out of -// every other reason the call could have failed. +// isNoDefaultEnvironmentError picks azd's "there is no environment to record +// anything in" out of every other reason the call could have failed. // // The distinction is the whole point: a transport failure must not be reported // as a missing environment, or a gRPC hiccup tells the user to create one they -// already have. Matching on text because that is what survives the trip -- the -// sentinel is wrapped in a gRPC status on the way out of azd, so errors.Is has -// nothing to compare against on this side. +// already have. // -// Deliberately not projectctx.AbsentFromAzd, which answers a different -// question. That one asks whether the cascade may carry on to the next level, -// and an unreachable daemon means yes. This one asks whether the reason is that -// no environment is selected, and an unreachable daemon means no -- it is not -// an answer about environments at all. The two agree on everything except that, -// and that is the case each exists for. +// Written as the cascade's own rule minus the one case the two disagree on, +// rather than as a second list of azd's sentinels. Keeping a second list is how +// `no project exists` came to be handled in the cascade and missed here, which +// told anyone running outside a project to publish an eval that already exists. func isNoDefaultEnvironmentError(err error) bool { if err == nil { return false } - return strings.Contains(strings.ToLower(err.Error()), azdNoDefaultEnvironment) + return projectctx.HostedSourceAbsent(err) && !projectctx.DaemonUnreachable(err) } // getEnvValue reads a value from the active azd environment, returning empty diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/no_environment_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/no_environment_test.go index 7944b3e3081..e4e7b43dc0f 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/no_environment_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/no_environment_test.go @@ -4,7 +4,6 @@ package cmd import ( - "errors" "fmt" "testing" @@ -25,20 +24,26 @@ import ( // every error as "no environment" would tell someone whose azd hiccupped to // create an environment they already have. // -// The sentinel is wrapped in a gRPC status on the way out of azd, so the text -// is what there is to match on. These are the real shapes. +// The only caller reads these off a gRPC call, so every error carries a status. +// These are the shapes that reach it. func TestNoDefaultEnvironmentIsToldApartFromAFailureToAsk(t *testing.T) { // The text of azd's environment.ErrDefaultEnvironmentNotFound. const azdText = "default environment not found" - assert.True(t, isNoDefaultEnvironmentError(errors.New(azdText)), - "the sentinel's own text") assert.True(t, isNoDefaultEnvironmentError(status.Error(codes.Unknown, azdText)), - "and the same thing after azd wraps it in a gRPC status") + "the sentinel, as azd wraps it in a gRPC status") assert.True(t, - isNoDefaultEnvironmentError(fmt.Errorf("getting environment: %w", errors.New(azdText))), + isNoDefaultEnvironmentError(fmt.Errorf("getting environment: %w", + status.Error(codes.Unknown, azdText))), "and wrapped again by a caller") + // Outside a project there is nowhere an id could have been recorded, which + // is the same answer for this caller. Missing it told anyone running + // standalone to publish an eval that may already exist. + assert.True(t, + isNoDefaultEnvironmentError(status.Error(codes.Unknown, + "no project exists; to create a new project, run `azd init`")), + "outside a project there is no environment either") assert.False(t, isNoDefaultEnvironmentError(status.Error(codes.Unavailable, "connection refused")), @@ -46,5 +51,8 @@ func TestNoDefaultEnvironmentIsToldApartFromAFailureToAsk(t *testing.T) { assert.False(t, isNoDefaultEnvironmentError(status.Error(codes.DeadlineExceeded, "context deadline exceeded")), "nor is a timeout") + assert.False(t, + isNoDefaultEnvironmentError(status.Error(codes.Unknown, "loading project state: permission denied")), + "nor is a daemon that broke while looking") assert.False(t, isNoDefaultEnvironmentError(nil), "nor is success") } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/foundry/projectctx/env_source_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/foundry/projectctx/env_source_test.go new file mode 100644 index 00000000000..1be5ee0deb2 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/foundry/projectctx/env_source_test.go @@ -0,0 +1,163 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package projectctx + +import ( + "context" + "testing" + + "github.com/azure/azure-dev/cli/azd/pkg/azdext" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +// fakeEnv answers the two reads with whatever the case under test needs. +type fakeEnv struct { + current *azdext.EnvironmentResponse + currentErr error + values map[string]string + valueErr map[string]error + asked []string +} + +func (f *fakeEnv) GetCurrent( + context.Context, *azdext.EmptyRequest, ...grpc.CallOption, +) (*azdext.EnvironmentResponse, error) { + return f.current, f.currentErr +} + +func (f *fakeEnv) GetValue( + _ context.Context, req *azdext.GetEnvRequest, _ ...grpc.CallOption, +) (*azdext.KeyValueResponse, error) { + f.asked = append(f.asked, req.Key) + if err, ok := f.valueErr[req.Key]; ok { + return nil, err + } + return &azdext.KeyValueResponse{Key: req.Key, Value: f.values[req.Key]}, nil +} + +func envNamed(name string) *azdext.EnvironmentResponse { + return &azdext.EnvironmentResponse{Environment: &azdext.Environment{Name: name}} +} + +// An answer of "nothing here" leaves the cascade free to carry on; a failure to +// answer has to stop it, because carrying on resolves to a lower-priority +// endpoint that can belong to a different project. +// +// This is the rule that has regressed twice while every test passed, because +// the only seam was the whole function. +func TestReadEnvHostedSource_TellsAbsenceApartFromFailure(t *testing.T) { + cases := []struct { + name string + env *fakeEnv + wantValue string + wantName string + wantErr string + }{ + { + name: "no environment selected", + env: &fakeEnv{currentErr: status.Error(codes.Unknown, + "default environment not found")}, + }, + { + name: "outside a project altogether", + env: &fakeEnv{currentErr: status.Error(codes.Unknown, + "no project exists; to create a new project, run `azd init`")}, + }, + { + name: "the environment named in config is gone", + env: &fakeEnv{currentErr: status.Error(codes.Unknown, "'dev': environment not found")}, + }, + { + name: "no daemon", + env: &fakeEnv{currentErr: status.Error(codes.Unavailable, "connection refused")}, + }, + { + name: "the login has expired", + env: &fakeEnv{currentErr: status.Error(codes.Unauthenticated, "expired")}, + wantErr: "expired", + }, + { + name: "the daemon broke while looking", + env: &fakeEnv{currentErr: status.Error(codes.Unknown, + "loading project state: permission denied")}, + wantErr: "loading project state", + }, + { + name: "the foundry key answers", + env: &fakeEnv{current: envNamed("dev"), values: map[string]string{foundryEnvKey: "https://a"}}, + wantValue: "https://a", + wantName: "dev", + }, + { + // The key `azd ai agent init` and `azd add` persist, read only + // when the newer one has nothing. + name: "the older key answers when the newer one is empty", + env: &fakeEnv{ + current: envNamed("dev"), + values: map[string]string{foundryEnvKey: "", azureAiEnvKey: "https://b"}, + }, + wantValue: "https://b", + wantName: "dev", + }, + { + name: "neither key is set", + env: &fakeEnv{current: envNamed("dev")}, + }, + { + // A key that is simply absent must not stop the second one being + // tried, nor the levels below. + name: "the first key is absent", + env: &fakeEnv{ + current: envNamed("dev"), + values: map[string]string{azureAiEnvKey: "https://b"}, + valueErr: map[string]error{foundryEnvKey: status.Error(codes.NotFound, "no such key")}, + }, + wantValue: "https://b", + wantName: "dev", + }, + { + name: "reading a key failed", + env: &fakeEnv{ + current: envNamed("dev"), + valueErr: map[string]error{foundryEnvKey: status.Error(codes.Internal, "boom")}, + }, + wantErr: "boom", + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + value, name, err := readEnvHostedSource(context.Background(), tc.env) + + if tc.wantErr != "" { + require.Error(t, err, "a failure to answer must stop the cascade") + assert.Contains(t, err.Error(), tc.wantErr) + return + } + require.NoError(t, err, "an answer of nothing must let the cascade carry on") + assert.Equal(t, tc.wantValue, value) + assert.Equal(t, tc.wantName, name) + }) + } +} + +// The newer key wins, so a project carrying both does not silently prefer the +// one an older command wrote. +func TestReadEnvHostedSource_PrefersTheNewerKey(t *testing.T) { + env := &fakeEnv{ + current: envNamed("dev"), + values: map[string]string{foundryEnvKey: "https://new", azureAiEnvKey: "https://old"}, + } + + value, _, err := readEnvHostedSource(context.Background(), env) + + require.NoError(t, err) + assert.Equal(t, "https://new", value) + assert.Equal(t, []string{foundryEnvKey}, env.asked, + "the older key is not even read once the newer one answers") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/foundry/projectctx/resolver.go b/cli/azd/extensions/azure.ai.evaluations/internal/foundry/projectctx/resolver.go index 5d17dcb16d9..46d392b7041 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/foundry/projectctx/resolver.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/foundry/projectctx/resolver.go @@ -10,6 +10,7 @@ import ( "strings" "github.com/azure/azure-dev/cli/azd/pkg/azdext" + "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" ) @@ -35,40 +36,18 @@ func readAzdHostedSources(ctx context.Context) (AzdHostedSources, error) { } defer azdClient.Close() - envResp, envErr := azdClient.Environment().GetCurrent(ctx, &azdext.EmptyRequest{}) - if envErr != nil && !hostedSourceAbsent(envErr) { - // The daemon answered, but not with "there is no current environment". - // Reading that as absence falls through to global config or the host - // variable, which can point at a different project -- and the command - // would then land there without anything having said so. + envValue, envName, envErr := readEnvHostedSource(ctx, azdClient.Environment()) + if envErr != nil { return out, envErr } - if envErr == nil && envResp.GetEnvironment() != nil { - for _, key := range []string{foundryEnvKey, azureAiEnvKey} { - envVal, valErr := azdClient.Environment().GetValue(ctx, &azdext.GetEnvRequest{ - EnvName: envResp.Environment.Name, - Key: key, - }) - if valErr != nil { - if !hostedSourceAbsent(valErr) { - return out, valErr - } - continue - } - if envVal.GetValue() != "" { - out.EnvValue = envVal.Value - out.EnvName = envResp.Environment.Name - break - } - } - } + out.EnvValue, out.EnvName = envValue, envName state, found, cfgErr := getProjectContext(ctx, azdClient) if cfgErr != nil { - // A gRPC Unavailable code means the azd daemon is not reachable; - // treat it the same as azdClient creation failing and fall through. - // Any other error (e.g. parse failure) is a hard error. - if !containsGRPCCode(cfgErr, codes.Unavailable) { + // The same rule the environment reads use. Today the config service can + // only fail here by being unreachable, but stating it differently in + // one of three places is how the three come to disagree. + if !hostedSourceAbsent(cfgErr) { return out, cfgErr } } else { @@ -79,6 +58,54 @@ func readAzdHostedSources(ctx context.Context) (AzdHostedSources, error) { return out, nil } +// envSource is the slice of azd's environment service this file reads. +// +// Narrowed to an interface so the classification below can be tested. The rule +// it applies -- carry on when the daemon answered "nothing", stop when it +// failed to answer -- has regressed twice while every test passed, because the +// only seam was the whole function. +type envSource interface { + GetCurrent(context.Context, *azdext.EmptyRequest, ...grpc.CallOption) (*azdext.EnvironmentResponse, error) + GetValue(context.Context, *azdext.GetEnvRequest, ...grpc.CallOption) (*azdext.KeyValueResponse, error) +} + +// readEnvHostedSource reads the active environment's project endpoint. +// +// Returns an empty value and no error when there is nothing to read: no +// environment selected, no project at all, or neither key set. An error means +// the daemon failed to answer, which the caller must not read as absence -- +// falling through would resolve to a lower-priority endpoint that can belong to +// a different project. +func readEnvHostedSource(ctx context.Context, env envSource) (value, name string, err error) { + envResp, envErr := env.GetCurrent(ctx, &azdext.EmptyRequest{}) + if envErr != nil { + if !hostedSourceAbsent(envErr) { + return "", "", envErr + } + return "", "", nil + } + if envResp.GetEnvironment() == nil { + return "", "", nil + } + + for _, key := range []string{foundryEnvKey, azureAiEnvKey} { + envVal, valErr := env.GetValue(ctx, &azdext.GetEnvRequest{ + EnvName: envResp.Environment.Name, + Key: key, + }) + if valErr != nil { + if !hostedSourceAbsent(valErr) { + return "", "", valErr + } + continue + } + if envVal.GetValue() != "" { + return envVal.Value, envResp.Environment.Name, nil + } + } + return "", "", nil +} + // azd's absence sentinels, as they reach us. // // `pkg/environment` and `pkg/environment/azdcontext` declare these with @@ -103,6 +130,27 @@ const ( azdNoProject = "no project exists; to create a new project, run `azd init`" ) +// HostedSourceAbsent reports whether an error from the azd daemon is an answer +// of "nothing here" rather than a failure to answer. +// +// Exported because more than the cascade has to ask it. Deriving the set of +// azd's absences a second time elsewhere is how a sentinel comes to be handled +// in one place and missed in another, which has happened three times. +func HostedSourceAbsent(err error) bool { + return hostedSourceAbsent(err) +} + +// DaemonUnreachable reports the one absence that is not an answer about +// anything: there was nobody to ask. +// +// The cascade carries on regardless -- an unreachable daemon has no endpoint to +// offer, so the next level should be consulted. A caller reporting *why* a +// value is missing has to tell it apart, or a gRPC hiccup ends up phrased as a +// fact about the project. +func DaemonUnreachable(err error) bool { + return containsGRPCCode(err, codes.Unavailable) +} + // hostedSourceAbsent reports whether an error from the azd daemon leaves the // cascade free to carry on to the next level. // From 85cf0dff7e90bd7d4e5f0bf764331fa2e574eeaa Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 18 Aug 2026 11:03:21 +0300 Subject: [PATCH 269/320] Take the lint gates at their word The constant lost its last reader when the predicate moved, and `unused` says so. cspell reads US English and does not carry "defence". Both were caught by CI rather than by the local gate, which runs gofmt, vet and the tests but not golangci-lint or cspell. --- .../azure.ai.evaluations/internal/cmd/context.go | 9 --------- .../internal/cmd/evaluator_version_live_test.go | 2 +- .../internal/foundry/projectctx/resolver.go | 2 +- 3 files changed, 2 insertions(+), 11 deletions(-) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/context.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/context.go index a7cf5b5b138..09d4ae0c30a 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/context.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/context.go @@ -192,15 +192,6 @@ func (ec *evalContext) setEnvValue(ctx context.Context, key, value string) error return nil } -// azdNoDefaultEnvironment is what azd's environment service answers with when -// the project has no environment selected. It arrives over gRPC as a status -// whose message carries the text, so the text is what there is to match on. -// -// azd returns this as an ERROR rather than an empty answer, which is the whole -// difficulty: "there is no environment" and "azd could not be reached" are both -// non-nil errors, and only the first is something to tell the user about. -const azdNoDefaultEnvironment = "default environment not found" - // confirmedNoAzdEnvironment reports that azd answered, and the answer was that // there is no current environment. // diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator_version_live_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator_version_live_test.go index fabc4f222b6..cecbf0a7129 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator_version_live_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator_version_live_test.go @@ -28,7 +28,7 @@ import ( // For a few seconds after a publish the service can answer the next one with // the version it just assigned, writing over it rather than adding one. // Nothing observable marks the end of that race — the version listing lags a -// publish as well, answering 404 immediately after a create — so the defence +// publish as well, answering 404 immediately after a create — so the guard // is the document the caller already read: it says which version exists and // when it was written. // diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/foundry/projectctx/resolver.go b/cli/azd/extensions/azure.ai.evaluations/internal/foundry/projectctx/resolver.go index 46d392b7041..f380ee1ee7b 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/foundry/projectctx/resolver.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/foundry/projectctx/resolver.go @@ -155,7 +155,7 @@ func DaemonUnreachable(err error) bool { // cascade free to carry on to the next level. // // Unavailable is no daemon at all. NotFound is a daemon with nothing under that -// name -- kept as a defence, though azd's environment service does not use it +// name -- kept as a guard, though azd's environment service does not use it // today. Unknown is the one that is not obvious: azd answers the ordinary // absences with plain Go errors that reach us with no status, and without // letting those through, a project with no environment selected -- or a command From ebd3aa1407e301df10672461ebae7d5cc8db4a98 Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 18 Aug 2026 11:53:42 +0300 Subject: [PATCH 270/320] Find the configuration where init put it, in every command that reads it `init --path ./quality` records EVAL_CONFIG_PATH and scaffolds there. The cascade that reads it back -- --path, then the recorded path, then ./evals -- was documented on evalDir and applied only by `run`. So in a project scaffolded outside ./evals: - `azd ai eval create` reported "no eval configuration at evals/azure.eval.yaml; run `azd ai eval init` to scaffold one", in a project that had just been scaffolded. - `azd ai eval generate` submitted a billed generation job and wrote a second configuration under ./evals, splitting the project in two. The rubric it produced was referenced by a file nothing else read. - `run start` then reported the eval "has not been deployed ... run `azd ai eval create` first", and `create` failed. The two errors pointed at each other. - `init`'s own next step, `azd ai eval create`, was one of the commands that failed -- printed by the command that had just recorded where it wrote. The rule is now one function, evalDirCascade, with two ways in: the method for commands holding an azd connection, and resolveEvalDir for the ones that have not built one yet. create, generate and init take the second, which keeps a missing configuration reported before a missing endpoint. The flag default is the load-bearing part. Level 1 only yields on an empty value, so a --path defaulting to "evals" opts the command out of the cascade without saying so; that is what all three were doing. init's printed steps now name the directory when it is not the default. EVAL_CONFIG_PATH would usually cover it, but recording it is best effort and init works without an azd environment, so the flag is what makes the line run as printed either way. `azd up` is left alone -- it reads azure.yaml, which $refs the configuration already. Verified by planting each defect in turn: all six make the intended test fail, and the whole flow was re-run against a live project. --- .../azure.ai.evaluations/CHANGELOG.md | 2 +- .../azure.ai.evaluations/extension.yaml | 2 +- .../internal/cmd/context.go | 60 ++++- .../internal/cmd/eval_dir_test.go | 223 ++++++++++++++++++ .../internal/cmd/eval_group.go | 7 +- .../internal/cmd/generate_commands.go | 5 +- .../internal/cmd/generate_composite.go | 5 + .../azure.ai.evaluations/internal/cmd/init.go | 57 ++++- .../azure.ai.evaluations/version.txt | 2 +- 9 files changed, 338 insertions(+), 25 deletions(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/eval_dir_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/CHANGELOG.md b/cli/azd/extensions/azure.ai.evaluations/CHANGELOG.md index eeab4b11adf..c171a93b5aa 100644 --- a/cli/azd/extensions/azure.ai.evaluations/CHANGELOG.md +++ b/cli/azd/extensions/azure.ai.evaluations/CHANGELOG.md @@ -1,6 +1,6 @@ # Release History -## 1.0.7-beta (Unreleased) +## 1.0.8-beta (Unreleased) First release of the Foundry evaluations extension. diff --git a/cli/azd/extensions/azure.ai.evaluations/extension.yaml b/cli/azd/extensions/azure.ai.evaluations/extension.yaml index da14c489457..9f944fd8bc0 100644 --- a/cli/azd/extensions/azure.ai.evaluations/extension.yaml +++ b/cli/azd/extensions/azure.ai.evaluations/extension.yaml @@ -5,7 +5,7 @@ displayName: Foundry evaluations (Beta) description: Define and run Foundry evaluations from your terminal. (Beta) usage: azd ai eval [options] # NOTE: Make sure version.txt is in sync with this version. -version: 1.0.7-beta +version: 1.0.8-beta requiredAzdVersion: ">=1.27.1" language: go capabilities: diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/context.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/context.go index 09d4ae0c30a..2c928a80bfd 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/context.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/context.go @@ -302,11 +302,16 @@ func (ec *evalContext) azdProject(ctx context.Context) (*azdext.ProjectConfig, e // but an endpoint. func deployCommandName(proj *azdext.ProjectConfig) string { if projectCanProvision(proj) { - return "azd up" + return azdUpCommand } return "azd ai eval create" } +// azdUpCommand provisions before it deploys. It is named rather than repeated +// because callers have to be able to tell it apart from this extension's own +// commands -- it takes none of their flags. +const azdUpCommand = "azd up" + // appInsightsEnvKey is where a connected Application Insights resource lands in // the azd environment. azd's own provisioning writes it, and the agents // extension reads the same key to pass tracing configuration to a running @@ -397,12 +402,59 @@ const ( // later command. Without it, `init --path ./quality` wrote a configuration that // `run` then looked for under ./evals and reported as missing -- while // azure.yaml's $ref pointed at it correctly the whole time. -func (ec *evalContext) evalDir(ctx context.Context, flagValue string) string { +// +// This is the whole rule, and every command that reads the configuration goes +// through it. Stating it here and applying it on only some paths is how +// `create` came to report the configuration missing and `generate` came to +// write a second one under ./evals, both in a project where init had recorded +// where it put the first. +func evalDirCascade(flagValue string, recorded func() string) string { if flagValue != "" { return flagValue } - if recorded := ec.getEnvValue(ctx, envKeyEvalPath); recorded != "" { - return recorded + if path := recorded(); path != "" { + return path } return project.DefaultEvalDir } + +// evalDir is the cascade for a command that already holds an azd connection. +func (ec *evalContext) evalDir(ctx context.Context, flagValue string) string { + return evalDirCascade(flagValue, func() string { + return ec.getEnvValue(ctx, envKeyEvalPath) + }) +} + +// resolveEvalDir is the cascade for a command that has not built an +// evalContext yet. +// +// `create`, `generate` and `init` all have to find the configuration before +// they resolve a Foundry endpoint, so that a project with no configuration is +// told to run `init` rather than told to set an endpoint. The extra azd +// connection is local and short-lived, and is what buys that ordering. +func resolveEvalDir(ctx context.Context, flagValue string) string { + return evalDirCascade(flagValue, func() string { return recordedEvalPath(ctx) }) +} + +// recordedEvalPath reads back what recordEvalPath wrote, or empty when there is +// no azd environment to read it from. +func recordedEvalPath(ctx context.Context) string { + azdClient, err := azdext.NewAzdClient() + if err != nil { + return "" + } + defer azdClient.Close() + + env, err := azdClient.Environment().GetCurrent(ctx, &azdext.EmptyRequest{}) + if err != nil || env.GetEnvironment() == nil { + return "" + } + val, err := azdClient.Environment().GetValue(ctx, &azdext.GetEnvRequest{ + EnvName: env.GetEnvironment().GetName(), + Key: envKeyEvalPath, + }) + if err != nil || val == nil { + return "" + } + return val.Value +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/eval_dir_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/eval_dir_test.go new file mode 100644 index 00000000000..9d23f82b250 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/eval_dir_test.go @@ -0,0 +1,223 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "strings" + "testing" + + "azureaieval/internal/project" + + "github.com/spf13/cobra" + "github.com/spf13/pflag" + "github.com/stretchr/testify/assert" +) + +// Where the configuration lives is settled by one cascade -- --path, then the +// path `init` recorded in the azd environment, then ./evals -- and these tests +// pin the two halves of it that have been got wrong. +// +// The bug they were written for: `init --path ./quality` recorded +// EVAL_CONFIG_PATH and scaffolded there, and then `azd ai eval create` reported +// "no eval configuration at evals/azure.eval.yaml" while `azd ai eval generate` +// silently billed a generation job and wrote a *second* configuration under +// ./evals that nothing else read. `run` had the cascade; the commands that +// write did not. + +// A --path that defaults to "evals" can never reach level 2, because level 1 +// only yields when the flag is empty. So the default is the invariant: a +// command that fills it in has opted out of the cascade without saying so, and +// that is exactly how create, generate and init came to skip it. +func TestPathFlagsLeaveRoomForTheRecordedPath(t *testing.T) { + root := NewRootCommand() + + var checked int + walk(t, root, nil, func(name string, cmd *cobra.Command) { + f := cmd.Flags().Lookup("path") + if f == nil { + return + } + checked++ + assert.Empty(t, f.DefValue, + "`azd ai %s --path` defaults to %q, so the path `init` recorded can "+ + "never be reached: level 1 of the cascade only yields on an empty value", + name, f.DefValue) + }) + + // If --path is ever renamed, the loop above passes by visiting nothing. + assert.GreaterOrEqual(t, checked, 3, + "expected --path on at least init, generate and eval create; found %d", checked) +} + +func TestEvalDirCascadeAnswersInOrder(t *testing.T) { + cases := []struct { + name string + flag string + recorded string + want string + }{ + { + name: "the flag wins", + flag: "./given", + recorded: "./recorded", + want: "./given", + }, + { + name: "the flag wins even over nothing recorded", + flag: "./given", + recorded: "", + want: "./given", + }, + { + name: "what init recorded is used when no flag was given", + flag: "", + recorded: "./quality", + want: "./quality", + }, + { + name: "the default is the last resort", + flag: "", + recorded: "", + want: project.DefaultEvalDir, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got := evalDirCascade(tc.flag, func() string { return tc.recorded }) + assert.Equal(t, tc.want, got) + }) + } +} + +// The azd environment is not always readable -- `init` works without one -- so +// level 2 can come back empty for a reason that is not "nothing was recorded". +// Falling through to the default is right; asking twice is not, because each +// call is a round trip. +func TestEvalDirCascadeAsksForTheRecordedPathOnce(t *testing.T) { + var asked int + got := evalDirCascade("", func() string { + asked++ + return "" + }) + + assert.Equal(t, project.DefaultEvalDir, got) + assert.Equal(t, 1, asked, "the recorded path should be read exactly once") + + asked = 0 + evalDirCascade("./given", func() string { + asked++ + return "" + }) + assert.Equal(t, 0, asked, "a --path that was given should not cost a round trip") +} + +// `init` prints the commands to run next, and the claim those lines make is +// that they run as printed. A scaffold written somewhere other than ./evals is +// only reachable by a command that names it, because EVAL_CONFIG_PATH is +// recorded best effort and `init` succeeds without an azd environment to record +// it in. +func TestNextStepsRunAsPrinted(t *testing.T) { + cases := []struct { + name string + evalDir string + deployCmd string + wantPath bool + }{ + { + name: "a scaffold outside ./evals names itself", + evalDir: "./quality", + deployCmd: "azd ai eval create", + wantPath: true, + }, + { + name: "the default directory needs no flag", + evalDir: project.DefaultEvalDir, + deployCmd: "azd ai eval create", + wantPath: false, + }, + { + name: "an unrecorded directory needs no flag", + evalDir: "", + deployCmd: "azd ai eval create", + wantPath: false, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + s := scaffold{ + eval: &project.Eval{Name: "an-eval"}, + evalDir: tc.evalDir, + } + for _, step := range s.nextSteps(tc.deployCmd) { + assert.Equal(t, tc.wantPath, strings.Contains(step, "--path "), + "step %q", step) + } + }) + } +} + +// `azd up` provisions and then deploys, reading azure.yaml -- which already +// $refs the configuration wherever it was written. It takes none of this +// extension's flags, so handing it --path prints a step that fails. +func TestNextStepsNeverFlagAzdUp(t *testing.T) { + s := scaffold{ + eval: &project.Eval{Name: "an-eval"}, + evalDir: "./quality", + } + + steps := s.nextSteps(azdUpCommand) + assert.Contains(t, steps, azdUpCommand, + "`azd up` should be suggested exactly as it is run") + for _, step := range steps { + if strings.HasPrefix(step, azdUpCommand) { + assert.NotContains(t, step, "--path", "step %q", step) + } + } +} + +// A generated next step already carried --target and --generation-model for the +// same reason. --path joins them. +func TestGenerateStepNamesTheScaffoldedDirectory(t *testing.T) { + s := scaffold{ + eval: &project.Eval{Name: "an-eval"}, + evalDir: "./quality", + target: "support-agent", + judgeModel: "gpt-4.1-nano", + rubricName: "support-agent-quality", + datasetName: "support-agent-dataset", + generateDataset: true, + generateRubric: true, + } + + steps := s.nextSteps("azd ai eval create") + if assert.Len(t, steps, 1) { + for _, want := range []string{ + "--target support-agent", + "--generation-model gpt-4.1-nano", + "--path ./quality", + } { + assert.Contains(t, steps[0], want) + } + } +} + +// Guards against the loop above passing because pflag stopped reporting +// defaults the way this test reads them. +func TestPathFlagIsStillCalledPath(t *testing.T) { + var names []string + walk(t, NewRootCommand(), nil, func(name string, cmd *cobra.Command) { + cmd.LocalFlags().VisitAll(func(f *pflag.Flag) { + if f.Name == "path" { + names = append(names, name) + } + }) + }) + + assert.Contains(t, names, "init") + assert.Contains(t, names, "generate") + assert.Contains(t, names, "create") + assert.Contains(t, names, "run start") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/eval_group.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/eval_group.go index 66750890f43..312614192b9 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/eval_group.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/eval_group.go @@ -42,7 +42,7 @@ func newEvalCreateCommand() *cobra.Command { path := fromFile if path == "" { - path = project.ResolveEvalConfigPath(evalDir) + path = project.ResolveEvalConfigPath(resolveEvalDir(ctx, evalDir)) } cfg, err := project.LoadEvalConfig(path) if err != nil { @@ -141,8 +141,9 @@ func newEvalCreateCommand() *cobra.Command { cmd.Flags().StringVar(&fromFile, "from-file", "", "Read the configuration from this path instead of the eval directory.") - cmd.Flags().StringVar(&evalDir, "path", project.DefaultEvalDir, - "Directory holding the evaluation configuration.") + cmd.Flags().StringVar(&evalDir, "path", "", + "Directory holding the evaluation configuration. Defaults to the directory "+ + "`init` scaffolded, otherwise ./evals.") cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") return cmd } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_commands.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_commands.go index cda84a10255..a0246c5e955 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_commands.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_commands.go @@ -42,8 +42,9 @@ type generateFlags struct { } func addGenerateFlags(cmd *cobra.Command, f *generateFlags) { - cmd.Flags().StringVar(&f.path, "path", project.DefaultEvalDir, - "Directory holding the evaluation configuration.") + cmd.Flags().StringVar(&f.path, "path", "", + "Directory holding the evaluation configuration. Defaults to the directory "+ + "`init` scaffolded, otherwise ./evals.") cmd.Flags().StringVar(&f.target, "target", "", "Agent whose context seeds generation.") cmd.Flags().StringVar(&f.instruction, "agent-instruction", "", "What the agent does and what to test.") diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_composite.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_composite.go index 717b1a784ee..a0e711c5a33 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_composite.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_composite.go @@ -79,6 +79,11 @@ func newGenerateCommand() *cobra.Command { } } + // Settled before anything reads or writes the configuration, so the + // catalog entry lands next to the eval `init` scaffolded rather than + // in a second configuration under ./evals that nothing else reads. + flags.path = resolveEvalDir(cmd.Context(), flags.path) + target := firstNonEmpty(flags.target, declaredTarget(flags.path)) plans, err := buildGeneratePlans(generateRequest{ flags: &flags, diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go index 08b4e364c0a..67d6f8eda3b 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go @@ -93,9 +93,12 @@ func newInitCommand() *cobra.Command { source = initSourceDataset } } - if path == "" { - path = project.DefaultEvalDir - } + // The same cascade every other command reads the configuration + // through. init merges into the configuration it finds, so a second + // `init` in a project scaffolded at ./quality has to find that one -- + // otherwise it writes a second configuration under ./evals and + // declares a second service pointing at it. + path = resolveEvalDir(cmd.Context(), path) // Asked before anything is written: the project is the one thing // init cannot supply for itself, and failing after creating @@ -290,8 +293,9 @@ func newInitCommand() *cobra.Command { "built-in. Passing this replaces the defaults, so it also opts out of rubric generation.") cmd.Flags().StringVar(&judgeModel, "judge-model", "", "Model deployment the graders judge with. Detected from the project when omitted.") - cmd.Flags().StringVar(&path, "path", project.DefaultEvalDir, - "Directory to write the configuration into. Used verbatim, never re-rooted.") + cmd.Flags().StringVar(&path, "path", "", + "Directory to write the configuration into. Used verbatim, never re-rooted. "+ + "Defaults to the directory an earlier `init` scaffolded, otherwise ./evals.") cmd.Flags().BoolVar(&force, "force", false, "Replace an eval of the same name instead of failing.") return cmd @@ -322,11 +326,14 @@ type scaffoldInput struct { // scaffold is what `init` added, and what it should suggest doing next. type scaffold struct { - eval *project.Eval - datasetName string - rubricName string - target string - judgeModel string + eval *project.Eval + datasetName string + rubricName string + target string + judgeModel string + // evalDir is where the configuration was written, so the next steps can + // name it when it is not the default. + evalDir string generateDataset bool generateRubric bool } @@ -340,7 +347,12 @@ type scaffold struct { // rubric generation. func planScaffold(in scaffoldInput) scaffold { cfg := in.cfg - out := scaffold{rubricName: in.rubricName, target: in.target, judgeModel: in.judgeModel} + out := scaffold{ + rubricName: in.rubricName, + target: in.target, + judgeModel: in.judgeModel, + evalDir: in.evalDir, + } eval := project.Eval{ Name: in.evalName, @@ -498,11 +510,30 @@ func (s scaffold) nextSteps(deployCmd string) []string { steps = append(steps, s.generateCommand("--evaluator --evaluator-name "+s.rubricName)) } if len(steps) == 0 { - steps = append(steps, deployCmd, "azd ai eval run start") + // `azd up` reads azure.yaml, which already $refs the configuration + // wherever it was written, so it is the one step --path must not join. + deploy := deployCmd + if deploy != azdUpCommand { + deploy = s.withPath(deploy) + } + steps = append(steps, deploy, s.withPath("azd ai eval run start")) } return steps } +// withPath appends --path to a step that needs it to run where init wrote. +// +// The recorded EVAL_CONFIG_PATH would usually supply this on its own, but +// recording it is best effort -- it needs an azd environment, and `init` works +// without one. Naming the directory makes the printed step run as printed +// either way, which is the claim these lines make. +func (s scaffold) withPath(step string) string { + if s.evalDir == "" || s.evalDir == project.DefaultEvalDir { + return step + } + return step + " --path " + s.evalDir +} + // generateCommand builds a `generate` invocation that runs as printed. func (s scaffold) generateCommand(what string) string { cmd := "azd ai eval generate" @@ -515,7 +546,7 @@ func (s scaffold) generateCommand(what string) string { if s.judgeModel != "" { cmd += " --generation-model " + s.judgeModel } - return cmd + return s.withPath(cmd) } // relativeToConfig rewrites a path given relative to the working directory so diff --git a/cli/azd/extensions/azure.ai.evaluations/version.txt b/cli/azd/extensions/azure.ai.evaluations/version.txt index 33955bc06df..e00d876d2a3 100644 --- a/cli/azd/extensions/azure.ai.evaluations/version.txt +++ b/cli/azd/extensions/azure.ai.evaluations/version.txt @@ -1 +1 @@ -1.0.7-beta \ No newline at end of file +1.0.8-beta From 1037bf9fbb30bdc19a1c8daee6c8e1da4406c369 Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 18 Aug 2026 12:16:57 +0300 Subject: [PATCH 271/320] Tell a failed read of the recorded path apart from nothing being recorded Two defects in the cascade added by the previous commit, both found by review and both confirmed by running the command. The printed next step was built by concatenation, so `init --path "./team evals"` printed Next: azd ai eval create --path ./team evals which resolves ./team and reports the configuration missing. The one case the flag was added for was the case it failed in. Paths are quoted now, with double quotes because cmd, PowerShell, bash and zsh all read those the same way and the line is printed without knowing which is reading it. Backslashes are left alone: doubling them is right for bash and wrong for the two shells most likely to be reading C:\Users\Me\My Evals. Reading the recorded path collapsed every failure into the same empty string as "nothing was recorded", and the cascade then defaulted to ./evals. So a transient azd failure would have `generate` bill a job and write the second configuration all over again -- this time for a reason nobody could reproduce. The read now returns an error, and azd's own "there is no project or environment" is the only thing read as an answer, using the same rule the endpoint cascade uses rather than a second list of sentinels. evaldir_test.go asserted the empty --path default for `run start` alone. That is the same defect as the one being fixed, sitting in the test that was meant to catch it: `create`, `generate` and `init` were all filling the default in, and `create` was missing from the list of commands the file checks at all. Both tests are written over the whole command tree now, and the two files that had grown up around this rule are one. Nine planted defects, nine caught by the intended test. --- .../internal/cmd/context.go | 96 ++++-- .../internal/cmd/eval_dir_test.go | 223 ------------- .../internal/cmd/eval_group.go | 6 +- .../internal/cmd/evaldir_test.go | 315 +++++++++++++++++- .../internal/cmd/generate_composite.go | 6 +- .../azure.ai.evaluations/internal/cmd/init.go | 28 +- .../azure.ai.evaluations/internal/cmd/run.go | 5 +- .../internal/cmd/run_output.go | 5 +- 8 files changed, 410 insertions(+), 274 deletions(-) delete mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/eval_dir_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/context.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/context.go index 2c928a80bfd..5788ca21acb 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/context.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/context.go @@ -392,7 +392,7 @@ const ( envKeyEvalPath = "EVAL_CONFIG_PATH" ) -// evalDir resolves where the configuration lives: +// evalDirCascade is the one rule for where the configuration lives: // // 1. --path // 2. the path `init` recorded in the azd environment @@ -408,20 +408,33 @@ const ( // `create` came to report the configuration missing and `generate` came to // write a second one under ./evals, both in a project where init had recorded // where it put the first. -func evalDirCascade(flagValue string, recorded func() string) string { +// +// recorded tells absence apart from failure, and the two get different +// answers. A project with no azd environment has genuinely recorded nothing, +// so ./evals is right. An azd that could not be asked has said nothing at all, +// and defaulting on that would write the second configuration all over again -- +// this time for a reason nobody could reproduce. +func evalDirCascade(flagValue string, recorded func() (string, error)) (string, error) { if flagValue != "" { - return flagValue + return flagValue, nil + } + path, err := recorded() + if err != nil { + return "", err } - if path := recorded(); path != "" { - return path + if path != "" { + return path, nil } - return project.DefaultEvalDir + return project.DefaultEvalDir, nil } // evalDir is the cascade for a command that already holds an azd connection. -func (ec *evalContext) evalDir(ctx context.Context, flagValue string) string { - return evalDirCascade(flagValue, func() string { - return ec.getEnvValue(ctx, envKeyEvalPath) +func (ec *evalContext) evalDir(ctx context.Context, flagValue string) (string, error) { + return evalDirCascade(flagValue, func() (string, error) { + if ec.envName == "" || ec.azdClient == nil { + return "", nil + } + return readRecordedEvalPath(ctx, ec.azdClient, ec.envName) }) } @@ -432,29 +445,54 @@ func (ec *evalContext) evalDir(ctx context.Context, flagValue string) string { // they resolve a Foundry endpoint, so that a project with no configuration is // told to run `init` rather than told to set an endpoint. The extra azd // connection is local and short-lived, and is what buys that ordering. -func resolveEvalDir(ctx context.Context, flagValue string) string { - return evalDirCascade(flagValue, func() string { return recordedEvalPath(ctx) }) +func resolveEvalDir(ctx context.Context, flagValue string) (string, error) { + return evalDirCascade(flagValue, func() (string, error) { + azdClient, err := azdext.NewAzdClient() + if err != nil { + // Nothing to ask. This extension is spawned by azd, so the case + // that reaches here is a test or a direct invocation, neither of + // which has an environment holding a recorded path. + return "", nil + } + defer azdClient.Close() + + env, err := azdClient.Environment().GetCurrent(ctx, &azdext.EmptyRequest{}) + if err != nil { + // The same rule the endpoint cascade uses: azd saying "there is no + // project or environment" is an answer, and anything else is not. + if isNoDefaultEnvironmentError(err) { + return "", nil + } + return "", err + } + if env.GetEnvironment() == nil { + return "", nil + } + return readRecordedEvalPath(ctx, azdClient, env.GetEnvironment().GetName()) + }) } -// recordedEvalPath reads back what recordEvalPath wrote, or empty when there is -// no azd environment to read it from. -func recordedEvalPath(ctx context.Context) string { - azdClient, err := azdext.NewAzdClient() - if err != nil { - return "" - } - defer azdClient.Close() - - env, err := azdClient.Environment().GetCurrent(ctx, &azdext.EmptyRequest{}) - if err != nil || env.GetEnvironment() == nil { - return "" - } - val, err := azdClient.Environment().GetValue(ctx, &azdext.GetEnvRequest{ - EnvName: env.GetEnvironment().GetName(), +// readRecordedEvalPath reads back what recordEvalPath wrote, distinguishing a +// key that was never set from a read that failed. +func readRecordedEvalPath( + ctx context.Context, + client *azdext.AzdClient, + envName string, +) (string, error) { + val, err := client.Environment().GetValue(ctx, &azdext.GetEnvRequest{ + EnvName: envName, Key: envKeyEvalPath, }) - if err != nil || val == nil { - return "" + if err != nil { + // An unset key is an answer; init records the path best effort and + // succeeds without an azd environment to record it in. + if isNoDefaultEnvironmentError(err) { + return "", nil + } + return "", err } - return val.Value + if val == nil { + return "", nil + } + return val.Value, nil } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/eval_dir_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/eval_dir_test.go deleted file mode 100644 index 9d23f82b250..00000000000 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/eval_dir_test.go +++ /dev/null @@ -1,223 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package cmd - -import ( - "strings" - "testing" - - "azureaieval/internal/project" - - "github.com/spf13/cobra" - "github.com/spf13/pflag" - "github.com/stretchr/testify/assert" -) - -// Where the configuration lives is settled by one cascade -- --path, then the -// path `init` recorded in the azd environment, then ./evals -- and these tests -// pin the two halves of it that have been got wrong. -// -// The bug they were written for: `init --path ./quality` recorded -// EVAL_CONFIG_PATH and scaffolded there, and then `azd ai eval create` reported -// "no eval configuration at evals/azure.eval.yaml" while `azd ai eval generate` -// silently billed a generation job and wrote a *second* configuration under -// ./evals that nothing else read. `run` had the cascade; the commands that -// write did not. - -// A --path that defaults to "evals" can never reach level 2, because level 1 -// only yields when the flag is empty. So the default is the invariant: a -// command that fills it in has opted out of the cascade without saying so, and -// that is exactly how create, generate and init came to skip it. -func TestPathFlagsLeaveRoomForTheRecordedPath(t *testing.T) { - root := NewRootCommand() - - var checked int - walk(t, root, nil, func(name string, cmd *cobra.Command) { - f := cmd.Flags().Lookup("path") - if f == nil { - return - } - checked++ - assert.Empty(t, f.DefValue, - "`azd ai %s --path` defaults to %q, so the path `init` recorded can "+ - "never be reached: level 1 of the cascade only yields on an empty value", - name, f.DefValue) - }) - - // If --path is ever renamed, the loop above passes by visiting nothing. - assert.GreaterOrEqual(t, checked, 3, - "expected --path on at least init, generate and eval create; found %d", checked) -} - -func TestEvalDirCascadeAnswersInOrder(t *testing.T) { - cases := []struct { - name string - flag string - recorded string - want string - }{ - { - name: "the flag wins", - flag: "./given", - recorded: "./recorded", - want: "./given", - }, - { - name: "the flag wins even over nothing recorded", - flag: "./given", - recorded: "", - want: "./given", - }, - { - name: "what init recorded is used when no flag was given", - flag: "", - recorded: "./quality", - want: "./quality", - }, - { - name: "the default is the last resort", - flag: "", - recorded: "", - want: project.DefaultEvalDir, - }, - } - - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - got := evalDirCascade(tc.flag, func() string { return tc.recorded }) - assert.Equal(t, tc.want, got) - }) - } -} - -// The azd environment is not always readable -- `init` works without one -- so -// level 2 can come back empty for a reason that is not "nothing was recorded". -// Falling through to the default is right; asking twice is not, because each -// call is a round trip. -func TestEvalDirCascadeAsksForTheRecordedPathOnce(t *testing.T) { - var asked int - got := evalDirCascade("", func() string { - asked++ - return "" - }) - - assert.Equal(t, project.DefaultEvalDir, got) - assert.Equal(t, 1, asked, "the recorded path should be read exactly once") - - asked = 0 - evalDirCascade("./given", func() string { - asked++ - return "" - }) - assert.Equal(t, 0, asked, "a --path that was given should not cost a round trip") -} - -// `init` prints the commands to run next, and the claim those lines make is -// that they run as printed. A scaffold written somewhere other than ./evals is -// only reachable by a command that names it, because EVAL_CONFIG_PATH is -// recorded best effort and `init` succeeds without an azd environment to record -// it in. -func TestNextStepsRunAsPrinted(t *testing.T) { - cases := []struct { - name string - evalDir string - deployCmd string - wantPath bool - }{ - { - name: "a scaffold outside ./evals names itself", - evalDir: "./quality", - deployCmd: "azd ai eval create", - wantPath: true, - }, - { - name: "the default directory needs no flag", - evalDir: project.DefaultEvalDir, - deployCmd: "azd ai eval create", - wantPath: false, - }, - { - name: "an unrecorded directory needs no flag", - evalDir: "", - deployCmd: "azd ai eval create", - wantPath: false, - }, - } - - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - s := scaffold{ - eval: &project.Eval{Name: "an-eval"}, - evalDir: tc.evalDir, - } - for _, step := range s.nextSteps(tc.deployCmd) { - assert.Equal(t, tc.wantPath, strings.Contains(step, "--path "), - "step %q", step) - } - }) - } -} - -// `azd up` provisions and then deploys, reading azure.yaml -- which already -// $refs the configuration wherever it was written. It takes none of this -// extension's flags, so handing it --path prints a step that fails. -func TestNextStepsNeverFlagAzdUp(t *testing.T) { - s := scaffold{ - eval: &project.Eval{Name: "an-eval"}, - evalDir: "./quality", - } - - steps := s.nextSteps(azdUpCommand) - assert.Contains(t, steps, azdUpCommand, - "`azd up` should be suggested exactly as it is run") - for _, step := range steps { - if strings.HasPrefix(step, azdUpCommand) { - assert.NotContains(t, step, "--path", "step %q", step) - } - } -} - -// A generated next step already carried --target and --generation-model for the -// same reason. --path joins them. -func TestGenerateStepNamesTheScaffoldedDirectory(t *testing.T) { - s := scaffold{ - eval: &project.Eval{Name: "an-eval"}, - evalDir: "./quality", - target: "support-agent", - judgeModel: "gpt-4.1-nano", - rubricName: "support-agent-quality", - datasetName: "support-agent-dataset", - generateDataset: true, - generateRubric: true, - } - - steps := s.nextSteps("azd ai eval create") - if assert.Len(t, steps, 1) { - for _, want := range []string{ - "--target support-agent", - "--generation-model gpt-4.1-nano", - "--path ./quality", - } { - assert.Contains(t, steps[0], want) - } - } -} - -// Guards against the loop above passing because pflag stopped reporting -// defaults the way this test reads them. -func TestPathFlagIsStillCalledPath(t *testing.T) { - var names []string - walk(t, NewRootCommand(), nil, func(name string, cmd *cobra.Command) { - cmd.LocalFlags().VisitAll(func(f *pflag.Flag) { - if f.Name == "path" { - names = append(names, name) - } - }) - }) - - assert.Contains(t, names, "init") - assert.Contains(t, names, "generate") - assert.Contains(t, names, "create") - assert.Contains(t, names, "run start") -} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/eval_group.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/eval_group.go index 312614192b9..d75f26258e2 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/eval_group.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/eval_group.go @@ -42,7 +42,11 @@ func newEvalCreateCommand() *cobra.Command { path := fromFile if path == "" { - path = project.ResolveEvalConfigPath(resolveEvalDir(ctx, evalDir)) + dir, err := resolveEvalDir(ctx, evalDir) + if err != nil { + return err + } + path = project.ResolveEvalConfigPath(dir) } cfg, err := project.LoadEvalConfig(path) if err != nil { diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaldir_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaldir_test.go index 89c74e79077..c7527a1f73a 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaldir_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaldir_test.go @@ -5,47 +5,188 @@ package cmd import ( "context" + "errors" + "strings" "testing" "azureaieval/internal/project" "github.com/spf13/cobra" + "github.com/spf13/pflag" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) +// Where the configuration lives is settled by one cascade -- --path, then the +// path `init` recorded in the azd environment, then ./evals -- and these tests +// pin it. +// // `init --path ./quality` wrote a configuration that `run` then looked for // under ./evals and reported as missing, while azure.yaml's $ref pointed at it // correctly the whole time. The path init used is remembered so the flag does // not have to be repeated on every later command. +// +// That fix reached `run` and stopped there. In a project scaffolded outside +// ./evals, `create` went on reporting the configuration missing and `generate` +// went on submitting a billed job and writing a *second* configuration under +// ./evals that nothing else read. So these tests are written over every +// command, not over the one that was wrong at the time. + func TestEvalDirCascade(t *testing.T) { - // No azd environment: getEnvValue returns empty, so only flag and default apply. + // No azd environment: there is nothing to read, so only flag and default apply. ec := &evalContext{} - assert.Equal(t, project.DefaultEvalDir, ec.evalDir(context.Background(), ""), - "nothing given anywhere is ./evals") - assert.Equal(t, "quality", ec.evalDir(context.Background(), "quality"), - "--path wins") + dir, err := ec.evalDir(context.Background(), "") + require.NoError(t, err) + assert.Equal(t, project.DefaultEvalDir, dir, "nothing given anywhere is ./evals") + + dir, err = ec.evalDir(context.Background(), "quality") + require.NoError(t, err) + assert.Equal(t, "quality", dir, "--path wins") +} + +func TestEvalDirCascadeAnswersInOrder(t *testing.T) { + cases := []struct { + name string + flag string + recorded string + want string + }{ + { + name: "the flag wins", + flag: "./given", + recorded: "./recorded", + want: "./given", + }, + { + name: "the flag wins even over nothing recorded", + flag: "./given", + recorded: "", + want: "./given", + }, + { + name: "what init recorded is used when no flag was given", + flag: "", + recorded: "./quality", + want: "./quality", + }, + { + name: "the default is the last resort", + flag: "", + recorded: "", + want: project.DefaultEvalDir, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got, err := evalDirCascade(tc.flag, func() (string, error) { + return tc.recorded, nil + }) + require.NoError(t, err) + assert.Equal(t, tc.want, got) + }) + } +} + +// A read that failed is not a project that recorded nothing. Defaulting on it +// is how `generate` would write a second configuration under ./evals for a +// reason nobody could reproduce, so the failure has to come back out. +func TestEvalDirCascadeDoesNotDefaultOnAFailedRead(t *testing.T) { + boom := errors.New("the environment could not be read") + + got, err := evalDirCascade("", func() (string, error) { return "", boom }) + + require.ErrorIs(t, err, boom) + assert.Empty(t, got, "a failed read must not answer with the default") +} + +// A --path that was given is the answer on its own, so a broken azd cannot +// stop a caller who already said where to look. +func TestEvalDirCascadeIgnoresAFailedReadWhenPathWasGiven(t *testing.T) { + got, err := evalDirCascade("./given", func() (string, error) { + return "", errors.New("the environment could not be read") + }) + + require.NoError(t, err) + assert.Equal(t, "./given", got) +} + +// Each read is a round trip, and a --path that was given makes it unnecessary. +func TestEvalDirCascadeAsksForTheRecordedPathOnce(t *testing.T) { + var asked int + got, err := evalDirCascade("", func() (string, error) { + asked++ + return "", nil + }) + + require.NoError(t, err) + assert.Equal(t, project.DefaultEvalDir, got) + assert.Equal(t, 1, asked, "the recorded path should be read exactly once") + + asked = 0 + _, err = evalDirCascade("./given", func() (string, error) { + asked++ + return "", nil + }) + require.NoError(t, err) + assert.Equal(t, 0, asked, "a --path that was given should not cost a round trip") +} + +// --path defaults to empty, not to ./evals, so "not given" stays +// distinguishable from "given the default". A non-empty default shadows the +// path init recorded, because level 1 only yields on an empty value -- so the +// command has opted out of the cascade without saying so. +// +// This was asserted for `run start` alone, which is exactly how `create`, +// `generate` and `init` came to be filling the default in. It is written over +// the whole tree now. +func TestPathFlagsLeaveRoomForTheRecordedPath(t *testing.T) { + var checked int + walk(t, NewRootCommand(), nil, func(name string, cmd *cobra.Command) { + f := cmd.Flags().Lookup("path") + if f == nil { + return + } + checked++ + assert.Empty(t, f.DefValue, + "`azd ai eval %s --path` defaults to %q, so the path `init` recorded can "+ + "never be reached: level 1 of the cascade only yields on an empty value", + name, f.DefValue) + }) + + // If --path is ever renamed, the loop above passes by visiting nothing. + assert.GreaterOrEqual(t, checked, 3, + "expected --path on at least init, generate and eval create; found %d", checked) } // Every command that reads the configuration has to be able to say where it is, // or a project scaffolded with --path is unreachable from that command. +// +// `create` was missing from this list, and was one of the two commands that +// could not find a configuration outside ./evals. func TestCommandsReadingTheConfigTakePath(t *testing.T) { - for _, path := range []string{"run start", "init", "generate"} { + for _, path := range []string{"run start", "init", "generate", "create"} { cmd := find(t, path) assert.NotNilf(t, cmd.Flags().Lookup("path"), "%s reads the configuration, so it must accept --path", path) } } -// --path defaults to empty, not to ./evals, so "not given" stays -// distinguishable from "given the default". Defaulting it to ./evals would -// shadow the path init recorded and reintroduce the bug. -func TestRunPathFlagDefaultsToEmpty(t *testing.T) { - flag := find(t, "run start").Flags().Lookup("path") - require.NotNil(t, flag) - assert.Empty(t, flag.DefValue, - "a non-empty default would always win over the recorded path") +// Guards against the tree walk above passing because the flag was renamed. +func TestPathFlagIsStillCalledPath(t *testing.T) { + var names []string + walk(t, NewRootCommand(), nil, func(name string, cmd *cobra.Command) { + cmd.LocalFlags().VisitAll(func(f *pflag.Flag) { + if f.Name == "path" { + names = append(names, name) + } + }) + }) + + for _, want := range []string{"init", "generate", "create", "run start"} { + assert.Contains(t, names, want) + } } // The recorded key is what `init` writes and what the other commands read; a @@ -54,5 +195,147 @@ func TestEvalPathEnvKey(t *testing.T) { assert.Equal(t, "EVAL_CONFIG_PATH", envKeyEvalPath) } -// find is shared with surface_test.go; this keeps the compiler honest about it. -var _ = func(t *testing.T) *cobra.Command { return find(t, "run start") } +// `init` prints the commands to run next, and the claim those lines make is +// that they run as printed. A scaffold written somewhere other than ./evals is +// only reachable by a command that names it, because EVAL_CONFIG_PATH is +// recorded best effort and `init` succeeds without an azd environment to record +// it in. +func TestNextStepsRunAsPrinted(t *testing.T) { + cases := []struct { + name string + evalDir string + deployCmd string + wantPath bool + }{ + { + name: "a scaffold outside ./evals names itself", + evalDir: "./quality", + deployCmd: "azd ai eval create", + wantPath: true, + }, + { + name: "the default directory needs no flag", + evalDir: project.DefaultEvalDir, + deployCmd: "azd ai eval create", + wantPath: false, + }, + { + name: "an unrecorded directory needs no flag", + evalDir: "", + deployCmd: "azd ai eval create", + wantPath: false, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + s := scaffold{eval: &project.Eval{Name: "an-eval"}, evalDir: tc.evalDir} + for _, step := range s.nextSteps(tc.deployCmd) { + assert.Equal(t, tc.wantPath, strings.Contains(step, "--path "), + "step %q", step) + } + }) + } +} + +// `azd up` provisions and then deploys, reading azure.yaml -- which already +// $refs the configuration wherever it was written. It takes none of this +// extension's flags, so handing it --path prints a step that fails. +func TestNextStepsNeverFlagAzdUp(t *testing.T) { + s := scaffold{eval: &project.Eval{Name: "an-eval"}, evalDir: "./quality"} + + steps := s.nextSteps(azdUpCommand) + assert.Contains(t, steps, azdUpCommand, + "`azd up` should be suggested exactly as it is run") + for _, step := range steps { + if strings.HasPrefix(step, azdUpCommand) { + assert.NotContains(t, step, "--path", "step %q", step) + } + } +} + +// A generated next step already carried --target and --generation-model for the +// same reason. --path joins them. +func TestGenerateStepNamesTheScaffoldedDirectory(t *testing.T) { + s := scaffold{ + eval: &project.Eval{Name: "an-eval"}, + evalDir: "./quality", + target: "support-agent", + judgeModel: "gpt-4.1-nano", + rubricName: "support-agent-quality", + datasetName: "support-agent-dataset", + generateDataset: true, + generateRubric: true, + } + + steps := s.nextSteps("azd ai eval create") + if assert.Len(t, steps, 1) { + for _, want := range []string{ + "--target support-agent", + "--generation-model gpt-4.1-nano", + "--path ./quality", + } { + assert.Contains(t, steps[0], want) + } + } +} + +// A directory with a space in it printed `--path ./team evals`, which resolves +// ./team and reports the configuration missing -- the printed step failing in +// the one case it was added for. Found by running it, not by reading it. +func TestNextStepQuotesADirectoryThatNeedsIt(t *testing.T) { + cases := []struct { + name string + evalDir string + want string + }{ + { + name: "a space", + evalDir: "./team evals", + want: `--path "./team evals"`, + }, + { + name: "a windows path with a space", + evalDir: `C:\Users\Me\My Evals`, + want: `--path "C:\Users\Me\My Evals"`, + }, + { + name: "a plain relative path is left alone", + evalDir: "./quality", + want: "--path ./quality", + }, + { + name: "a plain windows path is left alone", + evalDir: `C:\Users\Me\quality`, + want: `--path C:\Users\Me\quality`, + }, + { + name: "a character the shell would expand", + evalDir: "./eval$dir", + want: `--path "./eval$dir"`, + }, + { + name: "a character that would end the command", + evalDir: "./a;rm -rf b", + want: `--path "./a;rm -rf b"`, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + s := scaffold{eval: &project.Eval{Name: "an-eval"}, evalDir: tc.evalDir} + steps := s.nextSteps("azd ai eval create") + require.NotEmpty(t, steps) + for _, step := range steps { + assert.Contains(t, step, tc.want, "step %q", step) + } + }) + } +} + +// Backslashes must survive: doubling them is right for bash and wrong for the +// two shells most likely to be reading a path that looks like this. +func TestQuoteForShellLeavesBackslashesAlone(t *testing.T) { + assert.Equal(t, `"C:\Users\Me\My Evals"`, quoteForShell(`C:\Users\Me\My Evals`)) + assert.Equal(t, `C:\Users\Me\Evals`, quoteForShell(`C:\Users\Me\Evals`)) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_composite.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_composite.go index a0e711c5a33..59eee0fbbed 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_composite.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_composite.go @@ -82,7 +82,11 @@ func newGenerateCommand() *cobra.Command { // Settled before anything reads or writes the configuration, so the // catalog entry lands next to the eval `init` scaffolded rather than // in a second configuration under ./evals that nothing else reads. - flags.path = resolveEvalDir(cmd.Context(), flags.path) + resolvedPath, err := resolveEvalDir(cmd.Context(), flags.path) + if err != nil { + return err + } + flags.path = resolvedPath target := firstNonEmpty(flags.target, declaredTarget(flags.path)) plans, err := buildGeneratePlans(generateRequest{ diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go index 67d6f8eda3b..2e444aa2f73 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go @@ -98,7 +98,10 @@ func newInitCommand() *cobra.Command { // `init` in a project scaffolded at ./quality has to find that one -- // otherwise it writes a second configuration under ./evals and // declares a second service pointing at it. - path = resolveEvalDir(cmd.Context(), path) + path, err := resolveEvalDir(cmd.Context(), path) + if err != nil { + return err + } // Asked before anything is written: the project is the one thing // init cannot supply for itself, and failing after creating @@ -531,7 +534,28 @@ func (s scaffold) withPath(step string) string { if s.evalDir == "" || s.evalDir == project.DefaultEvalDir { return step } - return step + " --path " + s.evalDir + return step + " --path " + quoteForShell(s.evalDir) +} + +// quoteForShell wraps a value a shell would otherwise read as more than one +// argument. +// +// `--path "./team evals"` is the difference between a printed step that runs +// and one that resolves ./team and reports the configuration missing. Double +// quotes are what cmd, PowerShell, bash and zsh all read the same way, and +// these lines are printed without knowing which is reading them. +// +// Backslashes are left alone: C:\Users\Me\My Evals has to come back out as +// itself, and doubling them would be right for bash and wrong for the two +// shells most likely to be reading a path that looks like that. +func quoteForShell(v string) string { + if v == "" { + return `""` + } + if !strings.ContainsAny(v, " \t\n\"'`$&|;<>()*?[]#~!") { + return v + } + return `"` + strings.ReplaceAll(v, `"`, `\"`) + `"` } // generateCommand builds a `generate` invocation that runs as printed. diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go index fd82d9f422b..5452a7f0402 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go @@ -134,7 +134,10 @@ func buildRunCommand(use, short string) *cobra.Command { // One flag takes a name or an id. A declared name also brings the // declaration, which is what says where rows come from; a bare id // has none, so the pairing comes from the eval's previous run. - evalDir := ec.evalDir(ctx, evalPath) + evalDir, err := ec.evalDir(ctx, evalPath) + if err != nil { + return err + } ref, err := ec.resolveEvalRef(ctx, evalDir, chooseEvalIn(cmd, evalDir, groupName)) if err != nil { return err diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output.go index 651e3990244..b7ab9ee8f8c 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output.go @@ -294,7 +294,10 @@ func newRunOutputExportCommand() *cobra.Command { // described. The declaration is asked instead, which is how `run start` // decides it, so the two doors cannot pick different evals. func resolveEvalID(cmd *cobra.Command, ec *evalContext, groupName string) (string, error) { - evalDir := ec.evalDir(cmd.Context(), evalPathFlag(cmd)) + evalDir, err := ec.evalDir(cmd.Context(), evalPathFlag(cmd)) + if err != nil { + return "", err + } // The same prompt `run start` gets. Without it a project declaring two // evals could start a run by answering a question, and then not list, // show or cancel it without repeating the answer as a flag. From 87a208dbfaa3ad8a7902036148608e27b11d6f83 Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 18 Aug 2026 12:32:28 +0300 Subject: [PATCH 272/320] Make the one-configuration rule impossible to ask around A directory holding both azure.eval.yaml and the legacy eval.yaml has no right answer: azure.yaml $refs a single file by name, so preferring one silently means the CLI edits one configuration while `azd up` deploys the other, and nothing says so. checkOneConfig exists to refuse it. It was applied by OpenEvalConfig and SaveEvalConfig, which is every route into the configuration except one. `eval create` needs the path rather than the parsed configuration, so it called ResolveEvalConfigPath and read the file itself. `run start`, `init` and `generate` all refused such a directory; `create` -- the command that publishes -- resolved one without a word. Rather than add the check to the fourth caller, ResolveEvalConfigPath now returns an error and applies the rule itself. There is no longer a way to ask where the configuration is and not be told the question was ambiguous. The naming preference on its own is resolvedConfigPath, unexported, for the two functions that have already applied the guard. TestResolveEvalConfigPath_PrefersTheCurrentName asserted the silent preference on the public function, so the test that should have caught this documented the behaviour instead. It now asserts the refusal, and the preference is tested where it still applies. Reproduced before and after: with both files present, `create` exited 0 and now exits 1 with the same message the other three commands give. --- .../internal/cmd/catalog.go | 6 ++- .../internal/cmd/eval_group.go | 4 +- .../internal/cmd/evalref.go | 5 +- .../azure.ai.evaluations/internal/cmd/init.go | 5 +- .../internal/project/eval_config_name_test.go | 46 +++++++++++++++++-- .../internal/project/eval_config_store.go | 22 +++++++-- 6 files changed, 76 insertions(+), 12 deletions(-) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/catalog.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/catalog.go index 5e553e953a0..e92a76321f2 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/catalog.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/catalog.go @@ -107,7 +107,11 @@ func updateCatalog( // Resolved, not the current name: SaveEvalConfig writes back over a // legacy file when that is what the project has, and the line has to // name the file it actually wrote. - path := filepath.ToSlash(project.ResolveEvalConfigPath(evalDir)) + resolved, err := project.ResolveEvalConfigPath(evalDir) + if err != nil { + return err + } + path := filepath.ToSlash(resolved) if created { fmt.Fprint(cmd.OutOrStdout(), messages.CreatedCatalogFile(path)) } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/eval_group.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/eval_group.go index d75f26258e2..e6c7b5ef565 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/eval_group.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/eval_group.go @@ -46,7 +46,9 @@ func newEvalCreateCommand() *cobra.Command { if err != nil { return err } - path = project.ResolveEvalConfigPath(dir) + if path, err = project.ResolveEvalConfigPath(dir); err != nil { + return err + } } cfg, err := project.LoadEvalConfig(path) if err != nil { diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evalref.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evalref.go index f3e09668493..b41ba537fe3 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evalref.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evalref.go @@ -42,7 +42,10 @@ func (ec *evalContext) resolveEvalRef( ctx context.Context, evalDir, nameOrID string, ) (evalRef, error) { - configPath := project.ResolveEvalConfigPath(evalDir) + configPath, err := project.ResolveEvalConfigPath(evalDir) + if err != nil { + return evalRef{}, err + } cfg, err := project.OpenEvalConfig(evalDir) if err != nil { return evalRef{}, err diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go index 2e444aa2f73..b0f3acd52f1 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go @@ -129,7 +129,10 @@ func newInitCommand() *cobra.Command { } } - configPath := project.ResolveEvalConfigPath(path) + configPath, err := project.ResolveEvalConfigPath(path) + if err != nil { + return err + } // Captured before the write: init merges into an existing config, so // reporting it as created would claim a file it only added to. _, configExistedErr := os.Stat(configPath) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_name_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_name_test.go index b57f14fc630..b74ab1cdf96 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_name_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_name_test.go @@ -56,16 +56,38 @@ func TestSaveEvalConfig_WritesBackOverALegacyFile(t *testing.T) { "a second configuration beside the one azure.yaml references would be inert") } -// With both present the current name wins, so a project that has migrated is -// not dragged back to the old file. -func TestResolveEvalConfigPath_PrefersTheCurrentName(t *testing.T) { +// A directory holding both names is refused, not silently resolved. Preferring +// one is the dangerous answer: azure.yaml $refs a single file by name, so the +// CLI would edit one configuration while `azd up` deployed the other. +// +// This used to assert the preference instead, and `eval create` -- the one +// command that asks for the path rather than the parsed configuration -- read +// one of the two without a word while `run`, `init` and `generate` all refused. +func TestResolveEvalConfigPath_RefusesADirectoryHoldingBoth(t *testing.T) { dir := t.TempDir() require.NoError(t, os.WriteFile( filepath.Join(dir, LegacyEvalConfigBase), []byte(minimalConfig), 0o600)) require.NoError(t, os.WriteFile( filepath.Join(dir, EvalConfigBase), []byte(minimalConfig), 0o600)) - assert.Equal(t, EvalConfigPath(dir), ResolveEvalConfigPath(dir)) + path, err := ResolveEvalConfigPath(dir) + require.Error(t, err, "both names present has no single right answer") + assert.Empty(t, path) + assert.Contains(t, err.Error(), LegacyEvalConfigBase) + assert.Contains(t, err.Error(), EvalConfigBase) +} + +// The naming preference itself still holds for the callers that have already +// applied the guard: with only the legacy file there, that is the file a save +// writes back over. +func TestResolvedConfigPath_PrefersTheCurrentName(t *testing.T) { + dir := t.TempDir() + require.NoError(t, os.WriteFile( + filepath.Join(dir, LegacyEvalConfigBase), []byte(minimalConfig), 0o600)) + require.NoError(t, os.WriteFile( + filepath.Join(dir, EvalConfigBase), []byte(minimalConfig), 0o600)) + + assert.Equal(t, EvalConfigPath(dir), resolvedConfigPath(dir)) } // An empty directory resolves to the current name, which is what a first @@ -73,5 +95,19 @@ func TestResolveEvalConfigPath_PrefersTheCurrentName(t *testing.T) { func TestResolveEvalConfigPath_EmptyDirectoryUsesTheCurrentName(t *testing.T) { dir := t.TempDir() - assert.Equal(t, EvalConfigPath(dir), ResolveEvalConfigPath(dir)) + path, err := ResolveEvalConfigPath(dir) + require.NoError(t, err) + assert.Equal(t, EvalConfigPath(dir), path) +} + +// Only the legacy file present resolves to it, so a project that has not +// migrated keeps working. +func TestResolveEvalConfigPath_LegacyOnlyResolvesToLegacy(t *testing.T) { + dir := t.TempDir() + require.NoError(t, os.WriteFile( + filepath.Join(dir, LegacyEvalConfigBase), []byte(minimalConfig), 0o600)) + + path, err := ResolveEvalConfigPath(dir) + require.NoError(t, err) + assert.Equal(t, filepath.Join(dir, LegacyEvalConfigBase), path) } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_store.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_store.go index 6b9269814d9..aefd74e551d 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_store.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_store.go @@ -48,7 +48,23 @@ func EvalConfigPath(evalDir string) string { // ResolveEvalConfigPath is the configuration this directory actually holds: // the current name, or the legacy one when that is the only file there. -func ResolveEvalConfigPath(evalDir string) string { +// +// It refuses a directory holding both, rather than leaving that to the caller. +// The rule used to live in OpenEvalConfig alone, so `eval create` -- which +// needs the path rather than the parsed configuration -- resolved one silently +// while `run`, `init` and `generate` all refused. Returning an error is what +// makes the guard unavoidable: there is no longer a way to ask this question +// and not be told. +func ResolveEvalConfigPath(evalDir string) (string, error) { + if err := checkOneConfig(evalDir); err != nil { + return "", err + } + return resolvedConfigPath(evalDir), nil +} + +// resolvedConfigPath is the naming rule on its own, for the two functions that +// have already applied the guard. +func resolvedConfigPath(evalDir string) string { current := EvalConfigPath(evalDir) if _, err := os.Stat(current); err == nil { return current @@ -85,7 +101,7 @@ func OpenEvalConfig(evalDir string) (*EvalConfig, error) { if err := checkOneConfig(evalDir); err != nil { return nil, err } - cfg, err := LoadEvalConfig(ResolveEvalConfigPath(evalDir)) + cfg, err := LoadEvalConfig(resolvedConfigPath(evalDir)) if errors.Is(err, fs.ErrNotExist) { return nil, nil } @@ -143,7 +159,7 @@ func SaveEvalConfig(evalDir string, cfg *EvalConfig) error { if err := os.MkdirAll(evalDir, 0o750); err != nil { return messages.Creating(evalDir, err) } - return SaveEvalConfigTo(ResolveEvalConfigPath(evalDir), cfg) + return SaveEvalConfigTo(resolvedConfigPath(evalDir), cfg) } // SaveEvalConfigTo writes cfg over an explicit path, for callers that already From eb2b37698bf94516349380a31355f1318985404d Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 18 Aug 2026 12:43:25 +0300 Subject: [PATCH 273/320] Cut 1.0.9-beta for the third bug bash build --- cli/azd/extensions/azure.ai.evaluations/CHANGELOG.md | 2 +- cli/azd/extensions/azure.ai.evaluations/extension.yaml | 2 +- cli/azd/extensions/azure.ai.evaluations/version.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cli/azd/extensions/azure.ai.evaluations/CHANGELOG.md b/cli/azd/extensions/azure.ai.evaluations/CHANGELOG.md index c171a93b5aa..610c88d8f92 100644 --- a/cli/azd/extensions/azure.ai.evaluations/CHANGELOG.md +++ b/cli/azd/extensions/azure.ai.evaluations/CHANGELOG.md @@ -1,6 +1,6 @@ # Release History -## 1.0.8-beta (Unreleased) +## 1.0.9-beta (Unreleased) First release of the Foundry evaluations extension. diff --git a/cli/azd/extensions/azure.ai.evaluations/extension.yaml b/cli/azd/extensions/azure.ai.evaluations/extension.yaml index 9f944fd8bc0..fb5bc4f9d74 100644 --- a/cli/azd/extensions/azure.ai.evaluations/extension.yaml +++ b/cli/azd/extensions/azure.ai.evaluations/extension.yaml @@ -5,7 +5,7 @@ displayName: Foundry evaluations (Beta) description: Define and run Foundry evaluations from your terminal. (Beta) usage: azd ai eval [options] # NOTE: Make sure version.txt is in sync with this version. -version: 1.0.8-beta +version: 1.0.9-beta requiredAzdVersion: ">=1.27.1" language: go capabilities: diff --git a/cli/azd/extensions/azure.ai.evaluations/version.txt b/cli/azd/extensions/azure.ai.evaluations/version.txt index e00d876d2a3..f1c56e359fb 100644 --- a/cli/azd/extensions/azure.ai.evaluations/version.txt +++ b/cli/azd/extensions/azure.ai.evaluations/version.txt @@ -1 +1 @@ -1.0.8-beta +1.0.9-beta From 7c33f9f4829b751d90e4a3f338604502efd3ff23 Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 18 Aug 2026 13:08:49 +0300 Subject: [PATCH 274/320] Act on -e/--environment instead of parsing it and throwing it away The SDK registers -e/--environment and puts it in the ExtensionContext both root commands were discarding. Nothing read it, so the flag was accepted and ignored: - `azd -e envB ai eval create` read EVAL_CONFIG_PATH out of envA and found a configuration envB knows nothing about. - the eval id it created was written back to envA, so envB never learned about its own eval and envA's record was overwritten by another environment's work. - `-e doesnotexist` -- a name azd itself rejects with "environment does not exist" -- was accepted in silence and acted on the default environment. Which environment an invocation is about is now answered in one place per extension, azdEnvironmentName, and carried on the context so the shared endpoint cascade can ask the same question without a cobra command to hand. Five independent GetCurrent calls answered it before, and -e was honoured by none of them. Reproduced before and after with two environments and a path recorded in only one: -e now reads and writes the environment it names, and a name azd rejects produces a warning naming it rather than silently landing elsewhere. projectctx stays byte-identical between the two extensions. --- .../internal/cmd/context.go | 35 +++++- .../azure.ai.evaluations/internal/cmd/init.go | 6 +- .../internal/cmd/init_model.go | 10 +- .../azure.ai.evaluations/internal/cmd/root.go | 12 +- .../internal/foundry/projectctx/resolver.go | 54 +++++++-- .../foundry/projectctx/selected_env_test.go | 109 ++++++++++++++++++ 6 files changed, 202 insertions(+), 24 deletions(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/foundry/projectctx/selected_env_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/context.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/context.go index 5788ca21acb..b6256a30565 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/context.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/context.go @@ -126,19 +126,40 @@ func (c azdTokenRetry) GetToken( // lookupEndpointFromAzd reads the endpoint from the active azd environment, // returning empty strings when azd has no current environment. -func lookupEndpointFromAzd(ctx context.Context, azdClient *azdext.AzdClient) (endpoint, envName string) { +// azdEnvironmentName is the environment this invocation acts on: the one +// -e/--environment named, or azd's current one when it named none. +// +// Answered here because it was answered independently in five places and +// -e was honoured by none of them. `azd ai eval create -e staging` read its +// endpoint out of the default environment and wrote its eval id back there, +// and `-e a-name-azd-rejects` was accepted in silence. +// +// Empty means there is no environment to act on, which is ordinary: the atomic +// commands work standalone against the data plane. +func azdEnvironmentName(ctx context.Context, azdClient *azdext.AzdClient) string { + if name := projectctx.SelectedEnvironment(ctx); name != "" { + return name + } envResp, err := azdClient.Environment().GetCurrent(ctx, &azdext.EmptyRequest{}) - if err != nil || envResp == nil || envResp.Environment == nil { + if err != nil || envResp.GetEnvironment() == nil { + return "" + } + return envResp.Environment.Name +} + +func lookupEndpointFromAzd(ctx context.Context, azdClient *azdext.AzdClient) (endpoint, envName string) { + envName = azdEnvironmentName(ctx, azdClient) + if envName == "" { return "", "" } val, err := azdClient.Environment().GetValue(ctx, &azdext.GetEnvRequest{ - EnvName: envResp.Environment.Name, + EnvName: envName, Key: projectEndpointEnvKey, }) if err != nil || val == nil || val.Value == "" { - return "", envResp.Environment.Name + return "", envName } - return val.Value, envResp.Environment.Name + return val.Value, envName } // errNoAzdEnvironment reports that there is no azd environment to persist into. @@ -456,6 +477,10 @@ func resolveEvalDir(ctx context.Context, flagValue string) (string, error) { } defer azdClient.Close() + if name := projectctx.SelectedEnvironment(ctx); name != "" { + return readRecordedEvalPath(ctx, azdClient, name) + } + env, err := azdClient.Environment().GetCurrent(ctx, &azdext.EmptyRequest{}) if err != nil { // The same rule the endpoint cascade uses: azd saying "there is no diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go index b0f3acd52f1..f12c63f29f3 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go @@ -723,12 +723,12 @@ func recordEvalPath(ctx context.Context, path string) { } defer azdClient.Close() - env, err := azdClient.Environment().GetCurrent(ctx, &azdext.EmptyRequest{}) - if err != nil || env.GetEnvironment() == nil { + envName := azdEnvironmentName(ctx, azdClient) + if envName == "" { return } _, _ = azdClient.Environment().SetValue(ctx, &azdext.SetEnvRequest{ - EnvName: env.GetEnvironment().GetName(), + EnvName: envName, Key: envKeyEvalPath, Value: filepath.ToSlash(path), }) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_model.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_model.go index ab55ff8bcfd..330cc908f09 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_model.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_model.go @@ -113,8 +113,8 @@ func modelDeploymentFromAzdEnv(ctx context.Context) string { return azdEnvValue(ctx, judgeModelEnvKey) } -// azdEnvValue reads one key from the active azd environment, answering empty -// whenever there is no daemon, no environment, or no such key. +// azdEnvValue reads one key from the azd environment this invocation acts on, +// answering empty whenever there is no daemon, no environment, or no such key. func azdEnvValue(ctx context.Context, key string) string { azdClient, err := azdext.NewAzdClient() if err != nil { @@ -122,12 +122,12 @@ func azdEnvValue(ctx context.Context, key string) string { } defer azdClient.Close() - envResp, err := azdClient.Environment().GetCurrent(ctx, &azdext.EmptyRequest{}) - if err != nil || envResp.GetEnvironment() == nil { + envName := azdEnvironmentName(ctx, azdClient) + if envName == "" { return "" } val, err := azdClient.Environment().GetValue(ctx, &azdext.GetEnvRequest{ - EnvName: envResp.Environment.Name, + EnvName: envName, Key: key, }) if err != nil { diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/root.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/root.go index 8756ef72683..194d6c51ee6 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/root.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/root.go @@ -6,6 +6,8 @@ package cmd import ( "fmt" + "azureaieval/internal/foundry/projectctx" + "github.com/azure/azure-dev/cli/azd/pkg/azdext" "github.com/fatih/color" "github.com/spf13/cobra" @@ -13,7 +15,7 @@ import ( // NewRootCommand builds the `azd ai eval` command tree. func NewRootCommand() *cobra.Command { - rootCmd, _ := azdext.NewExtensionRootCommand(azdext.ExtensionCommandOptions{ + rootCmd, extCtx := azdext.NewExtensionRootCommand(azdext.ExtensionCommandOptions{ Name: "eval", Use: "eval [options]", Short: fmt.Sprintf( @@ -40,6 +42,14 @@ func NewRootCommand() *cobra.Command { return err } } + // -e/--environment is parsed by the SDK into extCtx and then has to be + // acted on. Discarding extCtx left the flag accepted and ignored: + // `azd ai eval create -e staging` read the endpoint out of the default + // environment and wrote its eval id back there, and even a name azd + // itself rejects was accepted in silence. Set here rather than at each + // reader, so there is one answer to which environment this invocation + // is about. + cmd.SetContext(projectctx.WithSelectedEnvironment(cmd.Context(), extCtx.Environment)) setupDebugLogging(cmd.Flags()) return nil } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/foundry/projectctx/resolver.go b/cli/azd/extensions/azure.ai.evaluations/internal/foundry/projectctx/resolver.go index f380ee1ee7b..62fa3b2033e 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/foundry/projectctx/resolver.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/foundry/projectctx/resolver.go @@ -76,21 +76,29 @@ type envSource interface { // the daemon failed to answer, which the caller must not read as absence -- // falling through would resolve to a lower-priority endpoint that can belong to // a different project. +// +// The environment is the one -e/--environment named, when it named one. Asking +// azd for the current environment instead is how `azd -e staging` came to read +// the endpoint out of the default environment and write its ids back there. func readEnvHostedSource(ctx context.Context, env envSource) (value, name string, err error) { - envResp, envErr := env.GetCurrent(ctx, &azdext.EmptyRequest{}) - if envErr != nil { - if !hostedSourceAbsent(envErr) { - return "", "", envErr + name = SelectedEnvironment(ctx) + if name == "" { + envResp, envErr := env.GetCurrent(ctx, &azdext.EmptyRequest{}) + if envErr != nil { + if !hostedSourceAbsent(envErr) { + return "", "", envErr + } + return "", "", nil } - return "", "", nil - } - if envResp.GetEnvironment() == nil { - return "", "", nil + if envResp.GetEnvironment() == nil { + return "", "", nil + } + name = envResp.Environment.Name } for _, key := range []string{foundryEnvKey, azureAiEnvKey} { envVal, valErr := env.GetValue(ctx, &azdext.GetEnvRequest{ - EnvName: envResp.Environment.Name, + EnvName: name, Key: key, }) if valErr != nil { @@ -100,12 +108,38 @@ func readEnvHostedSource(ctx context.Context, env envSource) (value, name string continue } if envVal.GetValue() != "" { - return envVal.Value, envResp.Environment.Name, nil + return envVal.Value, name, nil } } + // The name is reported only alongside a value: it says where the endpoint + // came from, and there is no endpoint here. return "", "", nil } +// selectedEnvKey carries the environment -e/--environment named. +type selectedEnvKey struct{} + +// WithSelectedEnvironment records the environment the caller named, so every +// azd read and write in this invocation acts on that one rather than on azd's +// default. +// +// It travels on the context because the answer is fixed for the whole +// invocation and is needed several layers below the flag -- including here, in +// the cascade both extensions share, which has no cobra command to ask. +func WithSelectedEnvironment(ctx context.Context, name string) context.Context { + if name == "" { + return ctx + } + return context.WithValue(ctx, selectedEnvKey{}, name) +} + +// SelectedEnvironment is the name -e/--environment gave, or empty when it gave +// none and azd's default is what to act on. +func SelectedEnvironment(ctx context.Context) string { + name, _ := ctx.Value(selectedEnvKey{}).(string) + return name +} + // azd's absence sentinels, as they reach us. // // `pkg/environment` and `pkg/environment/azdcontext` declare these with diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/foundry/projectctx/selected_env_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/foundry/projectctx/selected_env_test.go new file mode 100644 index 00000000000..2d9047a8535 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/foundry/projectctx/selected_env_test.go @@ -0,0 +1,109 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package projectctx + +import ( + "context" + "testing" + + "github.com/azure/azure-dev/cli/azd/pkg/azdext" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "google.golang.org/grpc" +) + +// -e/--environment is parsed by the SDK and then has to be acted on. It was +// discarded, so `azd ai eval create -e staging` read its endpoint out of the +// default environment and wrote its ids back there -- and `-e a-name-azd- +// rejects` was accepted in silence, because nothing ever asked azd about it. +// +// These tests pin that the named environment is the one read, and that azd is +// not asked which environment is current when a name was given: asking can only +// produce a second, disagreeing answer. + +// perEnv answers GetValue per environment, which is what tells "read staging" +// apart from "read whatever azd calls current". +type perEnv struct { + values map[string]map[string]string + current string + currentCalls int + askedEnvs []string +} + +func (f *perEnv) GetCurrent( + context.Context, *azdext.EmptyRequest, ...grpc.CallOption, +) (*azdext.EnvironmentResponse, error) { + f.currentCalls++ + return &azdext.EnvironmentResponse{ + Environment: &azdext.Environment{Name: f.current}, + }, nil +} + +func (f *perEnv) GetValue( + _ context.Context, req *azdext.GetEnvRequest, _ ...grpc.CallOption, +) (*azdext.KeyValueResponse, error) { + f.askedEnvs = append(f.askedEnvs, req.EnvName) + return &azdext.KeyValueResponse{ + Key: req.Key, + Value: f.values[req.EnvName][req.Key], + }, nil +} + +func twoEnvironments() *perEnv { + return &perEnv{ + values: map[string]map[string]string{ + "default": {foundryEnvKey: "https://from-default/"}, + "staging": {foundryEnvKey: "https://from-staging/"}, + }, + current: "default", + } +} + +func TestSelectedEnvironmentIsTheOneRead(t *testing.T) { + fake := twoEnvironments() + + ctx := WithSelectedEnvironment(context.Background(), "staging") + value, name, err := readEnvHostedSource(ctx, fake) + + require.NoError(t, err) + assert.Equal(t, "https://from-staging/", value) + assert.Equal(t, "staging", name) + assert.Zero(t, fake.currentCalls, + "a named environment is the answer; asking azd for the current one can only disagree") + assert.NotContains(t, fake.askedEnvs, "default") +} + +func TestWithoutSelectionAzdsCurrentEnvironmentIsRead(t *testing.T) { + fake := twoEnvironments() + + value, name, err := readEnvHostedSource(context.Background(), fake) + + require.NoError(t, err) + assert.Equal(t, "https://from-default/", value) + assert.Equal(t, "default", name) + assert.Equal(t, 1, fake.currentCalls) +} + +// A named environment holding no endpoint reports none. Falling back to the +// default's is the bug, restated. +func TestSelectedEnvironmentWithNoEndpointDoesNotFallBack(t *testing.T) { + fake := twoEnvironments() + fake.values["staging"] = map[string]string{} + + ctx := WithSelectedEnvironment(context.Background(), "staging") + value, _, err := readEnvHostedSource(ctx, fake) + + require.NoError(t, err) + assert.Empty(t, value, "staging has no endpoint; the default's is not an answer") + assert.Zero(t, fake.currentCalls) + assert.NotContains(t, fake.askedEnvs, "default") +} + +// An empty name is "none given", not "the environment called empty string". +func TestWithSelectedEnvironmentIgnoresAnEmptyName(t *testing.T) { + assert.Empty(t, SelectedEnvironment(WithSelectedEnvironment(context.Background(), ""))) + assert.Equal(t, "staging", + SelectedEnvironment(WithSelectedEnvironment(context.Background(), "staging"))) + assert.Empty(t, SelectedEnvironment(context.Background())) +} From 11b085e32f02b9900ab35e77cb95b170ce6136c2 Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 18 Aug 2026 13:51:33 +0300 Subject: [PATCH 275/320] Report an environment azd does not have instead of stepping over it Follow-up to the -e/--environment fix, from review of it. Making -e targeted left one gap: a name azd does not have still produced no error. `GetValue` answers `'typo': environment not found`, which the cascade classifies as absence, so the read fell through to the global config or the host variable and the command ran against a lower-priority endpoint -- possibly another project -- before writing its ids into an environment that does not exist, which azd accepts only far enough to warn about. Absence and a typo are not the same answer. Without -e, "this environment has no endpoint" is ordinary and the cascade should carry on. With -e, the caller named something, and azd saying it has no such environment is a mistake to report. That distinction is what noSuchEnvironment draws, and only for the named case. The error is built in projectctx rather than in either extension's messages package: resolver.go has no module-local imports, which is what lets it stay byte-identical in both extensions, and adding one for a single string would have ended that. Also corrected the quoteForShell comment, which claimed double quotes are read the same way by cmd, PowerShell, bash and zsh. That is true for a path with spaces, which is the case it exists for, and not true for $ or a backtick, which those shells still expand inside double quotes. Such a path has no portable form; it is wrapped anyway, because one argument that may expand beats two that certainly break, and the line is printed for a person rather than executed here. Three planted defects, three caught by the intended test. Verified live: -e with a typo now stops with the name it could not find, while a real -e and the unflagged path each still read and write their own environment. --- .../azure.ai.evaluations/internal/cmd/init.go | 12 +++- .../internal/foundry/projectctx/resolver.go | 32 ++++++++- .../foundry/projectctx/selected_env_test.go | 67 +++++++++++++++++++ 3 files changed, 107 insertions(+), 4 deletions(-) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go index f12c63f29f3..1f44c84c836 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go @@ -544,9 +544,15 @@ func (s scaffold) withPath(step string) string { // argument. // // `--path "./team evals"` is the difference between a printed step that runs -// and one that resolves ./team and reports the configuration missing. Double -// quotes are what cmd, PowerShell, bash and zsh all read the same way, and -// these lines are printed without knowing which is reading them. +// and one that resolves ./team and reports the configuration missing. That is +// the case worth getting right, and double quotes are what cmd, PowerShell, +// bash and zsh all read the same way for a path containing spaces. +// +// A path containing $ or a backtick has no portable answer: double quotes stop +// neither from expanding in bash, zsh or PowerShell, and single quotes -- which +// would -- are literal only on the POSIX shells. Such a path is wrapped anyway, +// because one argument that may expand still beats two that certainly break, +// and this line is printed for a person to read rather than executed here. // // Backslashes are left alone: C:\Users\Me\My Evals has to come back out as // itself, and doubling them would be right for bash and wrong for the two diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/foundry/projectctx/resolver.go b/cli/azd/extensions/azure.ai.evaluations/internal/foundry/projectctx/resolver.go index 62fa3b2033e..b782c84be0e 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/foundry/projectctx/resolver.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/foundry/projectctx/resolver.go @@ -6,6 +6,7 @@ package projectctx import ( "context" "errors" + "fmt" "os" "strings" @@ -81,7 +82,8 @@ type envSource interface { // azd for the current environment instead is how `azd -e staging` came to read // the endpoint out of the default environment and write its ids back there. func readEnvHostedSource(ctx context.Context, env envSource) (value, name string, err error) { - name = SelectedEnvironment(ctx) + selected := SelectedEnvironment(ctx) + name = selected if name == "" { envResp, envErr := env.GetCurrent(ctx, &azdext.EmptyRequest{}) if envErr != nil { @@ -102,6 +104,14 @@ func readEnvHostedSource(ctx context.Context, env envSource) (value, name string Key: key, }) if valErr != nil { + // A name the caller typed and azd does not have is a mistake to + // report, not an absence to step over. Falling through would run + // the command against a lower-priority endpoint -- possibly another + // project -- and then write its ids into an environment that does + // not exist, which azd accepts only far enough to warn about. + if selected != "" && noSuchEnvironment(valErr) { + return "", "", ErrNoSuchEnvironment(name) + } if !hostedSourceAbsent(valErr) { return "", "", valErr } @@ -116,6 +126,26 @@ func readEnvHostedSource(ctx context.Context, env envSource) (value, name string return "", "", nil } +// noSuchEnvironment is azd's answer for a named environment it does not have, +// as distinct from the other absences: there being no default, or no project. +func noSuchEnvironment(err error) bool { + st, ok := azdext.GRPCStatusFromError(err) + if !ok || st.Code() != codes.Unknown { + return false + } + return strings.HasSuffix(st.Message(), "': "+azdNoSuchEnvironment) +} + +// ErrNoSuchEnvironment reports a -e/--environment naming something azd does not +// have. Built here rather than in either extension's messages package, so this +// file stays free of module-local imports and identical in both. +func ErrNoSuchEnvironment(name string) error { + return fmt.Errorf( + "azd environment %q does not exist; run `azd env list` to see the ones that do", + name, + ) +} + // selectedEnvKey carries the environment -e/--environment named. type selectedEnvKey struct{} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/foundry/projectctx/selected_env_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/foundry/projectctx/selected_env_test.go index 2d9047a8535..57704a20507 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/foundry/projectctx/selected_env_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/foundry/projectctx/selected_env_test.go @@ -11,6 +11,8 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" ) // -e/--environment is parsed by the SDK and then has to be acted on. It was @@ -107,3 +109,68 @@ func TestWithSelectedEnvironmentIgnoresAnEmptyName(t *testing.T) { SelectedEnvironment(WithSelectedEnvironment(context.Background(), "staging"))) assert.Empty(t, SelectedEnvironment(context.Background())) } + +// failingEnv answers GetValue with a fixed error, which is how azd reports an +// environment it does not have. +type failingEnv struct { + err error + currentCalls int +} + +func (f *failingEnv) GetCurrent( + context.Context, *azdext.EmptyRequest, ...grpc.CallOption, +) (*azdext.EnvironmentResponse, error) { + f.currentCalls++ + return &azdext.EnvironmentResponse{ + Environment: &azdext.Environment{Name: "default"}, + }, nil +} + +func (f *failingEnv) GetValue( + context.Context, *azdext.GetEnvRequest, ...grpc.CallOption, +) (*azdext.KeyValueResponse, error) { + return nil, f.err +} + +// A name the caller typed and azd does not have is a mistake to report, not an +// absence to step over. Stepping over it runs the command against a +// lower-priority endpoint, which can belong to another project, and then writes +// its ids into an environment that does not exist. +func TestATypoedEnvironmentNameIsReportedNotSteppedOver(t *testing.T) { + fake := &failingEnv{ + err: status.Error(codes.Unknown, "'does-not-exist': environment not found"), + } + + ctx := WithSelectedEnvironment(context.Background(), "does-not-exist") + _, _, err := readEnvHostedSource(ctx, fake) + + require.Error(t, err, "a named environment azd does not have must stop the cascade") + assert.Contains(t, err.Error(), "does-not-exist") +} + +// The same answer without a name given is ordinary absence: there is simply no +// endpoint in the current environment, and the cascade carries on. +func TestTheSameAnswerWithoutANameIsStillAbsence(t *testing.T) { + fake := &failingEnv{ + err: status.Error(codes.Unknown, "'default': environment not found"), + } + + value, name, err := readEnvHostedSource(context.Background(), fake) + + require.NoError(t, err, "without -e this is absence, and the cascade continues") + assert.Empty(t, value) + assert.Empty(t, name) +} + +// A named environment that exists but cannot be read for some other reason is +// a failure, and must not be reported as a missing environment either. +func TestANamedEnvironmentThatFailsDifferentlyStillFails(t *testing.T) { + fake := &failingEnv{err: status.Error(codes.Internal, "the store is on fire")} + + ctx := WithSelectedEnvironment(context.Background(), "staging") + _, _, err := readEnvHostedSource(ctx, fake) + + require.Error(t, err) + assert.NotContains(t, err.Error(), "does not exist", + "a broken read is not a missing environment") +} From 4c18ff32f909ab80dd3bfbbcddf55f8bc8acdfcc Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 18 Aug 2026 14:04:13 +0300 Subject: [PATCH 276/320] Cut the fourth bug bash build --- cli/azd/extensions/azure.ai.evaluations/CHANGELOG.md | 2 +- cli/azd/extensions/azure.ai.evaluations/extension.yaml | 2 +- cli/azd/extensions/azure.ai.evaluations/version.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cli/azd/extensions/azure.ai.evaluations/CHANGELOG.md b/cli/azd/extensions/azure.ai.evaluations/CHANGELOG.md index 610c88d8f92..deb4fb20d78 100644 --- a/cli/azd/extensions/azure.ai.evaluations/CHANGELOG.md +++ b/cli/azd/extensions/azure.ai.evaluations/CHANGELOG.md @@ -1,6 +1,6 @@ # Release History -## 1.0.9-beta (Unreleased) +## 1.0.10-beta (Unreleased) First release of the Foundry evaluations extension. diff --git a/cli/azd/extensions/azure.ai.evaluations/extension.yaml b/cli/azd/extensions/azure.ai.evaluations/extension.yaml index fb5bc4f9d74..63b6f1c66ee 100644 --- a/cli/azd/extensions/azure.ai.evaluations/extension.yaml +++ b/cli/azd/extensions/azure.ai.evaluations/extension.yaml @@ -5,7 +5,7 @@ displayName: Foundry evaluations (Beta) description: Define and run Foundry evaluations from your terminal. (Beta) usage: azd ai eval [options] # NOTE: Make sure version.txt is in sync with this version. -version: 1.0.9-beta +version: 1.0.10-beta requiredAzdVersion: ">=1.27.1" language: go capabilities: diff --git a/cli/azd/extensions/azure.ai.evaluations/version.txt b/cli/azd/extensions/azure.ai.evaluations/version.txt index f1c56e359fb..25a15c011b0 100644 --- a/cli/azd/extensions/azure.ai.evaluations/version.txt +++ b/cli/azd/extensions/azure.ai.evaluations/version.txt @@ -1 +1 @@ -1.0.9-beta +1.0.10-beta From def17468e52673709beb5a948fd7bbf280cc7f5c Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 18 Aug 2026 15:02:28 +0300 Subject: [PATCH 277/320] Stop reading a failed existence check as proof the dataset is absent `create` and `update` differ in exactly one thing: whether the name is already taken. datasetPresence answered that from a version listing plus a point read, and discarded the error from both. So a 403, a throttle or a timeout on the listing made every read fail, presence came back "not there", and `create` went on to publish -- a further version of a dataset that already existed, which is precisely what create exists to refuse. The one decision separating the two verbs was being made by an error nobody looked at. Absence is still admitted without proof for the case it was built for: an empty 200 from a listing that has not caught up after a publish. That is a real answer, just a stale one, and refusing there strands a caller between two commands that each blame the other. A read that did not answer at all is a different thing and is now returned. The same rule on the point read: only a 404 means "not this version". Both extensions carry their own copy of these commands, so both are changed, and the presence tests are the same file in both bar the module path. Four planted defects across the two copies, four caught by the intended test. --- .../internal/cmd/dataset.go | 27 +++++-- .../internal/cmd/dataset_presence_test.go | 74 +++++++++++++++++-- .../internal/messages/messages.go | 9 +++ 3 files changed, 96 insertions(+), 14 deletions(-) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset.go index 8df34f24b8b..9eaf4e5a5a1 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset.go @@ -66,22 +66,34 @@ func newDatasetUpdateCommand() *cobra.Command { // Absence is only certain when the listing itself answered 404. An empty 200 // does not prove it: an unknown dataset and a listing that has not caught up // are indistinguishable. +// +// A read that failed for any other reason proves nothing at all, and is +// returned. Treating a 403 or a timeout as "not there" let `create` go on to +// publish a further version of a dataset that already existed -- the one thing +// separating create from update, decided by an error nobody looked at. func datasetPresence( ctx context.Context, client *dataset_api.DatasetClient, name string, -) (exists, absenceCertain bool) { +) (exists, absenceCertain bool, err error) { existing, listErr := client.ListDatasetVersions(ctx, name, ProjectEndpointAPIVersion) + if listErr != nil && !dataset_api.IsNotFound(listErr) { + return false, false, messages.CheckingDataset(name, listErr) + } if listErr == nil && existing != nil && len(existing.Value) > 0 { - return true, false + return true, false, nil } for _, v := range firstDatasetVersions { - if _, err := client.GetDataset(ctx, name, v, ProjectEndpointAPIVersion); err == nil { - return true, false + _, getErr := client.GetDataset(ctx, name, v, ProjectEndpointAPIVersion) + if getErr == nil { + return true, false, nil + } + if !dataset_api.IsNotFound(getErr) { + return false, false, messages.CheckingDataset(name, getErr) } } - return false, dataset_api.IsNotFound(listErr) + return false, dataset_api.IsNotFound(listErr), nil } // newDatasetWriteCommand builds create and update. Both run the same upload, @@ -120,7 +132,10 @@ func newDatasetWriteCommand(verb, short string) *cobra.Command { } defer ec.Close() - exists, absenceCertain := datasetPresence(ctx, ec.datasetClient, name) + exists, absenceCertain, err := datasetPresence(ctx, ec.datasetClient, name) + if err != nil { + return err + } if err := checkAssetExistence( verb, "dataset", name, exists, absenceCertain, ); err != nil { diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset_presence_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset_presence_test.go index a4fc77ac966..4ba2e04e7fb 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset_presence_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset_presence_test.go @@ -35,6 +35,9 @@ func (s *presenceServer) requested() []string { // newPresenceClient wires a DatasetClient to a server that answers the version // listing with listStatus/listBody, and answers a point read of a dataset // version with whatever found reports for that version. +// +// Retries are off: a test that means "the service said 404" should cost one +// request, not a backoff schedule. func newPresenceClient( t *testing.T, listStatus int, @@ -83,7 +86,8 @@ func TestPresenceTrustsANonEmptyVersionListing(t *testing.T) { client, rec := newPresenceClient(t, http.StatusOK, `{"value":[{"name":"ds","version":"1.0"}]}`, nil) - exists, absenceCertain := datasetPresence(t.Context(), client, "ds") + exists, absenceCertain, err := datasetPresence(t.Context(), client, "ds") + require.NoError(t, err) require.True(t, exists) require.False(t, absenceCertain) @@ -98,7 +102,8 @@ func TestPresenceProbesPastAListingThatHasNotCaughtUp(t *testing.T) { client, rec := newPresenceClient(t, http.StatusOK, `{"value":[]}`, map[string]bool{"1.0": true}) - exists, absenceCertain := datasetPresence(t.Context(), client, "ds") + exists, absenceCertain, err := datasetPresence(t.Context(), client, "ds") + require.NoError(t, err) require.True(t, exists, "the point read found the version the listing had not") require.False(t, absenceCertain) @@ -116,7 +121,8 @@ func TestPresenceProbesTheVersionSomethingElseRegistered(t *testing.T) { client, rec := newPresenceClient(t, http.StatusOK, `{"value":[]}`, map[string]bool{"1": true}) - exists, absenceCertain := datasetPresence(t.Context(), client, "ds") + exists, absenceCertain, err := datasetPresence(t.Context(), client, "ds") + require.NoError(t, err) require.True(t, exists) require.False(t, absenceCertain) @@ -132,7 +138,8 @@ func TestPresenceProbesTheVersionSomethingElseRegistered(t *testing.T) { func TestPresenceWillNotCallAnEmptyListingProofOfAbsence(t *testing.T) { client, _ := newPresenceClient(t, http.StatusOK, `{"value":[]}`, nil) - exists, absenceCertain := datasetPresence(t.Context(), client, "ds") + exists, absenceCertain, err := datasetPresence(t.Context(), client, "ds") + require.NoError(t, err) require.False(t, exists) require.False(t, absenceCertain, @@ -147,14 +154,65 @@ func TestPresenceTreatsA404ListingAsProofOfAbsence(t *testing.T) { client, _ := newPresenceClient(t, http.StatusNotFound, `{"error":{"code":"NotFound"}}`, nil) - exists, absenceCertain := datasetPresence(t.Context(), client, "ds") + exists, absenceCertain, err := datasetPresence(t.Context(), client, "ds") + require.NoError(t, err) require.False(t, exists) require.True(t, absenceCertain) - err := checkAssetExistence("update", "dataset", "ds", exists, absenceCertain) - require.Error(t, err) - require.Contains(t, err.Error(), `dataset "ds" does not exist`) + gateErr := checkAssetExistence("update", "dataset", "ds", exists, absenceCertain) + require.Error(t, gateErr) + require.Contains(t, gateErr.Error(), `dataset "ds" does not exist`) require.NoError(t, checkAssetExistence("create", "dataset", "ds", exists, absenceCertain), "create is exactly what a proven-absent name should allow") } + +// A read that failed proves nothing. Answering "not there" let `create` publish +// a further version of a dataset that already existed -- the one thing +// separating create from update, decided by an error nobody looked at. +func TestPresenceReportsAListingThatFailedRatherThanGuessing(t *testing.T) { + for _, status := range []int{ + http.StatusForbidden, + http.StatusUnauthorized, + http.StatusTooManyRequests, + http.StatusInternalServerError, + } { + t.Run(http.StatusText(status), func(t *testing.T) { + client, _ := newPresenceClient(t, status, `{"error":{"code":"Nope"}}`, nil) + + exists, absenceCertain, err := datasetPresence(t.Context(), client, "ds") + + require.Error(t, err, "a failed listing is not an answer about existence") + require.False(t, exists) + require.False(t, absenceCertain) + require.Contains(t, err.Error(), "ds") + }) + } +} + +// The same rule on the point read: only a 404 means "not this version". +func TestPresenceReportsAPointReadThatFailedRatherThanGuessing(t *testing.T) { + rec := &presenceServer{} + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + rec.mu.Lock() + rec.paths = append(rec.paths, r.URL.Path) + rec.mu.Unlock() + + if strings.HasSuffix(r.URL.Path, "/versions") { + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte(`{"value":[]}`)) + return + } + http.Error(w, `{"error":{"code":"Forbidden"}}`, http.StatusForbidden) + })) + t.Cleanup(srv.Close) + + client := dataset_api.NewDatasetClientFromPipeline( + srv.URL, runtime.NewPipeline("test", "v1", runtime.PipelineOptions{}, nil)) + + exists, absenceCertain, err := datasetPresence(t.Context(), client, "ds") + + require.Error(t, err) + require.False(t, exists) + require.False(t, absenceCertain) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go b/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go index 3805ef58b4b..77b26a00e86 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go @@ -837,6 +837,15 @@ func ReadingDatasetVersion(dataset, version string, err error) error { return fmt.Errorf("reading dataset %q version %s: %w", dataset, version, err) } +// CheckingDataset reports the read that decides whether a name is already +// taken. It is worth its own message because that read is what separates +// `create` from `update`, and a failure answered as "not there" turns a create +// into a silent update. +func CheckingDataset(dataset string, err error) error { + return fmt.Errorf( + "checking whether dataset %q already exists: %w", dataset, err) +} + // DatasetVersionEmpty reports a published version that holds no rows. func DatasetVersionEmpty(dataset, version string) error { return fmt.Errorf("dataset %q version %s has no rows", dataset, version) From 0f3193afbc66d2200026c450ff7e489ff2ea200b Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 18 Aug 2026 15:28:03 +0300 Subject: [PATCH 278/320] Read every page of the two listings that were reading one Two findings from a review of the client packages, both confirmed against the live service before changing anything. `job list` showed the first page and stopped. The generation-job routes answer with the same has_more/last_id cursor the OpenAI listings use, but GenerationJobList declared only `data`, so the cursor was parsed away and there was nothing to follow. Against the shared bug bash project that meant 20 of 102 dataset jobs and 20 of 31 evaluator jobs, with nothing to say the rest existed. collectPages already existed for exactly this shape and already says in its own comment that a listing stopping at the first page is a silent wrong answer; these two simply never used it. Now they do. The second is the recurring one. followPages returned a mid-walk page failure untouched, so a 404 on page two reached IsNotFound as a bare ResponseError and latestRegisteredVersion answered "no versions, and no error" -- for a dataset whose first page had just listed some. Its own comment says treating a failure as "no versions" would restart an existing dataset at 1.0; a later-page 404 was the hole in that. Page failures now carry a type that deliberately does not unwrap, because the first page answering is what proves the dataset exists. A first-page 404 still means absence, which is what lets create know the name is free, and that is pinned by its own test. Three planted defects, three caught by the intended test. Confirmed live: the two job listings now return 102 and 31. --- .../pkg/dataset_api/page_walk_test.go | 66 ++++++++++++++ .../internal/pkg/dataset_api/pages.go | 16 +++- .../eval_api/generation_job_paging_test.go | 91 +++++++++++++++++++ .../internal/pkg/eval_api/operations.go | 39 ++++++-- 4 files changed, 205 insertions(+), 7 deletions(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/page_walk_test.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/generation_job_paging_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/page_walk_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/page_walk_test.go new file mode 100644 index 00000000000..fc093e56f16 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/page_walk_test.go @@ -0,0 +1,66 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package dataset_api + +import ( + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// A 404 on the first page means the service does not know this dataset. A 404 +// on a later page means the continuation failed -- the first page already +// proved the dataset exists. Reading the second as the first answered "no +// versions, no error", which restarts an existing dataset at 1.0. +func TestALaterPageFailingIsNotAbsence(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Query().Get("page") == "2" { + http.Error(w, `{"error":{"code":"NotFound"}}`, http.StatusNotFound) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte( + `{"value":[{"name":"ds","version":"3.0"}],"nextLink":"` + + "http://" + r.Host + `/datasets/ds/versions?page=2"}`)) + })) + t.Cleanup(srv.Close) + + client := NewDatasetClientFromPipeline( + srv.URL, runtime.NewPipeline("test", "v1", runtime.PipelineOptions{}, nil)) + + _, err := client.ListDatasetVersions(t.Context(), "ds", "2025-11-15-preview") + require.Error(t, err) + assert.False(t, IsNotFound(err), + "the first page proved the dataset exists; a later 404 is the walk failing") + + version, err := client.latestRegisteredVersion(t.Context(), "ds", "2025-11-15-preview") + require.Error(t, err, "a failed walk must not answer with a version") + assert.Empty(t, version) + assert.Contains(t, strings.ToLower(err.Error()), "page") +} + +// The first page answering 404 is still absence, which is what lets a create +// know the name is free. +func TestAFirstPage404IsStillAbsence(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.Error(w, `{"error":{"code":"NotFound"}}`, http.StatusNotFound) + })) + t.Cleanup(srv.Close) + + client := NewDatasetClientFromPipeline( + srv.URL, runtime.NewPipeline("test", "v1", runtime.PipelineOptions{}, nil)) + + _, err := client.ListDatasetVersions(t.Context(), "ds", "2025-11-15-preview") + require.Error(t, err) + assert.True(t, IsNotFound(err)) + + version, err := client.latestRegisteredVersion(t.Context(), "ds", "2025-11-15-preview") + require.NoError(t, err, "an unknown dataset has no versions and that is not a failure") + assert.Empty(t, version) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/pages.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/pages.go index adaefffd1ea..6ad81b327ac 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/pages.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/pages.go @@ -23,6 +23,20 @@ import ( // nextLink cannot spin forever. const maxListPages = 100 +// pageWalkError marks a failure that happened after the first page. +// +// It deliberately does not unwrap. The first page answered, so the dataset +// exists; a 404 on a later page is the continuation failing, not the dataset +// being unknown. Left unwrapped, IsNotFound saw that 404 and +// latestRegisteredVersion answered "no versions, no error" -- which restarts an +// existing dataset at 1.0, the exact outcome its own comment says it exists to +// prevent. +type pageWalkError struct{ cause error } + +func (e pageWalkError) Error() string { + return "reading a later page of the listing: " + e.cause.Error() +} + // followPages walks nextLink until the service stops sending one, returning a // single list holding every page. Without this, a project with more than one // page lists incompletely and a latest-version check can decide from a stale @@ -47,7 +61,7 @@ func (c *DatasetClient) followPages(ctx context.Context, first *DatasetList) (*D body, err := c.doRequestGetURL(ctx, next) if err != nil { - return nil, err + return nil, pageWalkError{cause: err} } var page DatasetList // A page that answers 200 with no body ends the walk; unmarshaling it diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/generation_job_paging_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/generation_job_paging_test.go new file mode 100644 index 00000000000..c936ba71c87 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/generation_job_paging_test.go @@ -0,0 +1,91 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package eval_api + +import ( + "net/http" + "net/http/httptest" + "testing" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// The generation-job listings answer with the same has_more/last_id cursor the +// OpenAI listings use, but GenerationJobList did not carry those fields, so +// both read one page and stopped. Against the shared bug bash project that meant +// `job list` reported the first twenty jobs of many, with nothing to say so. +func TestGenerationJobListingsFollowTheCursor(t *testing.T) { + for _, tc := range []struct { + name string + path string + list func(*EvalClient) (*GenerationJobList, error) + }{ + { + name: "dataset jobs", + path: "/data_generation_jobs", + list: func(c *EvalClient) (*GenerationJobList, error) { + return c.ListDataGenerationJobs(t.Context(), "2025-11-15-preview") + }, + }, + { + name: "evaluator jobs", + path: "/evaluator_generation_jobs", + list: func(c *EvalClient) (*GenerationJobList, error) { + return c.ListEvaluatorGenerationJobs(t.Context(), "2025-11-15-preview") + }, + }, + } { + t.Run(tc.name, func(t *testing.T) { + var afters []string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + assert.Equal(t, tc.path, r.URL.Path) + after := r.URL.Query().Get("after") + afters = append(afters, after) + + w.WriteHeader(http.StatusOK) + switch after { + case "": + _, _ = w.Write([]byte( + `{"data":[{"id":"j1"},{"id":"j2"}],"has_more":true,"last_id":"j2"}`)) + case "j2": + _, _ = w.Write([]byte( + `{"data":[{"id":"j3"}],"has_more":false,"last_id":"j3"}`)) + default: + t.Errorf("unexpected cursor %q", after) + } + })) + t.Cleanup(srv.Close) + + client := NewEvalClientFromPipeline( + srv.URL, runtime.NewPipeline("test", "v1", runtime.PipelineOptions{}, nil)) + + list, err := tc.list(client) + require.NoError(t, err) + require.Len(t, list.Data, 3, "every page should be gathered, not just the first") + assert.Equal(t, []string{"", "j2"}, afters, + "the second page should be asked for with the cursor the first returned") + }) + } +} + +// A listing that fits in one page must not ask for a second. +func TestGenerationJobListingStopsWithoutACursor(t *testing.T) { + var requests int + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests++ + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte(`{"data":[{"id":"j1"}],"has_more":false,"last_id":"j1"}`)) + })) + t.Cleanup(srv.Close) + + client := NewEvalClientFromPipeline( + srv.URL, runtime.NewPipeline("test", "v1", runtime.PipelineOptions{}, nil)) + + list, err := client.ListDataGenerationJobs(t.Context(), "2025-11-15-preview") + require.NoError(t, err) + assert.Len(t, list.Data, 1) + assert.Equal(t, 1, requests) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/operations.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/operations.go index 3be46995945..94da5781dc4 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/operations.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/operations.go @@ -119,9 +119,16 @@ func (c *EvalClient) GetEvaluatorGenerationJob( } // GenerationJobList is the listing envelope both job types answer with. It is -// `data`, not the `value` the dataset and evaluator routes use. +// `data`, not the `value` the dataset and evaluator routes use, and it carries +// the same has_more/last_id cursor as the OpenAI listings. +// +// The cursor fields were missing, so both listings read one page and stopped: +// a project with more than a page of jobs answered `job list` with the first +// twenty and no sign there were more. type GenerationJobList struct { - Data []GenerationJob `json:"data"` + Data []GenerationJob `json:"data"` + HasMore bool `json:"has_more"` + LastID string `json:"last_id"` } // ListDataGenerationJobs returns the project's dataset generation jobs. @@ -129,8 +136,7 @@ func (c *EvalClient) ListDataGenerationJobs( ctx context.Context, apiVersion string, ) (*GenerationJobList, error) { - return doRequestTyped[GenerationJobList]( - c, ctx, http.MethodGet, pathDataGenerationJobs, nil, nil, apiVersion) + return c.listGenerationJobs(ctx, pathDataGenerationJobs, apiVersion) } // ListEvaluatorGenerationJobs returns the project's evaluator generation jobs. @@ -138,8 +144,29 @@ func (c *EvalClient) ListEvaluatorGenerationJobs( ctx context.Context, apiVersion string, ) (*GenerationJobList, error) { - return doRequestTyped[GenerationJobList]( - c, ctx, http.MethodGet, pathEvaluatorGenerationJobs, nil, nil, apiVersion) + return c.listGenerationJobs(ctx, pathEvaluatorGenerationJobs, apiVersion) +} + +// listGenerationJobs is the walk both job listings share. +func (c *EvalClient) listGenerationJobs( + ctx context.Context, + path string, + apiVersion string, +) (*GenerationJobList, error) { + all := &GenerationJobList{} + err := collectPages(0, func(query map[string]string) (int, bool, string, error) { + page, err := doRequestTyped[GenerationJobList]( + c, ctx, http.MethodGet, path, query, nil, apiVersion) + if err != nil { + return 0, false, "", err + } + all.Data = append(all.Data, page.Data...) + return len(page.Data), page.HasMore, page.LastID, nil + }) + if err != nil { + return nil, err + } + return all, nil } // CancelDataGenerationJob stops a dataset generation job. From 85e53978280315020dd6da8eada464486642f430 Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 18 Aug 2026 15:35:53 +0300 Subject: [PATCH 279/320] Build the test's nextLink from the server, not from the request gosec G705 flags echoing r.Host into a response body, and is right to: the test server hands back a nextLink the client then follows, which is a taint sink whatever the intent. dataset_presence_test.go already documents this trap for its point-read body; the page-walk test reintroduced it. The server's own URL says the same thing without reflecting anything, so the link is built from that instead. --- cli/azd/extensions/azure.ai.evaluations/CHANGELOG.md | 2 +- cli/azd/extensions/azure.ai.evaluations/extension.yaml | 2 +- .../internal/pkg/dataset_api/page_walk_test.go | 8 ++++++-- cli/azd/extensions/azure.ai.evaluations/version.txt | 2 +- 4 files changed, 9 insertions(+), 5 deletions(-) diff --git a/cli/azd/extensions/azure.ai.evaluations/CHANGELOG.md b/cli/azd/extensions/azure.ai.evaluations/CHANGELOG.md index deb4fb20d78..b0d81ba32a2 100644 --- a/cli/azd/extensions/azure.ai.evaluations/CHANGELOG.md +++ b/cli/azd/extensions/azure.ai.evaluations/CHANGELOG.md @@ -1,6 +1,6 @@ # Release History -## 1.0.10-beta (Unreleased) +## 1.0.11-beta (Unreleased) First release of the Foundry evaluations extension. diff --git a/cli/azd/extensions/azure.ai.evaluations/extension.yaml b/cli/azd/extensions/azure.ai.evaluations/extension.yaml index 63b6f1c66ee..2d77e0c8eab 100644 --- a/cli/azd/extensions/azure.ai.evaluations/extension.yaml +++ b/cli/azd/extensions/azure.ai.evaluations/extension.yaml @@ -5,7 +5,7 @@ displayName: Foundry evaluations (Beta) description: Define and run Foundry evaluations from your terminal. (Beta) usage: azd ai eval [options] # NOTE: Make sure version.txt is in sync with this version. -version: 1.0.10-beta +version: 1.0.11-beta requiredAzdVersion: ">=1.27.1" language: go capabilities: diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/page_walk_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/page_walk_test.go index fc093e56f16..6d69408b359 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/page_walk_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/page_walk_test.go @@ -19,7 +19,11 @@ import ( // proved the dataset exists. Reading the second as the first answered "no // versions, no error", which restarts an existing dataset at 1.0. func TestALaterPageFailingIsNotAbsence(t *testing.T) { - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // The nextLink is built from the server's own URL rather than echoed back + // from the request: reflecting r.Host into a response body is a taint sink, + // and gosec is right to refuse it even in a test. + var srv *httptest.Server + srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.URL.Query().Get("page") == "2" { http.Error(w, `{"error":{"code":"NotFound"}}`, http.StatusNotFound) return @@ -27,7 +31,7 @@ func TestALaterPageFailingIsNotAbsence(t *testing.T) { w.WriteHeader(http.StatusOK) _, _ = w.Write([]byte( `{"value":[{"name":"ds","version":"3.0"}],"nextLink":"` + - "http://" + r.Host + `/datasets/ds/versions?page=2"}`)) + srv.URL + `/datasets/ds/versions?page=2"}`)) })) t.Cleanup(srv.Close) diff --git a/cli/azd/extensions/azure.ai.evaluations/version.txt b/cli/azd/extensions/azure.ai.evaluations/version.txt index 25a15c011b0..a63e6a1b2d1 100644 --- a/cli/azd/extensions/azure.ai.evaluations/version.txt +++ b/cli/azd/extensions/azure.ai.evaluations/version.txt @@ -1 +1 @@ -1.0.10-beta +1.0.11-beta From c6305d1a5ca70e56be8bc77cd424afb3817963ff Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 18 Aug 2026 16:24:55 +0300 Subject: [PATCH 280/320] Show a score that was never produced as absent, not as a number Two findings from a review of the output and gating paths. An evaluator that errored on a row still sends a result, and its score decodes to NaN. criteriaMeans has always skipped those -- its comment says counting them as zero "would drag the average toward a number no evaluator produced" -- but the per-sample SCORE column averaged them in, so one errored evaluator made the whole sample read NaN, and the single-score column printed NaN outright. Same rule, applied in one of the three places that show a score. All three now use it, and a sample with nothing scored reads "-" rather than a computed number. The gate message rounded the actual rate and the threshold to the same one decimal while the gate compared exact values, so 7996/10000 against a 0.8 threshold breached correctly and then reported "pass rate 80.0% is below the required 80.0%". One decimal is what the spec's hero scenario shows, so it is kept: the extra precision appears only when rounding would otherwise put the two numbers on the same string. gating_test.go pins the spec's wording verbatim and still passes unchanged, which is the check that this did not drift from the spec to fix a cosmetic problem. Three planted defects, three caught by the intended test. --- .../internal/cmd/run_output.go | 24 ++++- .../internal/cmd/score_display_test.go | 91 +++++++++++++++++++ .../internal/messages/messages.go | 14 ++- 3 files changed, 123 insertions(+), 6 deletions(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/score_display_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output.go index b7ab9ee8f8c..52ddb291c71 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output.go @@ -469,21 +469,37 @@ func verdictWord(r eval_api.OutputResult) string { } // formatScore prints a judge's score at the two decimals the scale carries. +// formatScore shows a score, or a dash where there is none. An evaluator that +// errored on a row still sends a result, and its score decodes to NaN; printing +// that verbatim put "NaN" in the SCORE column. func formatScore(score eval_api.LenientFloat) string { + if !score.Defined() { + return "-" + } return strconv.FormatFloat(float64(score), 'f', 2, 64) } // meanScoreOf averages a sample's scores so the list can tell a bare pass from // a strong one. Pass/fail alone sent anyone asking "how well?" to the portal. +// +// Rows an evaluator errored on are left out rather than counted, the same rule +// criteriaMeans applies to the summary: averaging a NaN in makes the whole +// sample read NaN, and counting it as zero drags the mean toward a number no +// evaluator produced. func meanScoreOf(results []eval_api.OutputResult) string { - if len(results) == 0 { - return "-" - } total := 0.0 + scored := 0 for _, r := range results { + if !r.Score.Defined() { + continue + } total += float64(r.Score) + scored++ + } + if scored == 0 { + return "-" } - return strconv.FormatFloat(total/float64(len(results)), 'f', 2, 64) + return strconv.FormatFloat(total/float64(scored), 'f', 2, 64) } func renderResults( diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/score_display_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/score_display_test.go new file mode 100644 index 00000000000..9697156ff07 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/score_display_test.go @@ -0,0 +1,91 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "math" + "testing" + + "azureaieval/internal/pkg/eval_api" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// An evaluator that errored on a row still sends a result, and its score +// decodes to NaN. criteriaMeans has always skipped those -- its comment says +// counting them as zero "would drag the average toward a number no evaluator +// produced" -- but the per-sample column averaged them in, so one errored +// evaluator made the whole sample read NaN. +func TestASampleScoreLeavesOutWhatWasNotScored(t *testing.T) { + undefined := eval_api.LenientFloat(math.NaN()) + + cases := []struct { + name string + results []eval_api.OutputResult + want string + }{ + { + name: "no results at all", + results: nil, + want: "-", + }, + { + name: "every evaluator scored", + results: []eval_api.OutputResult{ + {Score: 1.0}, {Score: 0.5}, + }, + want: "0.75", + }, + { + name: "one evaluator errored", + results: []eval_api.OutputResult{ + {Score: 4.0}, {Score: undefined}, + }, + want: "4.00", + }, + { + name: "nothing was scored", + results: []eval_api.OutputResult{ + {Score: undefined}, {Score: undefined}, + }, + want: "-", + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + assert.Equal(t, tc.want, meanScoreOf(tc.results)) + }) + } +} + +// The same rule for a single score: a dash says "not scored", where NaN says +// nothing a reader can use. +func TestAScoreThatIsNotANumberShowsAsAbsent(t *testing.T) { + assert.Equal(t, "-", formatScore(eval_api.LenientFloat(math.NaN()))) + assert.Equal(t, "-", formatScore(eval_api.LenientFloat(math.Inf(1)))) + assert.Equal(t, "0.75", formatScore(eval_api.LenientFloat(0.75))) + assert.Equal(t, "0.00", formatScore(eval_api.LenientFloat(0)), + "a real zero is a score and must still be shown") +} + +// The gate compares exact values while the message rounds, so a rate just under +// the threshold could be reported as below itself. The spec's hero scenario +// shows one decimal, so that is kept for every case where the two differ. +func TestAGateBreachNeverReportsARateAsBelowItself(t *testing.T) { + gate, err := parseGate("pass-rate=0.8") + require.NoError(t, err) + + breach := gate.breach(&eval_api.EvalRunResultCounts{Total: 10000, Passed: 7996}) + require.NotEmpty(t, breach, "7996/10000 is under 0.8 and must breach") + assert.NotContains(t, breach, "80.0% is below the required 80.0%", + "a line saying a rate is below itself tells a reader nothing") + assert.Contains(t, breach, "79.96%") + + // The wording the spec shows is unchanged wherever rounding does not collide. + assert.Equal(t, + "pass rate 76.4% is below the required 80.0%", + gate.breach(&eval_api.EvalRunResultCounts{Total: 1000, Passed: 764})) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go b/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go index 77b26a00e86..137e891e970 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go @@ -26,6 +26,7 @@ import ( "net/http" "path/filepath" "sort" + "strconv" "strings" "time" @@ -480,9 +481,18 @@ func GateNoRowsScored() string { } // GatePassRateBelow reports a pass-rate gate that was breached. +// +// One decimal, which is what the spec's hero scenario shows -- except when that +// rounds the actual rate onto the threshold. The gate compares exact values, so +// 7996/10000 breaches 0.8 while both read "80.0%", and the line would say a +// rate is below itself. Only that case is given more precision. func GatePassRateBelow(actual, required float64) string { - return fmt.Sprintf("pass rate %.1f%% is below the required %.1f%%", - actual*100, required*100) + shown := fmt.Sprintf("%.1f", actual*100) + if shown == fmt.Sprintf("%.1f", required*100) { + shown = strconv.FormatFloat(actual*100, 'f', -1, 64) + } + return fmt.Sprintf("pass rate %s%% is below the required %.1f%%", + shown, required*100) } // GateBreached is the block a breached gate leaves in a pipeline's log. From 49d870fbccc1b09cebb16ad3e01c13ccce1b9e5d Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 18 Aug 2026 16:33:50 +0300 Subject: [PATCH 281/320] Cut the fifth bug bash build --- cli/azd/extensions/azure.ai.evaluations/CHANGELOG.md | 2 +- cli/azd/extensions/azure.ai.evaluations/extension.yaml | 2 +- cli/azd/extensions/azure.ai.evaluations/version.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cli/azd/extensions/azure.ai.evaluations/CHANGELOG.md b/cli/azd/extensions/azure.ai.evaluations/CHANGELOG.md index b0d81ba32a2..7d0e190ffbc 100644 --- a/cli/azd/extensions/azure.ai.evaluations/CHANGELOG.md +++ b/cli/azd/extensions/azure.ai.evaluations/CHANGELOG.md @@ -1,6 +1,6 @@ # Release History -## 1.0.11-beta (Unreleased) +## 1.0.12-beta (Unreleased) First release of the Foundry evaluations extension. diff --git a/cli/azd/extensions/azure.ai.evaluations/extension.yaml b/cli/azd/extensions/azure.ai.evaluations/extension.yaml index 2d77e0c8eab..226182806f1 100644 --- a/cli/azd/extensions/azure.ai.evaluations/extension.yaml +++ b/cli/azd/extensions/azure.ai.evaluations/extension.yaml @@ -5,7 +5,7 @@ displayName: Foundry evaluations (Beta) description: Define and run Foundry evaluations from your terminal. (Beta) usage: azd ai eval [options] # NOTE: Make sure version.txt is in sync with this version. -version: 1.0.11-beta +version: 1.0.12-beta requiredAzdVersion: ">=1.27.1" language: go capabilities: diff --git a/cli/azd/extensions/azure.ai.evaluations/version.txt b/cli/azd/extensions/azure.ai.evaluations/version.txt index a63e6a1b2d1..d28de92713b 100644 --- a/cli/azd/extensions/azure.ai.evaluations/version.txt +++ b/cli/azd/extensions/azure.ai.evaluations/version.txt @@ -1 +1 @@ -1.0.11-beta +1.0.12-beta From 687b4b3535cb7d7389ece213369e39f1c2669f24 Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 18 Aug 2026 17:08:00 +0300 Subject: [PATCH 282/320] Check the environment -e named wherever the endpoint comes from Three findings from a final review, the first of which is a hole in the -e fix made earlier today. The named-environment check lived inside level 2 of the endpoint cascade, and --project-endpoint answers at level 1, so the cascade never ran: azd ai eval list -e typo --project-endpoint https://... accepted azd ai eval list -e typo refused Two commands differing only by a flag disagreed about whether the same typo was an error, and on the accepted path every recorded id, version and fingerprint read as absent -- so the command behaved as though nothing had ever been deployed -- while every write went to a name azd does not have. The name decides which environment is read and written whatever supplies the endpoint, so the check belongs to none of the levels. It runs once, where the flag is read. Only azd saying it has no such environment refuses. A daemon that could not answer has not said the environment is missing, and turning that into "your environment does not exist" is the same mistake in the other direction. `azd ai eval dataset versions list ` answered "No datasets found." -- a report about the project, which may hold plenty, rather than an answer about the name asked for. The dataset extension already had the named message and the whenEmpty parameter; the evaluations copy never got them. Its command carries no placeholder, so the suggested line pastes and runs, which that module's own surface test requires. "--from-file is required (running with --no-prompt)" was printed on commands that never prompt, to callers who had not passed --no-prompt. The parenthetical named a flag they had not used and implied that dropping it would make the command ask. It is gone from both extensions. Verified live: -e with a typo now refuses identically with and without --project-endpoint, and a real environment still works. --- .../internal/cmd/dataset.go | 10 ++- .../internal/cmd/output.go | 4 +- .../azure.ai.evaluations/internal/cmd/root.go | 3 + .../internal/foundry/projectctx/resolver.go | 47 +++++++++++ .../foundry/projectctx/verify_env_test.go | 78 +++++++++++++++++++ .../internal/messages/messages.go | 21 ++++- 6 files changed, 155 insertions(+), 8 deletions(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/foundry/projectctx/verify_env_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset.go index 9eaf4e5a5a1..2478c74bd33 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset.go @@ -242,7 +242,7 @@ func newDatasetListCommand() *cobra.Command { if err != nil { return messages.ListingDatasets(err) } - return renderDatasets(cmd, list) + return renderDatasets(cmd, list, messages.NoDatasets()) }, } @@ -287,8 +287,10 @@ func newDatasetVersionsListCommand() *cobra.Command { } // An unknown name lists nothing and succeeds; it is not an error. // `-o json` callers range over the array, and `dataset delete` is - // checked for idempotence by listing what is left. - return renderDatasets(cmd, list) + // checked for idempotence by listing what is left. The empty sentence + // names the dataset, though: the project may hold plenty of others, so + // "No datasets found." would be answering a different question. + return renderDatasets(cmd, list, messages.NoDatasetVersions(name)) }, } @@ -296,7 +298,7 @@ func newDatasetVersionsListCommand() *cobra.Command { return cmd } -func renderDatasets(cmd *cobra.Command, list *dataset_api.DatasetList) error { +func renderDatasets(cmd *cobra.Command, list *dataset_api.DatasetList, whenEmpty string) error { // JSON is decided before emptiness: a caller piping this into a parser needs // an empty array, not the sentence a human would read. if list == nil { diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/output.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/output.go index aab83da518a..760844acff4 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/output.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/output.go @@ -162,8 +162,8 @@ func emitDetail(w io.Writer, fields []field) error { return tw.Flush() } -// requireFlag returns an error naming the missing flag, used when --no-prompt -// prevents asking for a required value. +// requireFlag returns an error naming a flag the command needs and has no way +// to settle for itself. func requireFlag(name string) error { return messages.FlagRequired(name) } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/root.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/root.go index 194d6c51ee6..1a015318e25 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/root.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/root.go @@ -50,6 +50,9 @@ func NewRootCommand() *cobra.Command { // reader, so there is one answer to which environment this invocation // is about. cmd.SetContext(projectctx.WithSelectedEnvironment(cmd.Context(), extCtx.Environment)) + if err := projectctx.VerifySelectedEnvironment(cmd.Context()); err != nil { + return err + } setupDebugLogging(cmd.Flags()) return nil } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/foundry/projectctx/resolver.go b/cli/azd/extensions/azure.ai.evaluations/internal/foundry/projectctx/resolver.go index b782c84be0e..99827dc9160 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/foundry/projectctx/resolver.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/foundry/projectctx/resolver.go @@ -170,6 +170,53 @@ func SelectedEnvironment(ctx context.Context) string { return name } +// envLookup is the one call needed to confirm a named environment exists. +type envLookup interface { + Get( + context.Context, *azdext.GetEnvironmentRequest, ...grpc.CallOption, + ) (*azdext.EnvironmentResponse, error) +} + +// VerifySelectedEnvironment refuses a -e/--environment azd does not have. +// +// Checked here rather than as a side effect of reading the endpoint, because +// the endpoint may not be read at all: --project-endpoint answers at level 1 +// and the cascade never runs, so `run start -e typo --project-endpoint ...` was +// accepted while the same command without the flag was refused. The name +// decides which environment every id, version and fingerprint is read from and +// written to, whichever level supplied the endpoint. +func VerifySelectedEnvironment(ctx context.Context) error { + name := SelectedEnvironment(ctx) + if name == "" { + return nil + } + client, err := azdext.NewAzdClient() + if err != nil { + // Nothing to ask: the extension is running outside azd. + return nil + } + defer client.Close() + + return verifyEnvironment(ctx, client.Environment(), name) +} + +// verifyEnvironment is the rule on its own, so it can be tested without a +// daemon. +// +// Only azd saying it has no such environment is an answer. Any other failure is +// not one, and is left to the commands that actually need azd to report, rather +// than turning a hiccup into "your environment does not exist". +func verifyEnvironment(ctx context.Context, env envLookup, name string) error { + _, err := env.Get(ctx, &azdext.GetEnvironmentRequest{Name: name}) + if err == nil { + return nil + } + if noSuchEnvironment(err) || containsGRPCCode(err, codes.NotFound) { + return ErrNoSuchEnvironment(name) + } + return nil +} + // azd's absence sentinels, as they reach us. // // `pkg/environment` and `pkg/environment/azdcontext` declare these with diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/foundry/projectctx/verify_env_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/foundry/projectctx/verify_env_test.go new file mode 100644 index 00000000000..81b866a9020 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/foundry/projectctx/verify_env_test.go @@ -0,0 +1,78 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package projectctx + +import ( + "context" + "testing" + + "github.com/azure/azure-dev/cli/azd/pkg/azdext" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +// lookupStub answers the one call that confirms an environment exists. +type lookupStub struct { + err error + asked []string +} + +func (l *lookupStub) Get( + _ context.Context, req *azdext.GetEnvironmentRequest, _ ...grpc.CallOption, +) (*azdext.EnvironmentResponse, error) { + l.asked = append(l.asked, req.Name) + if l.err != nil { + return nil, l.err + } + return &azdext.EnvironmentResponse{ + Environment: &azdext.Environment{Name: req.Name}, + }, nil +} + +// The named-environment check used to live inside the endpoint cascade, which +// --project-endpoint skips entirely: `run start -e typo --project-endpoint ...` +// was accepted while the same command without the flag was refused. The name +// decides where every id and version is read from and written to, whichever +// level supplied the endpoint, so the check does not belong to any level. +func TestAnEnvironmentAzdDoesNotHaveIsRefused(t *testing.T) { + stub := &lookupStub{ + err: status.Error(codes.Unknown, "'typo': environment not found"), + } + + err := verifyEnvironment(context.Background(), stub, "typo") + + require.Error(t, err) + assert.Contains(t, err.Error(), "typo") + assert.Equal(t, []string{"typo"}, stub.asked) +} + +func TestAnEnvironmentAzdHasIsAccepted(t *testing.T) { + stub := &lookupStub{} + + require.NoError(t, verifyEnvironment(context.Background(), stub, "staging")) + assert.Equal(t, []string{"staging"}, stub.asked) +} + +// A daemon that could not answer has not said the environment is missing. +// Refusing there would turn a hiccup into "your environment does not exist"; +// the commands that need azd report their own failures. +func TestAFailureThatIsNotAnAnswerDoesNotRefuse(t *testing.T) { + for _, err := range []error{ + status.Error(codes.Internal, "the store is on fire"), + status.Error(codes.Unavailable, "no daemon"), + status.Error(codes.Unknown, "no project exists; to create a new project, run `azd init`"), + } { + stub := &lookupStub{err: err} + assert.NoError(t, verifyEnvironment(context.Background(), stub, "staging"), + "unexpected refusal for %v", err) + } +} + +// Nothing named means azd's default, which needs no confirming. +func TestNoSelectionAsksNothing(t *testing.T) { + require.NoError(t, VerifySelectedEnvironment(context.Background())) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go b/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go index 137e891e970..00288f7eb62 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go @@ -939,6 +939,19 @@ func NoDatasets() string { return "No datasets found.\n" } +// NoDatasetVersions reports a name whose versions listed nothing. +// +// Listing a name that does not exist is not an error — a delete is checked for +// idempotence this way — so this has to read as an answer about that name +// rather than as a report about the project, which holds other datasets. +// +// The suggested command carries no placeholder, so it pastes and runs; the file +// is the one thing only the caller knows, and is named outside the command. +func NoDatasetVersions(dataset string) string { + return fmt.Sprintf("No versions of dataset %q. Publish one with "+ + "`azd ai eval dataset create %s` and a --from-file path.\n", dataset, dataset) +} + // ResolvingLatestDatasetVersion reports a failure to find what "latest" means. func ResolvingLatestDatasetVersion(dataset string, err error) error { return fmt.Errorf("resolving the latest version of %q: %w", dataset, err) @@ -2284,9 +2297,13 @@ func PortalLink(url string) string { return fmt.Sprintf("Portal: %s\n", url) } -// FlagRequired reports a value that cannot be prompted for. +// FlagRequired reports a value the command needs and cannot settle itself. +// +// It used to add "(running with --no-prompt)", which was untrue at every call +// site: none of them prompts, so the parenthetical named a flag the caller had +// not passed and implied that dropping it would make the command ask. func FlagRequired(name string) error { - return fmt.Errorf("--%s is required (running with --no-prompt)", name) + return fmt.Errorf("--%s is required", name) } // Creating reports a directory or file that could not be created. From 0a43e6a3144e39bae5547f20f09492cdbee92a47 Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 18 Aug 2026 17:12:53 +0300 Subject: [PATCH 283/320] Name what published, not a colleague who did nothing The two drift messages blamed "someone published a version outside this repo". The likeliest cause is this same extension: `azd ai eval dataset update`ndand `azd ai eval evaluator update` publish without recording the version the reconciler reads, so the next deploy sees a version it did not record and accuses a person who was never involved. The dataset message also offered `pull the newer content locally`, which is a no-op whenever the bytes already match -- the common case, since the update usually published the same file. Pinning is the remedy that works, so it leads. The checks themselves are unchanged and still fire: a version published outside the configuration is worth stopping a deploy for, whichever door published it. --- .../internal/cmd/evalref_drift_test.go | 2 +- .../internal/cmd/reconciler_drift_test.go | 4 +-- .../internal/messages/messages.go | 26 ++++++++++++++----- 3 files changed, 23 insertions(+), 9 deletions(-) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evalref_drift_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evalref_drift_test.go index 3e90e2acb83..719a58f1c17 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evalref_drift_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evalref_drift_test.go @@ -56,7 +56,7 @@ func TestCheckDatasetDriftRefusesANewerPublishedVersion(t *testing.T) { require.Error(t, err) assert.Contains(t, err.Error(), "3.0", "the version that is actually there") assert.Contains(t, err.Error(), "2.0", "and the one this repo last deployed") - assert.Contains(t, err.Error(), "outside this repo") + assert.Contains(t, err.Error(), "outside this configuration") assert.Contains(t, err.Error(), "version: 3.0", "the fix is a pin the user can paste") } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler_drift_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler_drift_test.go index d945b95eda9..488feccf38f 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler_drift_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler_drift_test.go @@ -25,8 +25,8 @@ func TestCheckEvaluatorDrift(t *testing.T) { assert.Contains(t, err.Error(), "support-quality") assert.Contains(t, err.Error(), "version 4") assert.Contains(t, err.Error(), "3 was recorded") - assert.Contains(t, err.Error(), "outside this repo", - "the message has to say who moved, not just that something did") + assert.Contains(t, err.Error(), "outside this configuration", + "the message has to say what moved, not just that something did") // A version that went backwards is not drift: a newer version was // deleted, and republishing is how the repo takes the name back. diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go b/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go index 00288f7eb62..b81cbed1df3 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go @@ -1040,12 +1040,20 @@ func DatasetVersionConflict(dataset, version string) error { dataset, version) } -// DatasetDrifted reports a version published outside the repo since the last deploy. +// DatasetDrifted reports a version published outside this configuration since +// the last deploy. +// +// `azd ai eval dataset update` publishes without recording the per-dataset +// version the reconciler reads, so it is a likely cause and naming it saves the +// reader looking for a colleague who did nothing. "Pull the newer content +// locally" was the other half of the old advice and is a no-op when the bytes +// already match, which is the common case. func DatasetDrifted(dataset, latest, recorded string) error { return fmt.Errorf( "dataset %q is at version %s on the project but %s was recorded at the last deploy; "+ - "someone published a version outside this repo. "+ - "Pin it with `version: %s` on the dataset, or pull the newer content locally, "+ + "something published outside this configuration, which `azd ai eval dataset update` "+ + "on the same dataset also does. Pin it with `version: %s` on the dataset to deploy "+ + "what is already there, or publish a new version from the configuration's source, "+ "then deploy again", dataset, latest, recorded, latest) } @@ -1251,12 +1259,18 @@ func EvaluatorNotLocalNorFound(evaluator string, err error) error { evaluator, err) } -// EvaluatorDrifted reports a version published outside the repo since the last deploy. +// EvaluatorDrifted reports a version published outside this configuration since +// the last deploy. +// +// `azd ai eval evaluator update` publishes without recording the version the +// reconciler reads, so it is a likely cause and naming it saves the reader +// looking for a colleague who did nothing. func EvaluatorDrifted(evaluator, remote, recorded string) error { return fmt.Errorf( "evaluator %q is at version %s on the project but %s was recorded at the last "+ - "deploy, and the local definition does not match it: someone published a "+ - "version outside this repo. Publishing over it would leave their change "+ + "deploy, and the local definition does not match it: something published a "+ + "version outside this configuration, which `azd ai eval evaluator update` on "+ + "the same evaluator also does. Publishing over it would leave that change "+ "behind, so bring version %s into the declared source and deploy again, or "+ "delete that version if it was a mistake", evaluator, remote, recorded, remote) From fdcb484aa209b4be012f43f448eb8e1375bc0d73 Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 18 Aug 2026 17:17:52 +0300 Subject: [PATCH 284/320] Cut the sixth bug bash build --- cli/azd/extensions/azure.ai.evaluations/CHANGELOG.md | 2 +- cli/azd/extensions/azure.ai.evaluations/extension.yaml | 2 +- cli/azd/extensions/azure.ai.evaluations/version.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cli/azd/extensions/azure.ai.evaluations/CHANGELOG.md b/cli/azd/extensions/azure.ai.evaluations/CHANGELOG.md index 7d0e190ffbc..4826099f955 100644 --- a/cli/azd/extensions/azure.ai.evaluations/CHANGELOG.md +++ b/cli/azd/extensions/azure.ai.evaluations/CHANGELOG.md @@ -1,6 +1,6 @@ # Release History -## 1.0.12-beta (Unreleased) +## 1.0.13-beta (Unreleased) First release of the Foundry evaluations extension. diff --git a/cli/azd/extensions/azure.ai.evaluations/extension.yaml b/cli/azd/extensions/azure.ai.evaluations/extension.yaml index 226182806f1..618dce84171 100644 --- a/cli/azd/extensions/azure.ai.evaluations/extension.yaml +++ b/cli/azd/extensions/azure.ai.evaluations/extension.yaml @@ -5,7 +5,7 @@ displayName: Foundry evaluations (Beta) description: Define and run Foundry evaluations from your terminal. (Beta) usage: azd ai eval [options] # NOTE: Make sure version.txt is in sync with this version. -version: 1.0.12-beta +version: 1.0.13-beta requiredAzdVersion: ">=1.27.1" language: go capabilities: diff --git a/cli/azd/extensions/azure.ai.evaluations/version.txt b/cli/azd/extensions/azure.ai.evaluations/version.txt index d28de92713b..375796cc26e 100644 --- a/cli/azd/extensions/azure.ai.evaluations/version.txt +++ b/cli/azd/extensions/azure.ai.evaluations/version.txt @@ -1 +1 @@ -1.0.12-beta +1.0.13-beta From 3fd7697eacf6a63f6f210ecb29e53298560e9419 Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 18 Aug 2026 18:14:52 +0300 Subject: [PATCH 285/320] Refuse an invalid evaluator name locally, and gate a run that errored The asset-name guard reached the dataset verbs and not the evaluator ones, so 'azd ai eval dataset show' explained itself while 'azd ai eval evaluator show' returned the service's 400 wrapped in four levels of JSON. runIsTerminal kept its own list of terminal run states, and 'error' had fallen out of it, so gating a run that had errored claimed the run was still in progress. It now reads the poller's vocabulary instead of repeating it. Both are the same shape: a rule enforced in one place and not in its sibling. The new tests walk the command tree rather than listing the verbs, so a verb added later is covered without anyone remembering to. --- .../azure.ai.evaluations/CHANGELOG.md | 2 +- .../azure.ai.evaluations/extension.yaml | 2 +- .../internal/cmd/asset_name_parity_test.go | 50 +++++++++++++++++++ .../internal/cmd/evaluator.go | 12 +++++ .../internal/cmd/gating_silent_test.go | 9 +++- .../azure.ai.evaluations/internal/cmd/run.go | 14 ++++-- .../internal/messages/messages.go | 13 ++++- .../azure.ai.evaluations/version.txt | 2 +- 8 files changed, 92 insertions(+), 12 deletions(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/asset_name_parity_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/CHANGELOG.md b/cli/azd/extensions/azure.ai.evaluations/CHANGELOG.md index 4826099f955..ccc30df61c4 100644 --- a/cli/azd/extensions/azure.ai.evaluations/CHANGELOG.md +++ b/cli/azd/extensions/azure.ai.evaluations/CHANGELOG.md @@ -1,6 +1,6 @@ # Release History -## 1.0.13-beta (Unreleased) +## 1.0.14-beta (Unreleased) First release of the Foundry evaluations extension. diff --git a/cli/azd/extensions/azure.ai.evaluations/extension.yaml b/cli/azd/extensions/azure.ai.evaluations/extension.yaml index 618dce84171..a192d331e23 100644 --- a/cli/azd/extensions/azure.ai.evaluations/extension.yaml +++ b/cli/azd/extensions/azure.ai.evaluations/extension.yaml @@ -5,7 +5,7 @@ displayName: Foundry evaluations (Beta) description: Define and run Foundry evaluations from your terminal. (Beta) usage: azd ai eval [options] # NOTE: Make sure version.txt is in sync with this version. -version: 1.0.13-beta +version: 1.0.14-beta requiredAzdVersion: ">=1.27.1" language: go capabilities: diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/asset_name_parity_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/asset_name_parity_test.go new file mode 100644 index 00000000000..f120a2cdfb3 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/asset_name_parity_test.go @@ -0,0 +1,50 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "strings" + "testing" + + "github.com/spf13/cobra" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// Every verb that takes a refuses an invalid one locally. +// +// The guard was written for the dataset verbs and never reached the evaluator +// ones, so `azd ai eval dataset show "my set"` explained itself and +// `azd ai eval evaluator show "my rubric"` returned the service's 400 wrapped +// in four levels of JSON. Walking the tree rather than listing the verbs is +// the point: a verb added later is covered without anyone remembering to. +func TestEveryNamedAssetVerbRefusesAnInvalidName(t *testing.T) { + const badName = "has space" + + var walk func(cmd *cobra.Command, path string) + checked := 0 + + walk = func(cmd *cobra.Command, path string) { + for _, sub := range cmd.Commands() { + walk(sub, strings.TrimSpace(path+" "+sub.Name())) + } + if cmd.RunE == nil || !strings.Contains(cmd.Use, "") { + return + } + + checked++ + // The guard has to come first: it runs before the client is built, so + // a mistyped name costs neither a round trip nor an azd connection. + err := cmd.RunE(cmd, []string{badName}) + require.Errorf(t, err, "%s accepted %q", path, badName) + assert.Containsf(t, err.Error(), "is invalid", + "%s refused %q, but not by naming the name", path, badName) + } + + walk(newDatasetCommand(), "dataset") + walk(newEvaluatorCommand(), "evaluator") + + assert.GreaterOrEqual(t, checked, 8, + "both command groups take a name on create, update, show, delete and versions list") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator.go index dc8679e7ec5..ac0222d9bdb 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator.go @@ -65,6 +65,9 @@ func newEvaluatorWriteCommand(verb, short string) *cobra.Command { Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { name := args[0] + if !validAssetName(name) { + return messages.InvalidEvaluatorName(name) + } if fromFile == "" { return requireFlag("from-file") } @@ -271,6 +274,9 @@ func newEvaluatorVersionsListCommand() *cobra.Command { Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { name := args[0] + if !validAssetName(name) { + return messages.InvalidEvaluatorName(name) + } ctx := cmd.Context() ec, err := newEvalContext(ctx, endpointFlg) @@ -362,6 +368,9 @@ func newEvaluatorShowCommand() *cobra.Command { Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { name := args[0] + if !validAssetName(name) { + return messages.InvalidEvaluatorName(name) + } ctx := cmd.Context() ec, err := newEvalContext(ctx, endpointFlg) @@ -467,6 +476,9 @@ func newEvaluatorDeleteCommand() *cobra.Command { Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { name := args[0] + if !validAssetName(name) { + return messages.InvalidEvaluatorName(name) + } if version == "" { return requireFlag("version") } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/gating_silent_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/gating_silent_test.go index 7b1699a6828..7a201f5c909 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/gating_silent_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/gating_silent_test.go @@ -79,10 +79,15 @@ func TestGatesStillJudgeRunsThatScoredSomething(t *testing.T) { // and silently skipping the gate would leave a pipeline believing it is // protected when it is not. func TestOnlyATerminalRunCanBeGated(t *testing.T) { - for _, status := range []string{"completed", "failed", "canceled", "cancelled", ""} { + // Read from the polling vocabulary instead of a second copy of it. Spelling + // the list out here is what hid "error" being gateable: the test agreed with + // the bug. + for status := range terminalRunStates { assert.Truef(t, runIsTerminal(&eval_api.OpenAIEvalRun{Status: status}), - "%q has stopped moving, so its counts are final", status) + "the poller stops on %q, so its counts are final", status) } + assert.True(t, runIsTerminal(&eval_api.OpenAIEvalRun{Status: ""}), + "a run the service reported no status for is gated on the counts it gave") for _, status := range []string{"in_progress", "queued", "running"} { assert.Falsef(t, runIsTerminal(&eval_api.OpenAIEvalRun{Status: status}), diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go index 5452a7f0402..72f7af4aed5 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go @@ -60,15 +60,19 @@ func runCompleted(run *eval_api.OpenAIEvalRun) error { // A gate read from a run still in progress is read from partial counts: it can // fail a run that would have passed, and it can pass one that has not finished // failing. +// +// Derived from the polling vocabulary rather than repeating it: a state the +// poller stops waiting on is a state whose counts are final, and keeping two +// lists let "error" fall out of this one -- which told anyone gating an errored +// run to pass --wait, on a run that had already stopped. The empty status is +// the one deliberate difference: polling keeps waiting on a run the service has +// not described yet, while a gate reads the counts it was handed. func runIsTerminal(run *eval_api.OpenAIEvalRun) bool { if run == nil { return false } - switch strings.ToLower(run.Status) { - case "completed", "failed", "canceled", "cancelled", "": - return true - } - return false + status := strings.ToLower(run.Status) + return status == "" || terminalRunStates[status] } // newRunCommand builds the run group. diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go b/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go index b81cbed1df3..7327a5bc5dd 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go @@ -909,9 +909,18 @@ func FromFileDirectoryIsAmbiguous(dir string, names []string) error { // InvalidDatasetName reports a name the service will not accept. func InvalidDatasetName(name string) error { + return invalidAssetName("dataset", name) +} + +// InvalidEvaluatorName reports a name the service will not accept. +func InvalidEvaluatorName(name string) error { + return invalidAssetName("evaluator", name) +} + +func invalidAssetName(kind, name string) error { return fmt.Errorf( - "dataset name %q is invalid: use letters, digits, dashes and underscores, "+ - "up to 255 characters", name) + "%s name %q is invalid: use letters, digits, dashes and underscores, "+ + "up to 255 characters", kind, name) } // RegisteringDataset reports the service refusing to publish the dataset. diff --git a/cli/azd/extensions/azure.ai.evaluations/version.txt b/cli/azd/extensions/azure.ai.evaluations/version.txt index 375796cc26e..f8730a057a6 100644 --- a/cli/azd/extensions/azure.ai.evaluations/version.txt +++ b/cli/azd/extensions/azure.ai.evaluations/version.txt @@ -1 +1 @@ -1.0.13-beta +1.0.14-beta From aec7333217bc7d1f7c2201a87a2e4fa5705b584f Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 18 Aug 2026 22:37:40 +0300 Subject: [PATCH 286/320] Address Copilot review of the split slices Copilot has never reviewed #9500 -- it exceeds the 20,000 line limit and declined 110 times -- so the PR was split into two temporary slices to get a review. These are the findings that held up. renderDatasets took a whenEmpty message and printed a different one, so the fix that added the parameter changed nothing and no test noticed: listing the versions of an unknown dataset still reported on the whole project. A non-404 failure reading an evaluator was treated as the evaluator being absent, publishing a new version with no drift check. Line 501 of the same file already did this correctly. yaml.Node.Decode does not inherit KnownFields from the decoder that reached it, so a misspelt key inside an evaluator entry was dropped in silence while the same misspelling one level up was named. A pinned verison: that does nothing grades against whatever version is latest and reports success. pageWalkError dropped Unwrap to stop a later-page 404 reading as absence, which also stopped a cancelled walk looking like a cancellation. IsNotFound now refuses the wrapper explicitly and the cause stays reachable. A lock that cannot be taken is advisory by design, but being cancelled is not the lock being busy: Ctrl-C went on to rewrite the configuration. build.ps1 stamped local time with a literal Z, and the changelog advertised a results command group that does not exist. --- .../azure.ai.evaluations/CHANGELOG.md | 2 +- .../extensions/azure.ai.evaluations/build.ps1 | 2 +- .../internal/cmd/dataset.go | 2 +- .../cmd/dataset_empty_message_test.go | 44 +++++++++++++ .../internal/cmd/reconciler.go | 11 +++- .../internal/pkg/dataset_api/operations.go | 7 +++ .../pkg/dataset_api/page_walk_test.go | 18 ++++++ .../internal/pkg/dataset_api/pages.go | 13 ++-- .../internal/pkg/evalcore/evaluator.go | 51 ++++++++++++++- .../internal/project/eval_config_keys.go | 4 ++ .../internal/project/eval_config_lock.go | 7 +++ .../project/unknown_keys_depth_test.go | 62 +++++++++++++++++++ 12 files changed, 209 insertions(+), 14 deletions(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset_empty_message_test.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/project/unknown_keys_depth_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/CHANGELOG.md b/cli/azd/extensions/azure.ai.evaluations/CHANGELOG.md index ccc30df61c4..1a0e537e970 100644 --- a/cli/azd/extensions/azure.ai.evaluations/CHANGELOG.md +++ b/cli/azd/extensions/azure.ai.evaluations/CHANGELOG.md @@ -21,7 +21,7 @@ First release of the Foundry evaluations extension. keys the author wrote, and eval groups are recreated only when their own declaration changes. - Atomic commands for every operation: `dataset`, `evaluator`, `run` and - `results` subcommands, all supporting `-o json` and `--no-prompt`. + `run output` subcommands, all supporting `-o json` and `--no-prompt`. - Testing criteria are shaped from each evaluator's published contract, so evaluators requiring inputs beyond the agent shape — `ground_truth`, `context`, `instruction_id_list` — work by binding them to dataset columns. diff --git a/cli/azd/extensions/azure.ai.evaluations/build.ps1 b/cli/azd/extensions/azure.ai.evaluations/build.ps1 index f37f80cabf0..e016344f65b 100644 --- a/cli/azd/extensions/azure.ai.evaluations/build.ps1 +++ b/cli/azd/extensions/azure.ai.evaluations/build.ps1 @@ -24,7 +24,7 @@ if ($LASTEXITCODE -ne 0) { Write-Host "Error: Failed to get git commit hash" exit 1 } -$BUILD_DATE = (Get-Date -Format "yyyy-MM-ddTHH:mm:ssZ") +$BUILD_DATE = ((Get-Date).ToUniversalTime().ToString("yyyy-MM-ddTHH:mm:ssZ")) # List of OS and architecture combinations if ($env:EXTENSION_PLATFORM) { diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset.go index 2478c74bd33..055bb60e1ac 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset.go @@ -312,7 +312,7 @@ func renderDatasets(cmd *cobra.Command, list *dataset_api.DatasetList, whenEmpty rows = append(rows, []string{d.Name, d.Version, d.Type}) } if len(rows) == 0 { - fmt.Fprint(cmd.OutOrStdout(), messages.NoDatasets()) + fmt.Fprint(cmd.OutOrStdout(), whenEmpty) return nil } // TYPE, not FORMAT: the service populates type (`uri_file`) and leaves diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset_empty_message_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset_empty_message_test.go new file mode 100644 index 00000000000..388c8d21106 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset_empty_message_test.go @@ -0,0 +1,44 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "bytes" + "testing" + + "azureaieval/internal/messages" + "azureaieval/internal/pkg/dataset_api" + + "github.com/spf13/cobra" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// An empty listing says which listing was empty. +// +// The caller already passed the sentence to print, and this printed a different +// one: `dataset versions list ` answered "No datasets found", a report +// about the whole project, for a name that simply had no versions. The +// parameter was threaded through and then ignored, so the fix that added it +// changed nothing and no test noticed. +func TestEmptyDatasetListingUsesTheCallersMessage(t *testing.T) { + cases := []struct { + name string + whenEmpty string + }{ + {"whole project", messages.NoDatasets()}, + {"one dataset's versions", messages.NoDatasetVersions("golden")}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + var out bytes.Buffer + cmd := &cobra.Command{} + cmd.SetOut(&out) + + require.NoError(t, renderDatasets(cmd, &dataset_api.DatasetList{}, tc.whenEmpty)) + assert.Equal(t, tc.whenEmpty, out.String()) + }) + } +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go index 630289cc78e..521f5210022 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go @@ -294,9 +294,16 @@ func (r *evalReconciler) EnsureEvaluator( // Compare against the definition already on the service. var known json.RawMessage - if existing, err := r.ec.evalClient.GetEvaluatorRaw( + existing, err := r.ec.evalClient.GetEvaluatorRaw( ctx, decl.Name, "", ProjectEndpointAPIVersion, - ); err == nil { + ) + // A read that failed is not a read that found nothing: falling through + // publishes a new version with no drift check, over whatever is already + // there. Only a confirmed absence is a first publish. + if err != nil && !eval_api.IsNotFound(err) { + return "", false, messages.CheckingEvaluatorExists(decl.Name, err) + } + if err == nil { remote := versionFromRaw(existing, "") if !authorEdited && sameDefinition(existing, body) { // Nothing to publish, but the version is still worth recording: diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/operations.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/operations.go index 93b178288c7..4e287c5ec21 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/operations.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/operations.go @@ -188,7 +188,14 @@ func IsVersionConflict(err error) bool { // IsNotFound reports whether the service answered 404, which is how it says a // dataset does not exist yet. +// +// A failure part-way through a page walk is refused before the status is read: +// the first page answered, so the dataset exists, and reading that 404 as +// absence restarts an existing dataset at 1.0. func IsNotFound(err error) bool { + if _, walking := errors.AsType[pageWalkError](err); walking { + return false + } respErr, ok := errors.AsType[*azcore.ResponseError](err) if !ok { return false diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/page_walk_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/page_walk_test.go index 6d69408b359..251214b7382 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/page_walk_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/page_walk_test.go @@ -4,6 +4,8 @@ package dataset_api import ( + "context" + "errors" "net/http" "net/http/httptest" "strings" @@ -14,6 +16,22 @@ import ( "github.com/stretchr/testify/require" ) +// Refusing IsNotFound must not cost the rest of the error's identity. +// +// The first version of this guard dropped Unwrap altogether, which did stop a +// later-page 404 reading as absence but also made a cancelled walk stop looking +// like a cancellation to everything upstream. +func TestAPageWalkFailureKeepsItsCause(t *testing.T) { + wrapped := pageWalkError{cause: context.Canceled} + + assert.True(t, errors.Is(wrapped, context.Canceled), + "a walk cancelled part-way through is still a cancellation") + assert.False(t, IsNotFound(wrapped), + "the first page answered, so the dataset is not missing") + assert.Contains(t, wrapped.Error(), "later page", + "and the message says which part of the listing failed") +} + // A 404 on the first page means the service does not know this dataset. A 404 // on a later page means the continuation failed -- the first page already // proved the dataset exists. Reading the second as the first answered "no diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/pages.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/pages.go index 6ad81b327ac..a220e1f7d83 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/pages.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/pages.go @@ -25,18 +25,19 @@ const maxListPages = 100 // pageWalkError marks a failure that happened after the first page. // -// It deliberately does not unwrap. The first page answered, so the dataset -// exists; a 404 on a later page is the continuation failing, not the dataset -// being unknown. Left unwrapped, IsNotFound saw that 404 and -// latestRegisteredVersion answered "no versions, no error" -- which restarts an -// existing dataset at 1.0, the exact outcome its own comment says it exists to -// prevent. +// The first page answered, so the dataset exists; a 404 on a later page is the +// continuation failing, not the dataset being unknown. IsNotFound refuses this +// wrapper for that reason -- but the cause is still reachable, so a cancelled +// context or an auth failure part-way through a walk classifies as itself +// rather than as an unreadable listing. type pageWalkError struct{ cause error } func (e pageWalkError) Error() string { return "reading a later page of the listing: " + e.cause.Error() } +func (e pageWalkError) Unwrap() error { return e.cause } + // followPages walks nextLink until the service stops sending one, returning a // single list holding every page. Without this, a project with more than one // page lists incompletely and a latest-version check can decide from a stale diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/evaluator.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/evaluator.go index 6ca405b2973..d9bd6715a6c 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/evaluator.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/evaluator.go @@ -6,6 +6,10 @@ package evalcore import ( "bytes" "encoding/json" + "errors" + "fmt" + "regexp" + "strconv" "strings" "azureaieval/internal/messages" @@ -90,9 +94,9 @@ func (el *EvaluatorList) UnmarshalYAML(value *yaml.Node) error { } return messages.BareEvaluatorEntry(name) case yaml.MappingNode: - var ref EvaluatorRef - if err := node.Decode(&ref); err != nil { - return messages.DecodingEvaluator(err) + ref, err := decodeEvaluatorRef(node) + if err != nil { + return err } if ref.Evaluator == "" { return messages.EvaluatorEntryMissingEvaluator() @@ -107,6 +111,47 @@ func (el *EvaluatorList) UnmarshalYAML(value *yaml.Node) error { return nil } +// decodeEvaluatorRef decodes one entry with the strictness the file promises. +// +// yaml.Node.Decode does not inherit KnownFields from the decoder that reached +// it, so `verison:` inside an evaluator entry was dropped in silence while the +// same typo one level up was named. Round-tripping the node through a strict +// decoder restores it; the error keeps yaml's own "field X not found in type Y" +// shape, which the caller rewrites into the file's vocabulary. +func decodeEvaluatorRef(node *yaml.Node) (EvaluatorRef, error) { + raw, err := yaml.Marshal(node) + if err != nil { + return EvaluatorRef{}, messages.DecodingEvaluator(err) + } + + decoder := yaml.NewDecoder(bytes.NewReader(raw)) + decoder.KnownFields(true) + + var ref EvaluatorRef + if err := decoder.Decode(&ref); err != nil { + return EvaluatorRef{}, messages.DecodingEvaluator(rebaseYAMLLines(err, node.Line)) + } + return ref, nil +} + +// yamlErrorLine matches the line number yaml puts on each unmarshal error. +var yamlErrorLine = regexp.MustCompile(`line (\d+):`) + +// rebaseYAMLLines moves line numbers from the extracted snippet back onto the +// file, so the reader is pointed at the key they typed rather than at line 2. +func rebaseYAMLLines(err error, startLine int) error { + if startLine <= 0 { + return err + } + return errors.New(yamlErrorLine.ReplaceAllStringFunc(err.Error(), func(m string) string { + n, convErr := strconv.Atoi(yamlErrorLine.FindStringSubmatch(m)[1]) + if convErr != nil { + return m + } + return fmt.Sprintf("line %d:", startLine+n-1) + })) +} + // UnmarshalJSON accepts the same mapping-only form as the YAML decoder. // // This matters for the service-target provider: azd hands the service entry to diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_keys.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_keys.go index 8eae2b01cd7..6f91532502f 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_keys.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_keys.go @@ -9,6 +9,8 @@ import ( "reflect" "regexp" "strings" + + "azureaieval/internal/pkg/evalcore" ) // goTypeInField matches what yaml.KnownFields reports for an unrecognized key: @@ -97,6 +99,8 @@ func keysOfType(goType string) []string { v = Target{} case "project.SourceDecl": v = SourceDecl{} + case "evalcore.EvaluatorRef": + v = evalcore.EvaluatorRef{} default: return nil } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_lock.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_lock.go index 730b1a25449..a35fa754b5c 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_lock.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_lock.go @@ -63,6 +63,13 @@ func LockEvalConfig(ctx context.Context, evalDir string) (func(), error) { locked, err := lock.TryLockContext(waitCtx, 50*time.Millisecond) if err != nil || !locked { + // Being cancelled is not the same as the lock being busy. The advisory + // behaviour below exists so a held lock cannot fail a scaffold; carrying + // it into Ctrl-C would go on to rewrite the configuration the user just + // asked to stop. + if ctxErr := ctx.Err(); ctxErr != nil { + return nil, ctxErr + } fmt.Fprint(os.Stderr, messages.Warning(messages.ConfigLockUnavailable(evalDir, err))) return func() {}, nil } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/unknown_keys_depth_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/unknown_keys_depth_test.go new file mode 100644 index 00000000000..e8c9dc47cf4 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/unknown_keys_depth_test.go @@ -0,0 +1,62 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package project + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// A typo is named wherever it appears, not only at the top level. +// +// yaml.Node.Decode does not inherit KnownFields from the decoder that reached +// it, so a misspelt key inside an evaluator entry was dropped in silence while +// the same misspelling one level up was reported. A pinned `verison` that does +// nothing is worse than one that is refused: the run grades against whatever +// version happens to be latest and reports success. +func TestUnknownKeysAreNamedAtEveryDepth(t *testing.T) { + cases := []struct { + where string + body string + key string + nearer string + line string + }{ + { + where: "top level of an eval", + body: "evals:\n - name: e1\n datasett: golden\n", + key: "datasett", + nearer: "dataset", + line: "line 3", + }, + { + where: "inside an evaluator entry", + body: "evals:\n - name: e1\n evaluators:\n - evaluator: builtin.x\n verison: \"3\"\n", + key: "verison", + nearer: "version", + line: "line 5", + }, + { + where: "inside a dataset declaration", + body: "datasets:\n - name: golden\n sourse: ./rows.jsonl\n", + key: "sourse", + nearer: "source", + line: "line 3", + }, + } + + for _, tc := range cases { + t.Run(tc.where, func(t *testing.T) { + _, err := DecodeEvalConfig([]byte(tc.body), "azure.eval.yaml") + require.Errorf(t, err, "%q was accepted in silence", tc.key) + + assert.Contains(t, err.Error(), tc.key, "the message has to name the key") + assert.Contains(t, err.Error(), tc.nearer, "and the key it was probably meant to be") + assert.Contains(t, err.Error(), tc.line, + "pointing at the line in the file, not inside an extracted fragment") + }) + } +} From e6bd4f88e47997cb16a1b269ee0c542e0ccc7a36 Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 18 Aug 2026 22:44:53 +0300 Subject: [PATCH 287/320] Keep the instruction_file pointer inside the project The optimize metadata is read out of the checkout, so its instruction_file pointer carries the checkout trust level. An absolute path, or one climbing out with .., read a file the project does not contain and passed it on as agent instructions -- so cloning a repository and running generate could send a named local file to the service. Contained to the project rather than to the baseline directory, so any layout inside the project a writer chooses still resolves. --- .../internal/messages/messages.go | 11 ++++++ .../internal/project/agent_instructions.go | 21 +++++++++- .../project/instruction_containment_test.go | 39 +++++++++++++++++++ 3 files changed, 70 insertions(+), 1 deletion(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/project/instruction_containment_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go b/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go index 7327a5bc5dd..d9a142b3017 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go @@ -2124,6 +2124,17 @@ func InstructionFileUnreadable(metadataPath, named string, err error) error { metadataPath, named, err) } +// InstructionFileOutsideProject reports metadata pointing outside the project. +// +// The pointer is read from a file in the checkout, so it is only as trustworthy +// as the checkout: an absolute path or one climbing out with `..` would read +// something the project does not contain and send it on as agent instructions. +func InstructionFileOutsideProject(metadataPath, named string) error { + return fmt.Errorf( + "%s names instruction_file %q, which is outside the project; "+ + "name a path inside it", metadataPath, named) +} + // FromNotASource reports a --from value the generation service has no path for. func FromNotASource(from string, sources []string) error { return fmt.Errorf( diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/agent_instructions.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/agent_instructions.go index 3d80db3333c..8c2b397ccb4 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/agent_instructions.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/agent_instructions.go @@ -85,7 +85,14 @@ func AgentInstructionsFromProject( if !filepath.IsAbs(instructionPath) { instructionPath = filepath.Join(configDir, instructionPath) } - text, err := os.ReadFile(instructionPath) //nolint:gosec // named by the metadata beside it + // The pointer comes out of the checkout, so it carries the checkout's + // trust. Left alone, an absolute path or one climbing out with `..` reads a + // file the project does not contain and sends it on as agent instructions. + if !withinDir(proj.GetPath(), instructionPath) { + return "", "", messages.InstructionFileOutsideProject( + filepath.Join(configDir, agentMetadataFile), meta.InstructionFile) + } + text, err := os.ReadFile(instructionPath) //nolint:gosec // checked to be inside the project if err != nil { // The metadata named a file that is not there. That is worth saying: // something wrote the pointer and not the target. @@ -96,6 +103,18 @@ func AgentInstructionsFromProject( return strings.TrimSpace(string(text)), instructionPath, nil } +// withinDir reports whether path resolves to somewhere inside root. +// +// Compared after cleaning both, so `..` segments are resolved before the +// question is asked rather than matched as text. +func withinDir(root, path string) bool { + rel, err := filepath.Rel(filepath.Clean(root), filepath.Clean(path)) + if err != nil { + return false + } + return rel != ".." && !strings.HasPrefix(rel, ".."+string(filepath.Separator)) +} + // findAgentService resolves a target name to the one service that is it. // // A name can match either the azure.yaml service key or the agent name the diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/instruction_containment_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/instruction_containment_test.go new file mode 100644 index 00000000000..2aa6c78def5 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/instruction_containment_test.go @@ -0,0 +1,39 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package project + +import ( + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" +) + +// The optimize metadata's instruction_file pointer is only as trustworthy as +// the checkout it was read from. Left unchecked, cloning a repository and +// running generate would read a named local file and send it on as agent +// instructions. +func TestInstructionPointerCannotLeaveTheProject(t *testing.T) { + root := filepath.Join(string(filepath.Separator), "work", "proj") + + inside := []string{ + filepath.Join(root, "instructions.md"), + filepath.Join(root, "src", "agent", ".agent_configs", "baseline", "i.md"), + filepath.Join(root, "a", "..", "b", "i.md"), + root, + } + for _, p := range inside { + assert.Truef(t, withinDir(root, p), "%q is inside the project", p) + } + + outside := []string{ + filepath.Join(root, "..", "other", "secrets.txt"), + filepath.Join(root, "..", "..", "etc", "passwd"), + filepath.Join(string(filepath.Separator), "etc", "passwd"), + filepath.Join(root+"-sibling", "i.md"), // prefix match, different directory + } + for _, p := range outside { + assert.Falsef(t, withinDir(root, p), "%q is outside the project", p) + } +} From 2dbb433da9d337f9d7e5df9a727550d5023bd6e5 Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 18 Aug 2026 23:38:13 +0300 Subject: [PATCH 288/320] Finish the Copilot review of the split slices Empty listings emitted null rather than the bare array every other list command promises; emitJSONList exists for exactly that and run output list was the one caller not using it. An evaluator that returned no verdict was named among the ones a sample failed, which states something about the sample nothing measured. The row still shows -- it is worth looking at -- but the two are now held apart, and --failed-only filters on the same predicate -o json already used. csv.Writer buffers, so a disk that filled showed up only in Error() after the final Flush. Deferring the flush and returning nil reported success over a file that had not finished writing. A portal prefix was built from any nested ARM resource, because the check was "has a parent and a slash in its type" -- true of a storage container. Names are escaped into portal paths now: they come from the service, not from this extension, so a space or slash produced a broken or wrong link. init reported the wiring present whenever a service matched by name and host, without comparing its $ref, so scaffolding elsewhere with --path left azure.yaml deploying the file that was left behind. A truncated page walk said so through log, which goes to io.Discard unless --debug -- the mistake eval_config_lock.go documents having made. A short evaluator listing resolves an older version and reports nothing unusual. The live cancel test skipped on any cancel error, including one raised while the run was still moving, which retires the test without anyone deciding to. --- .../azure.ai.evaluations/internal/cmd/init.go | 28 +++++++++- .../internal/cmd/run_output.go | 50 ++++++++++++++--- .../internal/messages/messages.go | 19 +++++++ .../internal/pkg/dataset_api/pages.go | 7 ++- .../internal/pkg/eval_api/pages.go | 4 +- .../pkg/eval_api/portal_scope_test.go | 56 +++++++++++++++++++ .../internal/pkg/eval_api/portal_urls.go | 26 +++++++-- .../tests/live/run_cancel_test.go | 17 ++++++ 8 files changed, 189 insertions(+), 18 deletions(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/portal_scope_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go index 1f44c84c836..0182326e691 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go @@ -763,12 +763,21 @@ func ensureRootEvalService( // A service already pointing at this configuration is left alone: // re-adding it would deploy the same evals twice. + // + // Pointing at a different one is not the same thing. Matching on name and + // host alone reported the wiring present after `init --path` moved the + // configuration, and `azd up` went on deploying the file that was left + // behind -- the scaffold the reader was looking at was never deployed. + wantRef := "./" + filepath.ToSlash(configPath) if svc, ok := resp.GetProject().GetServices()[serviceName]; ok && svc.GetHost() == project.EvalHost { + if have := serviceConfigRef(svc); have != "" && !sameRefTarget(have, wantRef) { + return "", messages.ServiceRefPointsElsewhere(serviceName, have, wantRef) + } return wiringPresent, nil } props, err := structpb.NewStruct(map[string]any{ - "$ref": "./" + filepath.ToSlash(configPath), + "$ref": wantRef, }) if err != nil { return "", messages.BuildingServiceEntry(err) @@ -788,6 +797,23 @@ func ensureRootEvalService( return wiringAdded, nil } +// serviceConfigRef reads the $ref a service entry was authored with, or empty +// when it holds its configuration inline. +func serviceConfigRef(svc *azdext.ServiceConfig) string { + props := svc.GetAdditionalProperties() + if props == nil { + return "" + } + ref, _ := props.AsMap()["$ref"].(string) + return ref +} + +// sameRefTarget compares two $ref values as paths rather than as text, so +// `evals/azure.eval.yaml` and `./evals/azure.eval.yaml` are one answer. +func sameRefTarget(a, b string) bool { + return filepath.Clean(filepath.FromSlash(a)) == filepath.Clean(filepath.FromSlash(b)) +} + // evalServiceUses orders the eval after the things it reads. // // It is conditional for the same reason the agents extension makes it diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output.go index 52ddb291c71..a897bb9a811 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output.go @@ -95,7 +95,7 @@ func newRunOutputListCommand() *cobra.Command { if err != nil { return messages.Creating(outFile, err) } - if err := emitJSON(f, rows); err != nil { + if err := emitJSONList(f, rows); err != nil { _ = f.Close() return err } @@ -107,7 +107,7 @@ func newRunOutputListCommand() *cobra.Command { return nil } if isJSON(cmd) { - return emitJSON(cmd.OutOrStdout(), rows) + return emitJSONList(cmd.OutOrStdout(), rows) } return renderResults(cmd.OutOrStdout(), run, rows, failedOnly) }, @@ -548,21 +548,41 @@ func renderResults( // One row per evaluated sample, not per verdict: a sample that // failed three evaluators is one sample to go and look at, and // listing it three times buries how much is actually wrong. - var failed []string + // + // An evaluator that returned no verdict is held apart from one that + // returned a failing verdict. Both keep the row, because the row is + // still worth looking at, but naming an errored evaluator among the + // ones the sample failed states something about the sample that + // nothing measured. + var failed, unjudged []string reason := "" for _, r := range it.Results { - if r.DidPass() { + switch { + case !r.Judged(): + unjudged = append(unjudged, r.Name) + case r.DidPass(): continue + default: + failed = append(failed, r.Name) } - failed = append(failed, r.Name) if reason == "" { reason = r.Reason } } - if failedOnly && len(failed) == 0 { + // The same predicate `-o json` filters on, so the two views of + // --failed-only cannot disagree about which rows went wrong. + if failedOnly && !it.Failed() { continue } verdicts := strings.Join(failed, ", ") + if len(unjudged) > 0 { + note := strings.Join(unjudged, ", ") + " (no verdict)" + if verdicts == "" { + verdicts = note + } else { + verdicts += "; " + note + } + } if verdicts == "" { verdicts = "-" } @@ -607,7 +627,6 @@ func truncate(s string, n int) string { func writeResultsCSV(w io.Writer, run *eval_api.OpenAIEvalRun) error { cw := csv.NewWriter(w) - defer cw.Flush() // Named as the service names it, and as the jsonl export already did, so a // pipeline reading both formats needs one spelling rather than two. @@ -615,7 +634,10 @@ func writeResultsCSV(w io.Writer, run *eval_api.OpenAIEvalRun) error { return err } if len(run.PerTestingCriteria) == 0 { - return cw.Write([]string{run.ID, run.Status, "", "", ""}) + if err := cw.Write([]string{run.ID, run.Status, "", "", ""}); err != nil { + return err + } + return flushCSV(cw) } for _, cr := range run.PerTestingCriteria { if err := cw.Write([]string{ @@ -625,7 +647,17 @@ func writeResultsCSV(w io.Writer, run *eval_api.OpenAIEvalRun) error { return err } } - return nil + return flushCSV(cw) +} + +// flushCSV reports what the buffer swallowed. +// +// csv.Writer buffers, so a disk that filled or a pipe that closed shows up only +// in Error() after the final Flush. Deferring the flush and returning nil made +// `run output export` report success over a file it had not finished writing. +func flushCSV(cw *csv.Writer) error { + cw.Flush() + return cw.Error() } // Export formats. csv is the default because the results are a table and a diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go b/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go index d9a142b3017..ac28c0274ba 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go @@ -2124,6 +2124,25 @@ func InstructionFileUnreadable(metadataPath, named string, err error) error { metadataPath, named, err) } +// ListingTruncated reports a page walk that stopped before the end. +// +// Worth saying out loud rather than logging: a short evaluator listing resolves +// the latest version from the pages that arrived, so a truncated one can pick +// an older version and report nothing unusual. +func ListingTruncated(pages int) error { + return fmt.Errorf( + "stopped reading the listing after %d pages, so it may be incomplete", pages) +} + +// ServiceRefPointsElsewhere reports an existing service entry wired to a +// different configuration than the one just scaffolded. +func ServiceRefPointsElsewhere(serviceName, have, want string) error { + return fmt.Errorf( + "service %q already points at %s, and the configuration just written is "+ + "%s; point the service's $ref at the one you want, or scaffold with "+ + "--path %s", serviceName, have, want, have) +} + // InstructionFileOutsideProject reports metadata pointing outside the project. // // The pointer is read from a file in the checkout, so it is only as trustworthy diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/pages.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/pages.go index a220e1f7d83..7027507f37a 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/pages.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/pages.go @@ -7,10 +7,12 @@ import ( "bytes" "context" "encoding/json" + "fmt" "io" "log" "net/http" "net/url" + "os" "strings" "azureaieval/internal/messages" @@ -54,8 +56,9 @@ func (c *DatasetClient) followPages(ctx context.Context, first *DatasetList) (*D for next := first.NextLink; next != ""; { if seen[next] || len(seen) >= maxListPages { // A repeated or endless link is the service misbehaving, not a reason - // to fail the command, but the list is short and nobody would know. - log.Printf("[dataset_api] stopped paging after %d pages; the listing may be incomplete", len(seen)) + // to fail the command -- but the list is short and, said through log, + // nobody would know: log goes to io.Discard unless --debug. + fmt.Fprint(os.Stderr, messages.Warning(messages.ListingTruncated(len(seen)))) break } seen[next] = true diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/pages.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/pages.go index 8b3e4200804..70cb023e17f 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/pages.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/pages.go @@ -7,10 +7,12 @@ import ( "bytes" "context" "encoding/json" + "fmt" "io" "log" "net/http" "net/url" + "os" "strings" "azureaieval/internal/messages" @@ -98,7 +100,7 @@ func walkNextLinks[T any]( // A repeated link, not just a self-referencing one, ends the walk: a // two-page cycle would otherwise run to maxPages for no benefit. if seen[link] || len(seen) >= maxPages { - log.Printf("[eval_api] stopped paging after %d pages; the listing may be incomplete", len(seen)) + fmt.Fprint(os.Stderr, messages.Warning(messages.ListingTruncated(len(seen)))) break } seen[link] = true diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/portal_scope_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/portal_scope_test.go new file mode 100644 index 00000000000..f4b642ee524 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/portal_scope_test.go @@ -0,0 +1,56 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package eval_api + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const testSub = "/subscriptions/00000000-0000-0000-0000-000000000000" + +// Only a Foundry project builds a portal prefix. +// +// The check was "has a parent and a slash in its type", which every nested +// resource satisfies: a storage blob container reached this far and produced a +// confident link to a portal page that cannot exist. +func TestOnlyAFoundryProjectBuildsAPortalPrefix(t *testing.T) { + project := testSub + "/resourceGroups/rg/providers/" + + "Microsoft.CognitiveServices/accounts/acct/projects/proj" + _, err := NewPortalPrefix(project) + require.NoError(t, err, "a Foundry project is the thing this is for") + + notProjects := map[string]string{ + "a storage container": testSub + "/resourceGroups/rg/providers/" + + "Microsoft.Storage/storageAccounts/sa/blobServices/default", + "the parent account": testSub + "/resourceGroups/rg/providers/" + + "Microsoft.CognitiveServices/accounts/acct", + "a different child of the account": testSub + "/resourceGroups/rg/providers/" + + "Microsoft.CognitiveServices/accounts/acct/deployments/dep", + } + for what, id := range notProjects { + _, err := NewPortalPrefix(id) + assert.Errorf(t, err, "%s is not a Foundry project", what) + } +} + +// Names reach these builders from the service, not from this extension's own +// validation, so a space or a slash would otherwise produce a link that breaks +// when pasted or points at a different route. +func TestPortalURLsEscapeTheNamesTheyCarry(t *testing.T) { + prefix, err := NewPortalPrefix(testSub + "/resourceGroups/rg/providers/" + + "Microsoft.CognitiveServices/accounts/acct/projects/proj") + require.NoError(t, err) + + got := prefix.DatasetURL("my set", "1.0") + assert.NotContains(t, got, "my set", "a raw space does not survive a paste") + assert.Contains(t, got, "my%20set") + + got = prefix.EvaluatorURL("a/b", "1") + assert.Contains(t, got, "a%2Fb", "a slash would otherwise change the route") + assert.Equal(t, 1, strings.Count(got, "/build/evaluations/catalog/")) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/portal_urls.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/portal_urls.go index 8ed19d149c0..2612c296eaa 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/portal_urls.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/portal_urls.go @@ -6,6 +6,7 @@ package eval_api import ( "encoding/base64" "fmt" + "net/url" "strings" "azureaieval/internal/messages" @@ -14,6 +15,11 @@ import ( "github.com/google/uuid" ) +// foundryProjectResourceType is the only resource a portal prefix can be built +// from. Every nested resource has a parent and a slash in its type, so matching +// on shape rather than on this let unrelated children through. +const foundryProjectResourceType = "Microsoft.CognitiveServices/accounts/projects" + // PortalPrefix holds the parsed project context needed to construct Foundry portal URLs. type PortalPrefix struct { prefix string // e.g. "https://ai.azure.com/nextgen/r/,,,," @@ -33,8 +39,11 @@ func NewPortalPrefix(projectResourceID string) (*PortalPrefix, error) { return nil, messages.EncodingSubscriptionID(err) } + // The exact type, not merely a nested one. Any child resource has a parent + // and a slash in its type -- a storage container reached this far and built a + // plausible URL onto a portal page that does not exist. if resourceID.Parent == nil || - !strings.Contains(string(resourceID.ResourceType.Type), "/") { + !strings.EqualFold(resourceID.ResourceType.String(), foundryProjectResourceType) { return nil, messages.NotAFoundryProjectResourceID(projectResourceID) } @@ -48,23 +57,30 @@ func NewPortalPrefix(projectResourceID string) (*PortalPrefix, error) { // EvalRunURL returns the portal URL for an eval run report. func (p *PortalPrefix) EvalRunURL(evalID, runID string) string { - return fmt.Sprintf("%s/build/evaluations/%s/run/%s", p.prefix, evalID, runID) + return fmt.Sprintf("%s/build/evaluations/%s/run/%s", + p.prefix, url.PathEscape(evalID), url.PathEscape(runID)) } // EvaluatorURL returns the portal URL for a generated evaluator. func (p *PortalPrefix) EvaluatorURL(evaluatorName, version string) string { - return fmt.Sprintf("%s/build/evaluations/catalog/%s/%s", p.prefix, evaluatorName, version) + return fmt.Sprintf("%s/build/evaluations/catalog/%s/%s", + p.prefix, url.PathEscape(evaluatorName), url.PathEscape(version)) } // DatasetURL returns the portal URL for a dataset. +// +// Escaped rather than interpolated: these names are the service's, not this +// extension's, so a space or a slash in one would otherwise produce a link that +// breaks when pasted or points somewhere else entirely. func (p *PortalPrefix) DatasetURL(datasetName, version string) string { - return fmt.Sprintf("%s/build/data/datasets/%s/%s", p.prefix, datasetName, version) + return fmt.Sprintf("%s/build/data/datasets/%s/%s", + p.prefix, url.PathEscape(datasetName), url.PathEscape(version)) } // OptimizationURL returns the portal URL for an optimization job. func (p *PortalPrefix) OptimizationURL(agentName, operationID string) string { return fmt.Sprintf("%s/build/agents/%s/optimization/%s", - p.prefix, agentName, operationID) + p.prefix, url.PathEscape(agentName), url.PathEscape(operationID)) } // encodeSubscriptionForURL encodes a subscription ID GUID as base64 without padding. diff --git a/cli/azd/extensions/azure.ai.evaluations/tests/live/run_cancel_test.go b/cli/azd/extensions/azure.ai.evaluations/tests/live/run_cancel_test.go index 170f33f673f..6b53228f03f 100644 --- a/cli/azd/extensions/azure.ai.evaluations/tests/live/run_cancel_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/tests/live/run_cancel_test.go @@ -85,6 +85,12 @@ func TestLiveRunCancel(t *testing.T) { // left here to observe. current, getErr := env.evalClient.GetOpenAIEvalRun(ctx, group.ID, run.ID) require.NoError(t, getErr, "reading the run whose cancel was refused") + + // Only that race is skipped. A refusal while the run is still moving is + // the failure this test exists to catch, and skipping on any error at + // all retires the test without anyone deciding to. + require.Truef(t, runAlreadyFinished(current.Status), + "cancel was refused while the run was still %q: %v", current.Status, err) t.Skipf("cancel was refused with the run already at %q: %v", current.Status, err) } require.NotNil(t, canceled) @@ -110,3 +116,14 @@ func TestLiveRunCancel(t *testing.T) { t.Logf("run reached %s", status) } + +// runAlreadyFinished reports a run that has stopped, so a refused cancel is the +// race rather than a fault. Spelled out here because tests/live cannot reach +// internal/cmd's copy, and both spellings of cancelled are in use. +func runAlreadyFinished(status string) bool { + switch strings.ToLower(status) { + case "completed", "failed", "canceled", "cancelled", "error": + return true + } + return false +} From 3f04da6b68a3a45c5011e2b3839dd543a3f57742 Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 18 Aug 2026 23:43:48 +0300 Subject: [PATCH 289/320] Register the two intentional words, and use the dictionary's spelling verison is deliberate test data, the way evaulators already is. unjudged joins unpassed and unscored. behaviour is not in the repo dictionary. --- cli/azd/extensions/azure.ai.evaluations/cspell.yaml | 4 +++- .../azure.ai.evaluations/internal/project/eval_config_lock.go | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/cli/azd/extensions/azure.ai.evaluations/cspell.yaml b/cli/azd/extensions/azure.ai.evaluations/cspell.yaml index a328b6bc634..96751b9f9a0 100644 --- a/cli/azd/extensions/azure.ai.evaluations/cspell.yaml +++ b/cli/azd/extensions/azure.ai.evaluations/cspell.yaml @@ -24,8 +24,9 @@ words: - CODEOWNERS # Possessive of an acronym cspell does not inflect on its own - CLI's - # Deliberate misspelling: the fixture the unknown-key tests are about + # Deliberate misspellings: the fixtures the unknown-key tests are about - evaulators + - verison # Terms - inlines - negotiables @@ -38,5 +39,6 @@ words: - unbuildable - unorderable - unpassed + - unjudged - unscored - Unparseable diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_lock.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_lock.go index a35fa754b5c..136f3addeb5 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_lock.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_lock.go @@ -64,7 +64,7 @@ func LockEvalConfig(ctx context.Context, evalDir string) (func(), error) { locked, err := lock.TryLockContext(waitCtx, 50*time.Millisecond) if err != nil || !locked { // Being cancelled is not the same as the lock being busy. The advisory - // behaviour below exists so a held lock cannot fail a scaffold; carrying + // behavior below exists so a held lock cannot fail a scaffold; carrying // it into Ctrl-C would go on to rewrite the configuration the user just // asked to stop. if ctxErr := ctx.Err(); ctxErr != nil { From 03fbd433c0900635e915ce320f4b16c2b30bcc03 Mon Sep 17 00:00:00 2001 From: mohessie Date: Wed, 19 Aug 2026 00:20:50 +0300 Subject: [PATCH 290/320] Match azd core on the dependencies it pins A new consistency gate landed upstream that requires an extension to use the versions azd core does. Existing extensions carry grandfather entries in dependency-versions.json for drift that predates the gate; a module added after it has no business adding more of those, so these move instead. azcore v1.21.0 -> v1.20.0, azidentity v1.14.0-beta.3 -> v1.13.1, and grpc v1.80.0 -> v1.82.1, where this module was the one behind. The azidentity change comes off a beta: only AzureDeveloperCLICredential and the two authentication error types are used, all of which are stable in v1.13.1. checkDependencyVersions passes locally in both worktrees. --- cli/azd/extensions/azure.ai.evaluations/go.mod | 8 ++++---- cli/azd/extensions/azure.ai.evaluations/go.sum | 16 ++++++++-------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/cli/azd/extensions/azure.ai.evaluations/go.mod b/cli/azd/extensions/azure.ai.evaluations/go.mod index ac9e1bfbc3d..50ad86238b1 100644 --- a/cli/azd/extensions/azure.ai.evaluations/go.mod +++ b/cli/azd/extensions/azure.ai.evaluations/go.mod @@ -3,8 +3,8 @@ module azureaieval go 1.26.4 require ( - github.com/Azure/azure-sdk-for-go/sdk/azcore v1.21.0 - github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.14.0-beta.3 + github.com/Azure/azure-sdk-for-go/sdk/azcore v1.20.0 + github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1 github.com/azure/azure-dev/cli/azd v1.28.0 github.com/fatih/color v1.18.0 github.com/gofrs/flock v0.12.1 @@ -13,7 +13,7 @@ require ( github.com/spf13/pflag v1.0.10 github.com/stretchr/testify v1.11.1 go.yaml.in/yaml/v3 v3.0.4 - google.golang.org/grpc v1.80.0 + google.golang.org/grpc v1.82.1 google.golang.org/protobuf v1.36.11 ) @@ -101,6 +101,6 @@ require ( golang.org/x/term v0.44.0 // indirect golang.org/x/text v0.38.0 // indirect golang.org/x/time v0.9.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20260401024825-9d38bb4040a9 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260414002931-afd174a4e478 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/cli/azd/extensions/azure.ai.evaluations/go.sum b/cli/azd/extensions/azure.ai.evaluations/go.sum index 81c10e45793..6c14ec16af1 100644 --- a/cli/azd/extensions/azure.ai.evaluations/go.sum +++ b/cli/azd/extensions/azure.ai.evaluations/go.sum @@ -3,10 +3,10 @@ dario.cat/mergo v1.0.2 h1:85+piFYR1tMbRrLcDwR18y4UKJ3aH1Tbzi24VRW1TK8= dario.cat/mergo v1.0.2/go.mod h1:E/hbnu0NxMFBjpMIE34DRGLWqDy0g5FuKDhCb31ngxA= github.com/AlecAivazis/survey/v2 v2.3.7 h1:6I/u8FvytdGsgonrYsVn2t8t4QiRnh6QSTqkkhIiSjQ= github.com/AlecAivazis/survey/v2 v2.3.7/go.mod h1:xUTIdE4KCOIjsBAE1JYsUPoCqYdZ1reCfTwbto0Fduo= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.21.0 h1:fou+2+WFTib47nS+nz/ozhEBnvU96bKHy6LjRsY4E28= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.21.0/go.mod h1:t76Ruy8AHvUAC8GfMWJMa0ElSbuIcO03NLpynfbgsPA= -github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.14.0-beta.3 h1:0g4UTtvRA9goC37cmD9ZHdW6CCNJR4cOXBnHz0r4ubM= -github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.14.0-beta.3/go.mod h1:fEiHi0sbYqbo3shUkIF1SNxm8GyeEJl+Poc/djOvbdE= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.20.0 h1:JXg2dwJUmPB9JmtVmdEB16APJ7jurfbY5jnfXpJoRMc= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.20.0/go.mod h1:YD5h/ldMsG0XiIw7PdyNhLxaM317eFh5yNLccNfGdyw= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1 h1:Hk5QBxZQC1jb2Fwj6mpzme37xbCDdNTxU7O9eb5+LB4= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1/go.mod h1:IYus9qsFobWIc2YVwe/WPjcnyCkPKtnHAqUYeebc8z0= github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.3.2 h1:yz1bePFlP5Vws5+8ez6T3HWXPmwOK7Yvq8QxDBD3SKY= github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.3.2/go.mod h1:Pa9ZNPuoNu/GztvBSKk9J1cDJW6vk/n0zLtV4mgd8N8= github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2 h1:9iefClla7iYpfYWdzPCRDozdmndjTm8DXdpCzPajMgA= @@ -300,10 +300,10 @@ golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260401024825-9d38bb4040a9 h1:m8qni9SQFH0tJc1X0vmnpw/0t+AImlSvp30sEupozUg= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260401024825-9d38bb4040a9/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= -google.golang.org/grpc v1.80.0 h1:Xr6m2WmWZLETvUNvIUmeD5OAagMw3FiKmMlTdViWsHM= -google.golang.org/grpc v1.80.0/go.mod h1:ho/dLnxwi3EDJA4Zghp7k2Ec1+c2jqup0bFkw07bwF4= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260414002931-afd174a4e478 h1:RmoJA1ujG+/lRGNfUnOMfhCy5EipVMyvUE+KNbPbTlw= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260414002931-afd174a4e478/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= +google.golang.org/grpc v1.82.1 h1:NnAxzGRA0677vCa4BUkOAnO5+FfQqVl9iUXeD0IqcGE= +google.golang.org/grpc v1.82.1/go.mod h1:yzTZ1TB1Z3SG+LIYaI+WiE8D5+PZ3ArnrSp8zF3+/ZA= google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= From ed89da0705d5756eecd3e5561e7dd9af9184862b Mon Sep 17 00:00:00 2001 From: Mohamed Hessien Date: Wed, 19 Aug 2026 01:23:54 +0300 Subject: [PATCH 291/320] Hold the concurrent config read to every field, and say what failed Counting evals reports a document caught mid-write as a whole file whenever it happens to parse with two of them, having lost the datasets or a field off the second eval. The reader now compares against a baseline read back after the first save, so every field has to survive. The round-trip test wrote the same payload twice, so an implementation that silently left the original file in place would have passed it. The first write is now a different config. And the failure is reported by category: one counter, with an example off whichever loss happened first, leaves a run where both a transient read failure and a genuinely incomplete config occurred looking like whichever won the race -- which is the distinction the message exists to make. All three found by Copilot review, which #9500 cannot get: it is 38k lines against a 20,000 line limit, so this went up as a small PR against the feature branch instead. --- .../project/eval_config_atomic_test.go | 59 ++++++++++++++++--- 1 file changed, 51 insertions(+), 8 deletions(-) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_atomic_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_atomic_test.go index ff2d043a603..f41a20dcd66 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_atomic_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_atomic_test.go @@ -4,8 +4,10 @@ package project import ( + "fmt" "os" "path/filepath" + "reflect" "sync" "sync/atomic" "testing" @@ -36,9 +38,24 @@ func TestSaveEvalConfigNeverExposesAHalfWrittenFile(t *testing.T) { } require.NoError(t, SaveEvalConfigTo(path, full)) + // Every field, not the eval count. A document caught mid-write can still + // parse with two evals while having lost the datasets, or a field off the + // second one, and counting entries reports that as a whole file. Read back + // what a correct read returns and hold every later read to it. + baseline, err := LoadEvalConfig(path) + require.NoError(t, err) + require.Len(t, baseline.Evals, 2) + require.Len(t, baseline.Datasets, 1) + var wg sync.WaitGroup stop := make(chan struct{}) - var truncated int + // Counted apart, because they want opposite responses. A config observed + // with fields missing is the bug this test guards. A read that failed under + // contention may only mean the retry budget was short on a loaded machine. + // One combined counter, or one example off whichever happened first, leaves + // a run where both occurred looking like whichever won the race. + var readErrors, mismatches int + var firstReadError, firstMismatch string var replacements int64 wg.Go(func() { @@ -50,11 +67,20 @@ func TestSaveEvalConfigNeverExposesAHalfWrittenFile(t *testing.T) { // remove-then-rename exposes, and OpenEvalConfig turns it into // "there is no configuration yet" -- the same loss this guards // against, by another route. - truncated++ + readErrors++ + if firstReadError == "" { + firstReadError = err.Error() + } continue } - if len(cfg.Evals) != 2 { - truncated++ + if !reflect.DeepEqual(cfg, baseline) { + mismatches++ + if firstMismatch == "" { + firstMismatch = fmt.Sprintf( + "%d evals and %d datasets, wanted %d and %d", + len(cfg.Evals), len(cfg.Datasets), + len(baseline.Evals), len(baseline.Datasets)) + } } } close(stop) @@ -76,8 +102,20 @@ func TestSaveEvalConfigNeverExposesAHalfWrittenFile(t *testing.T) { require.NotZero(t, atomic.LoadInt64(&replacements), "the writer has to have replaced the file, or nothing was under test") - assert.Zerof(t, truncated, - "a concurrent reader failed to see the whole config %d times", truncated) + assert.Zerof(t, readErrors+mismatches, + "over %d replacements a concurrent reader saw %d incomplete configs (first: %s) "+ + "and %d failed reads (first: %s)", + atomic.LoadInt64(&replacements), + mismatches, orNone(firstMismatch), + readErrors, orNone(firstReadError)) +} + +// orNone keeps an absent example from reading as an empty one. +func orNone(s string) string { + if s == "" { + return "none" + } + return s } // OpenEvalConfig maps a missing file to "no configuration yet", which callers @@ -131,13 +169,18 @@ func TestSaveEvalConfigRoundTripsThroughTheRename(t *testing.T) { path := filepath.Join(t.TempDir(), "azure.eval.yaml") want := &EvalConfig{Evals: []Eval{{Name: "only", EvaluationLevel: "turn"}}} + // A different first write, so the second one has something to replace. + // Writing the same payload twice passes even if the second save silently + // left the original file where it was, which is the case worth catching. + first := &EvalConfig{Evals: []Eval{{Name: "replaced", EvaluationLevel: "conversation"}}} + require.NoError(t, SaveEvalConfigTo(path, first)) require.NoError(t, SaveEvalConfigTo(path, want)) - require.NoError(t, SaveEvalConfigTo(path, want)) // over an existing file got, err := LoadEvalConfig(path) require.NoError(t, err) require.Len(t, got.Evals, 1) - assert.Equal(t, "only", got.Evals[0].Name) + assert.Equal(t, "only", got.Evals[0].Name, "the second save has to have replaced the first") + assert.Equal(t, "turn", got.Evals[0].EvaluationLevel) // The temporary file is this function's business and must not be left over. entries, err := os.ReadDir(filepath.Dir(path)) From e31bfba25a92ff4d550187f8a989bd0df502c354 Mon Sep 17 00:00:00 2001 From: Mohamed Hessien Date: Wed, 19 Aug 2026 07:03:05 +0300 Subject: [PATCH 292/320] Make the run results agree with each other and with the portal (AB#5530202) Closes findings F031, F033 and F034 of AB#5530202. F031 - pass-rate is measured over the rows that were scored, matching the portal. The arithmetic lived in three places and now lives in one, which is how the CLI managed to disagree with the portal and with its own evaluator table two lines above. The flip is made visible rather than quiet: the rate names its denominator, any-failure still counts errored rows, and a pass-rate gate warns when it judged only part of a run. F033 - the footer counted unscored rows as failures, printing "13 sample(s) failed" under totals reading 5 failed and 8 errored. F034 - the SAMPLE column numbered within the current filter, so the same sample carried a different number depending on the flags while reading like an identifier. Dropped; ITEM already carries the id run output show accepts. --- .../internal/cmd/gating.go | 53 +++++++++++++++--- .../internal/cmd/gating_conformance_test.go | 24 +++++--- .../internal/cmd/gating_silent_test.go | 6 +- .../internal/cmd/gating_test.go | 20 ++++--- .../azure.ai.evaluations/internal/cmd/run.go | 5 +- .../internal/cmd/run_list_test.go | 27 +++++++-- .../internal/cmd/run_ops.go | 19 +++++-- .../internal/cmd/run_output.go | 21 +++++-- .../internal/cmd/run_render_test.go | 36 +++++++++++- .../internal/cmd/score_display_test.go | 4 +- .../internal/messages/messages.go | 56 +++++++++++++++---- 11 files changed, 216 insertions(+), 55 deletions(-) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/gating.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/gating.go index 2978db2e50c..e17392fd75b 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/gating.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/gating.go @@ -67,13 +67,32 @@ func parseGate(spec string) (gate, error) { return gate{set: true, passRate: value}, nil } -// breach reports why the run missed the threshold, or empty when it met it. +// scoredPassRate is the one definition of a run's pass rate: the share of the +// rows an evaluator actually scored. +// +// Errored and skipped rows are outside the denominator because nothing graded +// them, and an infrastructure failure is not a quality signal. This is what the +// portal reports and what `--fail-on pass-rate` compares against, so the two +// figures a reader sees two lines apart cannot disagree. // -// Errored and skipped rows count against the pass rate, and they can: the -// service puts them inside `total`, verified live on a run that reported -// total=3 passed=2 errored=1. Were they outside it, a run with two passes and -// one error would report total=2 and score a perfect rate, which is precisely -// the broken evaluation a gate exists to catch. +// ok is false when nothing was scored at all: a rate over no rows is not zero, +// it is absent, and the caller has to say so rather than print it. +// +// The consequence is worth stating. A run where almost everything errored can +// now report a high rate off the few rows that survived, so the count that did +// not score is printed beside it. +func scoredPassRate(counts *eval_api.EvalRunResultCounts) (rate float64, scored int, ok bool) { + if counts == nil { + return 0, 0, false + } + scored = counts.Passed + counts.Failed + if scored <= 0 { + return 0, 0, false + } + return float64(counts.Passed) / float64(scored), scored, true +} + +// breach reports why the run missed the threshold, or empty when it met it. // // A run that scored nothing at all breaches every threshold rather than // dividing by zero — "no rows passed" is the honest reading of an empty result. @@ -91,13 +110,19 @@ func (g gate) breach(counts *eval_api.EvalRunResultCounts) string { return messages.GateNoRowsScored() } if g.anyFailure { + // Deliberately stricter than the rate: this counts a row nothing could + // grade against the run, because "everything passed" is not true of a + // run that failed to grade half of what it was given. unpassed := counts.Total - counts.Passed if unpassed > 0 { return messages.GateSamplesDidNotPass(unpassed, counts.Total) } return "" } - actual := float64(counts.Passed) / float64(counts.Total) + actual, _, ok := scoredPassRate(counts) + if !ok { + return messages.GateNoRowsScored() + } if actual < g.passRate { return messages.GatePassRateBelow(actual, g.passRate) } @@ -121,6 +146,18 @@ func applyGate(cmd *cobra.Command, g gate, run *eval_api.OpenAIEvalRun) { if run == nil { return } + // Rows nothing could grade are outside the rate, so a run that errored on + // most of what it was given can clear a threshold on the few that survived. + // That is the cost of measuring quality over scored rows only, and the gate + // is where it has to be said: this is the line a pipeline log keeps. + if g.set && !g.anyFailure { + if c := run.ResultCounts; c != nil { + if _, scored, ok := scoredPassRate(c); ok && c.Total > scored { + fmt.Fprint(os.Stderr, + messages.Warning(messages.GateSawUnscoredRows(c.Total-scored, c.Total))) + } + } + } reason := g.breach(run.ResultCounts) if reason == "" { return @@ -135,5 +172,7 @@ func addFailOnFlag(cmd *cobra.Command, target *string) { // writes a condition that never fires. cmd.Flags().StringVar(target, "fail-on", "", "Fail when the run misses this threshold: any-failure, or pass-rate=<0..1>. "+ + "pass-rate is measured over the rows that were scored, so rows nothing "+ + "could grade are outside it; any-failure counts them against the run. "+ "Exits 1.") } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/gating_conformance_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/gating_conformance_test.go index 9637402f061..d6757667900 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/gating_conformance_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/gating_conformance_test.go @@ -49,19 +49,27 @@ func TestRunCompletedSeparatesRegressionFromFailureToRun(t *testing.T) { } } -// pass-rate is passed/total, and the service puts errored and skipped rows -// inside total. Were they outside it, a run with two passes and one error -// would score a perfect rate -- exactly the broken evaluation a gate exists to -// catch. -func TestPassRateCountsErroredAndSkippedAgainstTheThreshold(t *testing.T) { +// pass-rate is passed/(passed+failed): the share of the rows something actually +// graded. Errored and skipped rows are outside it, because an infrastructure +// failure is not a quality signal, and this is the figure the portal reports. +// +// The cost is real and deliberate: a run with two passes and thirteen errors +// scores a perfect rate. `any-failure` is the gate that still counts those, and +// a pass-rate gate warns when it judged only part of a run. +func TestPassRateIsMeasuredOverTheRowsThatWereScored(t *testing.T) { g, err := parseGate("pass-rate=0.8") require.NoError(t, err) - // 2 passed of 3 total, the third errored: 66.7%, below 80%. - breach := g.breach(&eval_api.EvalRunResultCounts{Total: 3, Passed: 2, Errored: 1}) - require.NotEmpty(t, breach, "an errored row is not a pass") + // 2 passed, 1 failed, 1 errored: 2 of 3 scored is 66.7%, below 80%. + breach := g.breach(&eval_api.EvalRunResultCounts{Total: 4, Passed: 2, Failed: 1, Errored: 1}) + require.NotEmpty(t, breach, "a scored row that failed still counts") assert.Contains(t, breach, "66.7%") + // The same run without the failure: everything scored, passed, so the + // errored row does not drag a quality number down on its own. + assert.Empty(t, g.breach(&eval_api.EvalRunResultCounts{Total: 3, Passed: 2, Errored: 1}), + "nothing graded the errored row, so it is not evidence of a regression") + assert.Empty(t, g.breach(&eval_api.EvalRunResultCounts{Total: 3, Passed: 3})) } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/gating_silent_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/gating_silent_test.go index 7a201f5c909..37d6e56afef 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/gating_silent_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/gating_silent_test.go @@ -67,8 +67,10 @@ func TestGatesStillJudgeRunsThatScoredSomething(t *testing.T) { rate, err := parseGate("pass-rate=0.8") require.NoError(t, err) - assert.Empty(t, rate.breach(&eval_api.EvalRunResultCounts{Total: 10, Passed: 9})) - assert.NotEmpty(t, rate.breach(&eval_api.EvalRunResultCounts{Total: 10, Passed: 7})) + // Spelled out with Failed rather than left to total, because the rate is + // measured over what was scored: passed plus failed. + assert.Empty(t, rate.breach(&eval_api.EvalRunResultCounts{Total: 10, Passed: 9, Failed: 1})) + assert.NotEmpty(t, rate.breach(&eval_api.EvalRunResultCounts{Total: 10, Passed: 7, Failed: 3})) // Counts the service never sent are not a pass. assert.NotEmpty(t, rate.breach(nil)) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/gating_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/gating_test.go index 5952e53d694..f723bc6ebd2 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/gating_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/gating_test.go @@ -51,19 +51,23 @@ func TestGateBreach(t *testing.T) { require.NotEmpty(t, anyFailure.breach(&eval_api.EvalRunResultCounts{Total: 2, Passed: 1, Failed: 1})) }) - // Errored rows are inside the total, verified live, so they count against - // the threshold the same way a failing row does. - t.Run("errored rows count against the rate", func(t *testing.T) { - counts := &eval_api.EvalRunResultCounts{Total: 10, Passed: 8, Errored: 2} - require.Empty(t, eighty.breach(counts), "0.8 exactly meets a 0.8 threshold") + // Errored rows sit outside the rate: nothing graded them, so they are not + // evidence of a regression. `any-failure` is the gate that counts them. + t.Run("errored rows are outside the rate", func(t *testing.T) { + counts := &eval_api.EvalRunResultCounts{Total: 12, Passed: 8, Failed: 2, Errored: 2} + require.Equal(t, "", eighty.breach(counts), "8 of the 10 scored passed, which meets 0.8") - counts = &eval_api.EvalRunResultCounts{Total: 10, Passed: 7, Errored: 3} - require.NotEmpty(t, eighty.breach(counts)) + counts = &eval_api.EvalRunResultCounts{Total: 13, Passed: 7, Failed: 3, Errored: 3} + require.NotEmpty(t, eighty.breach(counts), "7 of the 10 scored is under 0.8") + + counts = &eval_api.EvalRunResultCounts{Total: 10, Passed: 8, Errored: 2} + require.Empty(t, eighty.breach(counts), + "everything that was scored passed, so the errored rows do not breach it") }) // The wording is pinned because the hero scenario shows it verbatim. t.Run("reads as a percentage", func(t *testing.T) { - counts := &eval_api.EvalRunResultCounts{Total: 1000, Passed: 764} + counts := &eval_api.EvalRunResultCounts{Total: 1000, Passed: 764, Failed: 236} require.Equal(t, "pass rate 76.4% is below the required 80.0%", eighty.breach(counts)) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go index 72f7af4aed5..778afa3d029 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go @@ -992,7 +992,10 @@ func renderRun( // evaluators is one sample to go and look at, and reporting it as two // overstates how much is wrong. if c := run.ResultCounts; c != nil && c.Total > 0 { - fmt.Fprint(out, messages.OverallPassRate(formatRate(c.Passed, c.Total), c.Passed, c.Total)) + if rate, scored, ok := scoredPassRate(c); ok { + fmt.Fprint(out, messages.OverallPassRate( + fmt.Sprintf("%.1f%%", rate*100), c.Passed, scored, c.Total-scored)) + } if c.Errored > 0 { fmt.Fprint(out, messages.SamplesErrored(c.Errored)) } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_list_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_list_test.go index bec0c241092..86330aee194 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_list_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_list_test.go @@ -26,19 +26,34 @@ func TestRunListColumnsMatchTheScenario(t *testing.T) { "the scenario compares 80.0% against 93.3%, so the row has to carry the rate") } -// The rate is the gate's arithmetic: passed over total, with errored and -// skipped inside the total. A row a reader gates on must not disagree with the -// gate that acts on it. +// The rate is the gate's arithmetic: passed over the rows that were scored, +// with errored and skipped outside it. A row a reader gates on must not +// disagree with the gate that acts on it. +// +// The list is the one view that shows a rate next to a sample count, so it also +// carries how many rows the rate covers. Without that, two passes and one +// errored row read as SAMPLES 3, PASS RATE 100.0%. func TestRunListPassRateAgreesWithTheGate(t *testing.T) { - counts := &eval_api.EvalRunResultCounts{Total: 3, Passed: 2, Errored: 1} + counts := &eval_api.EvalRunResultCounts{Total: 4, Passed: 2, Failed: 1, Errored: 1} - assert.Equal(t, "66.7%", runPassRate(counts), - "an errored row is not a pass, here or in the gate") + assert.Equal(t, "66.7% (3 scored)", runPassRate(counts), + "2 of the 3 rows that were scored passed, here and in the gate") g, err := parseGate("pass-rate=0.8") assert.NoError(t, err) assert.NotEmpty(t, g.breach(counts), "the same counts that read 66.7% must breach an 80% threshold") + + // The errored row is outside the rate rather than counted as a failure, and + // the cell says so rather than reading as a clean sweep of the run. + assert.Equal(t, "100.0% (2 scored)", + runPassRate(&eval_api.EvalRunResultCounts{Total: 3, Passed: 2, Errored: 1}), + "nothing graded the errored row, so it is not a miss, but the rate is not the whole run") + + // Nothing unscored, nothing to qualify. + assert.Equal(t, "75.0%", + runPassRate(&eval_api.EvalRunResultCounts{Total: 4, Passed: 3, Failed: 1}), + "every row was scored, so the bare rate is the whole story") } // A run that has not scored yet has no rate to show. An empty cell says that; diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go index ab9b08325e5..e8916d89fc5 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go @@ -392,11 +392,22 @@ func sampleCount(counts *eval_api.EvalRunResultCounts) string { return strconv.Itoa(counts.Total) } -// runPassRate is the same passed/total the gate uses, so a row a reader gates -// on cannot disagree with the gate. +// runPassRate is the same scored pass rate the gate uses, so a row a reader +// gates on cannot disagree with the gate. +// +// The rate is followed by the rows it was measured over whenever that is fewer +// than the run's samples. Without it the comparison view reads a run of two +// passes and one errored row as SAMPLES 3, PASS RATE 100.0%, which is the one +// place the scored denominator was not stated and so the one place a partly +// errored run looked perfect. func runPassRate(counts *eval_api.EvalRunResultCounts) string { - if counts == nil || counts.Total == 0 { + rate, scored, ok := scoredPassRate(counts) + if !ok { return "" } - return formatRate(counts.Passed, counts.Total) + out := fmt.Sprintf("%.1f%%", rate*100) + if counts.Total > scored { + out += fmt.Sprintf(" (%d scored)", scored) + } + return out } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output.go index a897bb9a811..0eb2bb36186 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output.go @@ -544,7 +544,8 @@ func renderResults( } else { fmt.Fprintln(w) rows := make([][]string, 0, len(items)) - for i, it := range items { + var rowsFailed, rowsUnscored int + for _, it := range items { // One row per evaluated sample, not per verdict: a sample that // failed three evaluators is one sample to go and look at, and // listing it three times buries how much is actually wrong. @@ -586,9 +587,17 @@ func renderResults( if verdicts == "" { verdicts = "-" } + if len(failed) > 0 { + rowsFailed++ + } else { + rowsUnscored++ + } + // No position column. It numbered within the current filter, so the + // same sample carried a different number depending on the flags while + // reading like an identifier -- and ITEM already carries the id, which + // is what `run output show` accepts. rows = append(rows, []string{ it.ID, - strconv.Itoa(i + 1), meanScoreOf(it.Results), truncate(verdicts, 40), truncate(reason, 44), @@ -597,12 +606,14 @@ func renderResults( // Only the first failure's reason fits a cell; `run output show` has // the rest. if err := emitTable(w, - []string{"ITEM", "SAMPLE", "SCORE", "FAILED EVALUATORS", "REASON"}, + []string{"ITEM", "SCORE", "EVALUATORS", "REASON"}, rows); err != nil { return err } - if n := len(rows); failedOnly && n > 0 { - fmt.Fprint(w, messages.SamplesFailedAtLeastOne(n)) + // Counting unscored rows as failures put a number here that contradicted + // the totals two lines above, which is what a reader compares it with. + if failedOnly && len(rows) > 0 { + fmt.Fprint(w, messages.SamplesNeedingALook(rowsFailed, rowsUnscored)) } } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_render_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_render_test.go index d222ec7bd93..3b75a2fdf86 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_render_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_render_test.go @@ -46,9 +46,17 @@ func TestRenderResultsIsOneRowPerSample(t *testing.T) { assert.Equal(t, 1, strings.Count(text, "oi_2"), "a sample that failed two evaluators must still be one row:\n%s", text) - for _, header := range []string{"ITEM", "SAMPLE", "FAILED EVALUATORS", "REASON"} { + for _, header := range []string{"ITEM", "SCORE", "EVALUATORS", "REASON"} { assert.Containsf(t, text, header, "the listing lost its %s column", header) } + + // The old SAMPLE column numbered within the current filter, so the same + // sample carried a different number depending on the flags while reading + // like an identifier. ITEM carries the id `run output show` accepts. + assert.NotContains(t, text, "SAMPLE", + "a position that changes with the filter must not sit beside the id") + assert.NotContains(t, text, "FAILED EVALUATORS", + "the column also carries evaluators that returned no verdict, which did not fail") } // The failing row has to name every evaluator that failed it, because that is @@ -66,6 +74,32 @@ func TestRenderResultsNamesEveryFailedEvaluator(t *testing.T) { assert.Contains(t, text, "1 sample(s) failed at least one evaluator.") } +// The footer is read against the totals printed a few lines above it, so it +// cannot count a row nothing scored as a row that failed. The reported run +// closed "13 sample(s) failed at least one evaluator" over totals that said 5 +// failed and 8 errored. +func TestFailedOnlyFooterHoldsUnscoredRowsApart(t *testing.T) { + items := []eval_api.OutputItem{ + {ID: "oi_fail", Results: []eval_api.OutputResult{ + {Name: "relevance", Passed: new(false), Score: 1, Reason: "Answered a different question."}, + }}, + // No verdict: the evaluator errored on this row rather than scoring it. + {ID: "oi_unscored", Results: []eval_api.OutputResult{{Name: "relevance"}}}, + } + + var out bytes.Buffer + run := &eval_api.OpenAIEvalRun{ID: "evalrun_1", Status: "completed"} + require.NoError(t, renderResults(&out, run, items, true)) + + text := out.String() + assert.Contains(t, text, "1 sample(s) failed at least one evaluator, and 1 could not be scored.", + "the two have to be counted apart:\n%s", text) + assert.NotContains(t, text, "2 sample(s) failed", + "an unscored row is not a failing one") + assert.Contains(t, text, "(no verdict)", + "and the row itself has to say which evaluator returned nothing") +} + // The run summary carries pass and fail counts but no score, so the mean has // to be averaged over the rows an evaluator actually scored. func TestCriteriaMeans(t *testing.T) { diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/score_display_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/score_display_test.go index 9697156ff07..1c65d16c8b5 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/score_display_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/score_display_test.go @@ -78,7 +78,7 @@ func TestAGateBreachNeverReportsARateAsBelowItself(t *testing.T) { gate, err := parseGate("pass-rate=0.8") require.NoError(t, err) - breach := gate.breach(&eval_api.EvalRunResultCounts{Total: 10000, Passed: 7996}) + breach := gate.breach(&eval_api.EvalRunResultCounts{Total: 10000, Passed: 7996, Failed: 2004}) require.NotEmpty(t, breach, "7996/10000 is under 0.8 and must breach") assert.NotContains(t, breach, "80.0% is below the required 80.0%", "a line saying a rate is below itself tells a reader nothing") @@ -87,5 +87,5 @@ func TestAGateBreachNeverReportsARateAsBelowItself(t *testing.T) { // The wording the spec shows is unchanged wherever rounding does not collide. assert.Equal(t, "pass rate 76.4% is below the required 80.0%", - gate.breach(&eval_api.EvalRunResultCounts{Total: 1000, Passed: 764})) + gate.breach(&eval_api.EvalRunResultCounts{Total: 1000, Passed: 764, Failed: 236})) } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go b/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go index ac28c0274ba..326be1adcf0 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go @@ -1,3 +1,6 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + // Package messages holds every string this extension shows a user. // // One file, so the whole voice of the CLI can be reviewed in one sitting and a @@ -52,7 +55,7 @@ func NoEvalToRun() error { // different: this one is answered by a dataset, not by a source block. func EvalHasNoDataset(eval string) error { return fmt.Errorf( - "eval %q references no dataset and declares no source:. Add a dataset: to "+ + "eval %q references no dataset and declares no source. Add a dataset: to "+ "score rows you supply, or a source: to score traces or stored responses", eval) } @@ -199,14 +202,19 @@ func RunFinishedWithStatus(runID, status string) error { return fmt.Errorf("run %s finished with status %s", runID, status) } -// OverallPassRate reports the share of samples that passed every evaluator. +// OverallPassRate reports the share of the rows an evaluator scored that passed +// every evaluator. // -// The parenthetical is the bare fraction the spec prints. Spelling out "samples -// passed every evaluator" on every run reads as a caveat on the number rather -// than a definition of it; the doc comment above is where that belongs. -func OverallPassRate(rate string, passed, total int) string { - return fmt.Sprintf("\nOverall pass rate: %s (%d/%d)\n", - rate, passed, total) +// The denominator is named rather than left as a bare fraction. Rows nothing +// could grade are outside it, so a run that errored on most of its samples can +// report a high rate, and "of N scored" is what stops that reading as a verdict +// on the whole run. It is also the figure `--fail-on pass-rate` compares. +func OverallPassRate(rate string, passed, scored, unscored int) string { + if unscored > 0 { + return fmt.Sprintf("\nOverall pass rate: %s (%d of %d scored; %d not scored)\n", + rate, passed, scored, unscored) + } + return fmt.Sprintf("\nOverall pass rate: %s (%d/%d)\n", rate, passed, scored) } // SamplesErrored reports rows the run could not score at all. @@ -382,9 +390,35 @@ func NoRowsScored() string { return "\nNo rows have been scored yet.\n" } -// SamplesFailedAtLeastOne closes a --failed-only listing with its count. -func SamplesFailedAtLeastOne(samples int) string { - return fmt.Sprintf("\n%d sample(s) failed at least one evaluator.\n", samples) +// SamplesNeedingALook closes a --failed-only listing, holding the rows that +// failed apart from the rows nothing managed to score. +// +// One count covering both contradicted the totals printed two lines above it, +// which is what a reader compares it with: a run reporting 5 failed and 8 +// errored closed with "13 sample(s) failed at least one evaluator". +func SamplesNeedingALook(failed, unscored int) string { + if unscored == 0 { + return fmt.Sprintf("\n%d sample(s) failed at least one evaluator.\n", failed) + } + if failed == 0 { + return fmt.Sprintf("\n%d sample(s) could not be scored.\n", unscored) + } + return fmt.Sprintf( + "\n%d sample(s) failed at least one evaluator, and %d could not be scored.\n", + failed, unscored) +} + +// GateSawUnscoredRows warns that a pass-rate gate judged only part of the run. +// +// The rate excludes rows nothing could grade, so a run that errored on most of +// its samples can clear a threshold on the few that survived. The gate is the +// one place a pipeline is guaranteed to read, so it is said there rather than +// left for someone to notice in the summary. +func GateSawUnscoredRows(unscored, total int) error { + return fmt.Errorf( + "%d of %d samples were not scored, so the pass rate this gate read covers "+ + "only the rest; use --fail-on any-failure to count them against the run", + unscored, total) } // GeneratedNameNotAFileName reports a generated artifact name that would not From 9e6277c91a232ed89933237922909b2b8e0195e8 Mon Sep 17 00:00:00 2001 From: mohessie Date: Wed, 19 Aug 2026 07:24:58 +0300 Subject: [PATCH 293/320] Cut the eleventh bug bash build --- cli/azd/extensions/azure.ai.evaluations/CHANGELOG.md | 2 +- cli/azd/extensions/azure.ai.evaluations/extension.yaml | 2 +- cli/azd/extensions/azure.ai.evaluations/version.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cli/azd/extensions/azure.ai.evaluations/CHANGELOG.md b/cli/azd/extensions/azure.ai.evaluations/CHANGELOG.md index 1a0e537e970..fef7e79f0e1 100644 --- a/cli/azd/extensions/azure.ai.evaluations/CHANGELOG.md +++ b/cli/azd/extensions/azure.ai.evaluations/CHANGELOG.md @@ -1,6 +1,6 @@ # Release History -## 1.0.14-beta (Unreleased) +## 1.0.15-beta (Unreleased) First release of the Foundry evaluations extension. diff --git a/cli/azd/extensions/azure.ai.evaluations/extension.yaml b/cli/azd/extensions/azure.ai.evaluations/extension.yaml index a192d331e23..f132df4c01d 100644 --- a/cli/azd/extensions/azure.ai.evaluations/extension.yaml +++ b/cli/azd/extensions/azure.ai.evaluations/extension.yaml @@ -5,7 +5,7 @@ displayName: Foundry evaluations (Beta) description: Define and run Foundry evaluations from your terminal. (Beta) usage: azd ai eval [options] # NOTE: Make sure version.txt is in sync with this version. -version: 1.0.14-beta +version: 1.0.15-beta requiredAzdVersion: ">=1.27.1" language: go capabilities: diff --git a/cli/azd/extensions/azure.ai.evaluations/version.txt b/cli/azd/extensions/azure.ai.evaluations/version.txt index f8730a057a6..c2af346c4b5 100644 --- a/cli/azd/extensions/azure.ai.evaluations/version.txt +++ b/cli/azd/extensions/azure.ai.evaluations/version.txt @@ -1 +1 @@ -1.0.14-beta +1.0.15-beta From 47900041893c702ab4b1670ab45d0eb0439e6236 Mon Sep 17 00:00:00 2001 From: mohessie Date: Wed, 19 Aug 2026 08:01:59 +0300 Subject: [PATCH 294/320] Keep anchors working, and follow the symlink the read follows Two findings from a review round on the strictness and containment the last two commits added. Deciding an evaluator entry's keys by re-serializing it lifts the entry out of the document, and away from the anchors its aliases point at. `*judge` on a sibling entry, and `<<: *base`, decoded against an empty anchor table and failed a file that had loaded before -- a config file people write by hand, and one whose whole reason to carry an anchor is to share a judge between entries. Aliases are resolved into the snippet first, so the strictness costs nothing that worked. A recursive anchor is named rather than followed. Containment was decided lexically, and the read that follows it is os.ReadFile, which follows symlinks. A link committed to the repository satisfied the written form of the path while naming a file the project does not contain -- the same escape `..` is refused for, needing no more privilege to commit. The check now asks the question twice: of the path as written, and of what it resolves to. And the containment guard was tested through its helper only, so deleting the call site left the suite green. Both escapes are now asked of the command. --- .../internal/messages/messages.go | 7 ++ .../internal/pkg/evalcore/evaluator.go | 42 ++++++- .../pkg/evalcore/evaluator_anchor_test.go | 107 ++++++++++++++++++ .../internal/project/agent_instructions.go | 24 +++- .../project/instruction_containment_test.go | 54 +++++++++ 5 files changed, 231 insertions(+), 3 deletions(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/evaluator_anchor_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go b/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go index 326be1adcf0..c7ab1fc471b 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go @@ -1377,6 +1377,13 @@ func EvaluatorEntryMustBeMapping(kind any) error { return fmt.Errorf("evaluator entry must be a mapping, got %v", kind) } +// EvaluatorAliasIsCircular reports an anchor that contains its own alias. +// +// Expanding it has no end, so it is named rather than followed. +func EvaluatorAliasIsCircular(anchor string) error { + return fmt.Errorf("anchor %q refers to itself, so it cannot be expanded", anchor) +} + // --------------------------------------------------------------------------- // Deploy and reconcile // --------------------------------------------------------------------------- diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/evaluator.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/evaluator.go index d9bd6715a6c..12b4d9a8267 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/evaluator.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/evaluator.go @@ -119,7 +119,12 @@ func (el *EvaluatorList) UnmarshalYAML(value *yaml.Node) error { // decoder restores it; the error keeps yaml's own "field X not found in type Y" // shape, which the caller rewrites into the file's vocabulary. func decodeEvaluatorRef(node *yaml.Node) (EvaluatorRef, error) { - raw, err := yaml.Marshal(node) + resolved, err := resolveAliases(node, map[*yaml.Node]bool{}) + if err != nil { + return EvaluatorRef{}, err + } + + raw, err := yaml.Marshal(resolved) if err != nil { return EvaluatorRef{}, messages.DecodingEvaluator(err) } @@ -134,6 +139,41 @@ func decodeEvaluatorRef(node *yaml.Node) (EvaluatorRef, error) { return ref, nil } +// resolveAliases copies node with every alias replaced by what it names. +// +// The strict decode above re-serializes one entry, which lifts it out of the +// document and away from the anchors its aliases point at: `*judge` defined on +// a sibling entry, and `<<: *base`, decoded against an empty anchor table and +// failed a file that had loaded before. Resolving first keeps both working +// without giving up the strictness. +// +// active holds the anchors being expanded on this path. yaml permits an anchor +// that contains its own alias, which would otherwise expand forever. +func resolveAliases(node *yaml.Node, active map[*yaml.Node]bool) (*yaml.Node, error) { + if node.Kind == yaml.AliasNode && node.Alias != nil { + if active[node.Alias] { + return nil, messages.EvaluatorAliasIsCircular(node.Value) + } + active[node.Alias] = true + defer delete(active, node.Alias) + return resolveAliases(node.Alias, active) + } + + // The anchor is dropped with the alias it fed: keeping it would emit the + // same name twice once two aliases resolve to it. + copied := *node + copied.Anchor = "" + copied.Content = make([]*yaml.Node, len(node.Content)) + for i, child := range node.Content { + resolvedChild, err := resolveAliases(child, active) + if err != nil { + return nil, err + } + copied.Content[i] = resolvedChild + } + return &copied, nil +} + // yamlErrorLine matches the line number yaml puts on each unmarshal error. var yamlErrorLine = regexp.MustCompile(`line (\d+):`) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/evaluator_anchor_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/evaluator_anchor_test.go new file mode 100644 index 00000000000..e11a2b5c444 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/evaluator_anchor_test.go @@ -0,0 +1,107 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package evalcore + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + yaml "go.yaml.in/yaml/v3" +) + +// decodeEvaluators reads an evaluators: block the way the config reader does. +func decodeEvaluators(t *testing.T, doc string) (EvaluatorList, error) { + t.Helper() + + var holder struct { + Evaluators EvaluatorList `yaml:"evaluators"` + } + err := yaml.Unmarshal([]byte(doc), &holder) + return holder.Evaluators, err +} + +// An anchor shares one judge configuration across entries, which is the reason +// to write one. Decoding an entry on its own lifts it away from the anchor, so +// this is the case that broke when the entry gained a strict decoder. +func TestAnAnchorSharedBetweenEvaluatorsResolves(t *testing.T) { + list, err := decodeEvaluators(t, ` +evaluators: + - evaluator: builtin.relevance + initialization_parameters: &judge + deployment_name: gpt-4o + - evaluator: builtin.coherence + initialization_parameters: *judge +`) + + require.NoError(t, err) + require.Len(t, list, 2) + assert.Equal(t, map[string]any{"deployment_name": "gpt-4o"}, list[0].InitializationParameters) + assert.Equal(t, map[string]any{"deployment_name": "gpt-4o"}, list[1].InitializationParameters, + "the second entry should carry what the anchor holds") +} + +// A merge key is the other half of the same feature: it inherits the anchored +// entry and overrides one key. +func TestAMergeKeyInheritsTheEntryItNames(t *testing.T) { + list, err := decodeEvaluators(t, ` +evaluators: + - &base + evaluator: builtin.relevance + initialization_parameters: + deployment_name: gpt-4o + - <<: *base + evaluator: builtin.coherence +`) + + require.NoError(t, err) + require.Len(t, list, 2) + assert.Equal(t, "builtin.coherence", list[1].Evaluator, "the override should win") + assert.Equal(t, map[string]any{"deployment_name": "gpt-4o"}, list[1].InitializationParameters, + "the inherited key should survive") +} + +// Resolving aliases must not cost the strictness it was added around: a key +// nobody declared is still named, whether it is written out or inherited. +func TestAMisspeltKeyIsStillRefusedThroughAnAlias(t *testing.T) { + tests := map[string]string{ + "written out": ` +evaluators: + - evaluator: builtin.relevance + verison: 3 +`, + "inherited through a merge": ` +evaluators: + - &base + evaluator: builtin.relevance + verison: 3 + - <<: *base + evaluator: builtin.coherence +`, + } + + for name, doc := range tests { + t.Run(name, func(t *testing.T) { + _, err := decodeEvaluators(t, doc) + + require.Error(t, err) + assert.Contains(t, err.Error(), "verison", "the key they typed should be named") + }) + } +} + +// yaml permits an anchor holding its own alias. Expanding it has no end, so it +// has to be refused rather than followed. +func TestAnAnchorThatContainsItselfIsRefused(t *testing.T) { + _, err := decodeEvaluators(t, ` +evaluators: + - evaluator: builtin.relevance + data_mapping: &loop + query: *loop +`) + + require.Error(t, err) + assert.Contains(t, err.Error(), `anchor "loop" refers to itself`, + "the refusal should be ours, not yaml's own report of an anchor it could not find") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/agent_instructions.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/agent_instructions.go index 8c2b397ccb4..02079ad1737 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/agent_instructions.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/agent_instructions.go @@ -105,9 +105,29 @@ func AgentInstructionsFromProject( // withinDir reports whether path resolves to somewhere inside root. // -// Compared after cleaning both, so `..` segments are resolved before the -// question is asked rather than matched as text. +// Asked twice: once of the path as written, and once of what it resolves to. +// The read that follows this check follows symlinks, so a link committed to the +// repository would otherwise satisfy the written form while naming a file the +// project does not contain — the same escape `..` is refused for, needing no +// more privilege to commit. func withinDir(root, path string) bool { + if !liesWithin(root, path) { + return false + } + + realRoot, rootErr := filepath.EvalSymlinks(root) + realPath, pathErr := filepath.EvalSymlinks(path) + if rootErr != nil || pathErr != nil { + // Nothing to follow: a path that is not there yet leads nowhere, and + // the read below reports it as missing rather than as an escape. + return true + } + return liesWithin(realRoot, realPath) +} + +// liesWithin compares two paths as text, after cleaning both so that `..` +// segments are resolved before the question is asked rather than matched. +func liesWithin(root, path string) bool { rel, err := filepath.Rel(filepath.Clean(root), filepath.Clean(path)) if err != nil { return false diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/instruction_containment_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/instruction_containment_test.go index 2aa6c78def5..e0483582b3a 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/instruction_containment_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/instruction_containment_test.go @@ -4,10 +4,13 @@ package project import ( + "os" "path/filepath" + "runtime" "testing" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) // The optimize metadata's instruction_file pointer is only as trustworthy as @@ -37,3 +40,54 @@ func TestInstructionPointerCannotLeaveTheProject(t *testing.T) { assert.Falsef(t, withinDir(root, p), "%q is outside the project", p) } } + +// The guard is worth nothing if the command does not consult it, so this asks +// the command rather than the helper: both escapes have to come back as the +// refusal, not as the contents of the file they named. +func TestGenerateRefusesInstructionsFromOutsideTheProject(t *testing.T) { + t.Run("named with ..", func(t *testing.T) { + root := t.TempDir() + secret := filepath.Join(root, "outside-secret.md") + require.NoError(t, os.WriteFile(secret, []byte("a file the project does not contain"), 0o600)) + + project := filepath.Join(root, "proj") + serviceDir := filepath.Join(project, "agent") + require.NoError(t, os.MkdirAll(serviceDir, 0o750)) + writeOptimizeConfig(t, serviceDir, + "instruction_file: ../../../../outside-secret.md\n", "") + + _, _, err := AgentInstructionsFromProject(agentService(t, project, "agent", ""), "agent") + + require.Error(t, err) + assert.Contains(t, err.Error(), "outside the project") + assert.NotContains(t, err.Error(), "a file the project does not contain") + }) + + t.Run("reached through a symlink", func(t *testing.T) { + root := t.TempDir() + secret := filepath.Join(root, "outside-secret.md") + require.NoError(t, os.WriteFile(secret, []byte("a file the project does not contain"), 0o600)) + + project := filepath.Join(root, "proj") + baseline := filepath.Join(project, "agent", ".agent_configs", "baseline") + require.NoError(t, os.MkdirAll(baseline, 0o750)) + writeOptimizeConfig(t, filepath.Join(project, "agent"), + "instruction_file: instructions.md\n", "") + + // A symlink is what git materializes on checkout, so it needs no more + // privilege to commit than the `..` above. + link := filepath.Join(baseline, "instructions.md") + if err := os.Symlink(secret, link); err != nil { + if runtime.GOOS == "windows" { + t.Skip("creating a symlink on Windows needs a privilege this run does not have") + } + require.NoError(t, err) + } + + _, _, err := AgentInstructionsFromProject(agentService(t, project, "agent", ""), "agent") + + require.Error(t, err) + assert.Contains(t, err.Error(), "outside the project") + assert.NotContains(t, err.Error(), "a file the project does not contain") + }) +} From d049e35a0eb0bb388900170e14fca06ca72e8d89 Mon Sep 17 00:00:00 2001 From: mohessie Date: Wed, 19 Aug 2026 08:28:43 +0300 Subject: [PATCH 295/320] Say which run, one word for the link, and stop offering a refused source Spec-conformance round over the command surface. A run-scoped command given no id falls back to the last run, which is what makes `run output list` usable, and also how a reader ends up reading a different run than the one they have in mind. It now says which run it settled on, on stderr so a redirected listing does not gain a line, and not at all when the caller named the run or asked for JSON. The link at the bottom of a run was labelled `Report:` while the same link under `dataset show` and `evaluator show` was labelled `Portal:`. One destination, one word, and it is the one the spec fixes and two of the four views already used. `run show` hand-rolled a lowercase `name : / status : / results :` block while every other detail view in the extension goes through emitDetail with Title Case keys -- including `run output show`, printed by the neighbouring command. It uses emitDetail now, and the shared portal writer with it. `--from`'s help listed `file` among the sources it offers, and generate guarantees to refuse that one, naming `dataset create` as the way to register a file. It is still recognized, so asking for it still earns the remedy rather than a list of the others; it is just not advertised. `generate` documented `--wait` and did not have it. Waiting is already its default, so the flag changes nothing, but a script that says what it wants should not be refused for saying it -- the same reasoning already applied on `run start`. --- .../internal/cmd/generate_commands.go | 6 ++ .../internal/cmd/generate_flag_guards_test.go | 20 ++++ .../internal/cmd/portal_test.go | 11 ++- .../azure.ai.evaluations/internal/cmd/run.go | 5 +- .../internal/cmd/run_fallback_named_test.go | 91 +++++++++++++++++++ .../internal/cmd/run_ops.go | 17 ++-- .../internal/cmd/run_output.go | 17 +++- .../internal/cmd/surface_test.go | 28 +++++- .../internal/messages/messages.go | 47 +++------- .../internal/project/artifacts.go | 9 +- 10 files changed, 194 insertions(+), 57 deletions(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_fallback_named_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_commands.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_commands.go index a0246c5e955..3311f243942 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_commands.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_commands.go @@ -57,6 +57,12 @@ func addGenerateFlags(cmd *cobra.Command, f *generateFlags) { "Directory the generated artifact is written to.") cmd.Flags().BoolVar(&f.noWait, "no-wait", false, "Submit the job and return its id without polling.") + // Waiting is already the default, so --wait changes nothing. It is here + // because the spec documents the pair and a script that spells out what it + // wants should not be refused for saying the default out loud. + var wait bool + cmd.Flags().BoolVar(&wait, "wait", true, "Block until the job finishes.") + cmd.MarkFlagsMutuallyExclusive("wait", "no-wait") cmd.Flags().BoolVar(&f.force, "force", false, "Overwrite an artifact file that already exists.") cmd.Flags().StringVar(&f.endpoint, "project-endpoint", "", "Foundry project endpoint.") diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_flag_guards_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_flag_guards_test.go index 6445937d1b9..171e0ef8478 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_flag_guards_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_flag_guards_test.go @@ -126,3 +126,23 @@ func TestOutputDirAloneIsStillAccepted(t *testing.T) { "an output directory without --no-wait must not be refused") } } + +// --wait says the default out loud, so it has to be accepted; asking for both +// at once says two things and has to be refused, as it is on `run start`. +func TestWaitAndNoWaitContradictEachOther(t *testing.T) { + err := runGenerate(t, "--dataset", "--dataset-name", "ds", "--wait", "--no-wait") + + require.Error(t, err) + assert.Contains(t, err.Error(), "no-wait") + assert.Contains(t, err.Error(), "wait") +} + +// And --wait on its own is not a refusal, whatever else the run goes on to do. +func TestWaitAloneIsAccepted(t *testing.T) { + err := runGenerate(t, "--dataset", "--dataset-name", "ds", "--wait") + + if err != nil { + assert.NotContains(t, err.Error(), "--wait", + "--wait names the default, so it must not be refused") + } +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/portal_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/portal_test.go index f4173a516e9..ecf5ae0a9b1 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/portal_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/portal_test.go @@ -97,7 +97,8 @@ func TestPortalRunURLShape(t *testing.T) { // A run has one destination. The service's report_url and the portal URL the // extension builds resolve to the same page, and printing both put two labels -// on it with no rule a reader could infer. +// on it with no rule a reader could infer. One label, and it is the one every +// other view uses. func TestRenderRunPrintsOneLink(t *testing.T) { run := &eval_api.OpenAIEvalRun{ ID: "evalrun_1", @@ -110,10 +111,10 @@ func TestRenderRunPrintsOneLink(t *testing.T) { require.NoError(t, renderRun(&buf, run, nil)) out := buf.String() - assert.Contains(t, out, "Report: https://service.example/report/1", + assert.Contains(t, out, "Portal: https://service.example/report/1", "the service's url wins where it sent one") - assert.NotContains(t, out, "Portal: ", - "the second label named the same destination") + assert.Equal(t, 1, strings.Count(out, "Portal: "), + "a second label would name the same destination") assert.NotContains(t, out, run.PortalURL) } @@ -129,7 +130,7 @@ func TestRenderRunFallsBackToTheBuiltLink(t *testing.T) { var buf bytes.Buffer require.NoError(t, renderRun(&buf, run, nil)) - assert.Contains(t, buf.String(), "Report: "+run.PortalURL) + assert.Contains(t, buf.String(), "Portal: "+run.PortalURL) } // A run with neither prints no link rather than an empty label. diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go index 778afa3d029..ca83e99f42e 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go @@ -22,7 +22,6 @@ import ( "azureaieval/internal/pkg/eval_api" "azureaieval/internal/project" - "github.com/fatih/color" "github.com/spf13/cobra" ) @@ -1004,9 +1003,7 @@ func renderRun( } } - if url := runLink(run.ReportURL, run.PortalURL); url != "" { - fmt.Fprint(out, messages.ReportLink(color.CyanString(url))) - } + writePortalLink(out, runLink(run.ReportURL, run.PortalURL)) return nil } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_fallback_named_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_fallback_named_test.go new file mode 100644 index 00000000000..3e1e209f5fa --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_fallback_named_test.go @@ -0,0 +1,91 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "bytes" + "context" + "net/http" + "net/http/httptest" + "testing" + + "azureaieval/internal/pkg/eval_api" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime" + "github.com/spf13/cobra" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// evalContextListingOneRun builds a context whose service has exactly one run, +// so the run-scoped commands have something to fall back to. +func evalContextListingOneRun(t *testing.T, runID string) *evalContext { + t.Helper() + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte( + `{"data":[{"id":"` + runID + `","status":"completed"}]}`)) + })) + t.Cleanup(srv.Close) + + pipeline := runtime.NewPipeline("test", "v1", runtime.PipelineOptions{}, + &policy.ClientOptions{Retry: policy.RetryOptions{MaxRetries: -1}}) + return &evalContext{evalClient: eval_api.NewEvalClientFromPipeline(srv.URL, pipeline)} +} + +// commandWritingTo returns a command whose two streams can be read apart. +func commandWritingTo(t *testing.T, out, errOut *bytes.Buffer, jsonOutput bool) *cobra.Command { + t.Helper() + cmd := &cobra.Command{Use: "show"} + cmd.SetContext(context.Background()) + cmd.SetOut(out) + cmd.SetErr(errOut) + cmd.Flags().StringP("output", "o", "", "") + if jsonOutput { + require.NoError(t, cmd.Flags().Set("output", "json")) + } + return cmd +} + +// Falling back to the last run is what makes these commands usable without an +// id, and it is also how a reader ends up looking at a different run than they +// think. The id it settled on is named. +func TestTheRunAFallbackSettledOnIsNamed(t *testing.T) { + ec := evalContextListingOneRun(t, "evalrun_last") + + var out, errOut bytes.Buffer + run, err := ec.latestOrNamedRun(commandWritingTo(t, &out, &errOut, false), "eval_1", "", false) + + require.NoError(t, err) + assert.Equal(t, "evalrun_last", run.ID) + assert.Contains(t, errOut.String(), "Using last run: evalrun_last") + assert.Empty(t, out.String(), + "the line is context, so it must not land in a redirected listing") +} + +// A caller who named the run already knows which one it is, and a parser is +// not reading prose. +func TestTheRunIsNotNamedBackToWhoeverNamedIt(t *testing.T) { + t.Run("named explicitly", func(t *testing.T) { + ec := evalContextListingOneRun(t, "evalrun_asked") + + var out, errOut bytes.Buffer + _, err := ec.latestOrNamedRun( + commandWritingTo(t, &out, &errOut, false), "eval_1", "evalrun_asked", true) + + require.NoError(t, err) + assert.Empty(t, errOut.String()) + }) + + t.Run("output is json", func(t *testing.T) { + ec := evalContextListingOneRun(t, "evalrun_last") + + var out, errOut bytes.Buffer + _, err := ec.latestOrNamedRun(commandWritingTo(t, &out, &errOut, true), "eval_1", "", false) + + require.NoError(t, err) + assert.Empty(t, errOut.String()) + }) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go index e8916d89fc5..c42bf5c03d3 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go @@ -11,7 +11,6 @@ import ( "azureaieval/internal/messages" "azureaieval/internal/pkg/eval_api" - "github.com/fatih/color" "github.com/spf13/cobra" ) @@ -180,15 +179,15 @@ func newRunShowCommand() *cobra.Command { } out := cmd.OutOrStdout() - fmt.Fprint(out, messages.RunHeading(run.ID)) - fmt.Fprint(out, messages.RunNameLine(run.Name)) - fmt.Fprint(out, messages.RunStatusDetail(run.Status)) - if counts := summarizeCounts(run.ResultCounts); counts != "" { - fmt.Fprint(out, messages.RunResultsLine(counts)) - } - if url := runLink(run.ReportURL, run.PortalURL); url != "" { - fmt.Fprint(out, messages.RunReportLine(color.CyanString(url))) + if err := emitDetail(out, []field{ + {"Run", run.ID}, + {"Name", run.Name}, + {"Status", run.Status}, + {"Results", summarizeCounts(run.ResultCounts)}, + }); err != nil { + return err } + writePortalLink(out, runLink(run.ReportURL, run.PortalURL)) if gateOnStatus { if err := runCompleted(run); err != nil { return err diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output.go index 0eb2bb36186..3e8b43243e5 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output.go @@ -371,6 +371,7 @@ func (ec *evalContext) latestOrNamedRun( if runID != "" { run, err := ec.evalClient.GetOpenAIEvalRun(ctx, evalID, runID) if err == nil { + ec.sayWhichRun(cmd, explicit, run.ID) return run, nil } if explicit { @@ -388,9 +389,23 @@ func (ec *evalContext) latestOrNamedRun( if list == nil || len(list.Data) == 0 { return nil, messages.EvalHasNoRuns(evalID) } + ec.sayWhichRun(cmd, explicit, list.Data[0].ID) return &list.Data[0], nil } +// sayWhichRun names the run a command settled on for itself. +// +// The fallback is the reason these commands are usable without an id, and it +// is also the reason a reader can be looking at a different run than they +// think. Naming it costs one line and removes the doubt. A caller that named +// the run already knows, and JSON is parsed rather than read. +func (ec *evalContext) sayWhichRun(cmd *cobra.Command, explicit bool, runID string) { + if explicit || isJSON(cmd) { + return + } + fmt.Fprint(cmd.ErrOrStderr(), messages.UsingLastRun(runID)) +} + // renderOutputItem is the detail view for one evaluated row. // // This was the one `show` that emitted raw JSON whatever was asked for, which @@ -618,7 +633,7 @@ func renderResults( } if url := runLink(run.ReportURL, run.PortalURL); url != "" { - fmt.Fprint(w, messages.ReportLinkAfterRows(color.CyanString(url))) + fmt.Fprint(w, messages.PortalLinkAfterRows(color.CyanString(url))) } return nil } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/surface_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/surface_test.go index 9a5f26b3372..1f0125d6421 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/surface_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/surface_test.go @@ -238,14 +238,36 @@ func TestGenerateFromTakesMoreThanOneSource(t *testing.T) { "--from selects one or more sources, so it cannot be a single string") } -// `--from` names sources; the set it accepts is the set the service has a path -// for, and the help has to list exactly that set. +// `--from` names sources; the set it offers is the set generate can build +// from, and the help has to list exactly that set. func TestGenerateFromListsEverySource(t *testing.T) { usage := find(t, "generate").Flags().Lookup("from").Usage for _, source := range project.GenerateSources { assert.Containsf(t, usage, source, - "--from accepts %q, so its help has to say so", source) + "--from offers %q, so its help has to say so", source) + } +} + +// `file` is recognized so that asking for it earns the remedy rather than a +// list of the others, but generate never builds from one. Advertising it would +// offer a value the same command guarantees to refuse. +func TestGenerateDoesNotOfferTheSourceItAlwaysRefuses(t *testing.T) { + usage := find(t, "generate").Flags().Lookup("from").Usage + + assert.NotContains(t, usage, project.GenerateFromFile, + "--from file is refused, so the help must not list it") + assert.NoError(t, project.ValidateGenerateSource(project.GenerateFromFile), + "it is still recognized, so the refusal can name what to do instead") +} + +// Waiting is already generate's default, so --wait changes nothing. A script +// that spells out what it wants should not be refused for saying so. +func TestGenerateTakesTheWaitFlagItDocuments(t *testing.T) { + flags := find(t, "generate").Flags() + + for _, name := range []string{"wait", "no-wait"} { + assert.NotNilf(t, flags.Lookup(name), "generate must offer --%s", name) } } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go b/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go index c7ab1fc471b..cb3583661ac 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go @@ -232,11 +232,6 @@ func ErroredNotScored(errored int) string { return fmt.Sprintf("(%d errored, not scored)", errored) } -// ReportLink closes a run summary with the one link the run has. -func ReportLink(url string) string { - return fmt.Sprintf("Report: %s\n", url) -} - // EvalNotDeployed reports an eval id the project does not hold. func EvalNotDeployed(evalID, deployCmd string) error { return fmt.Errorf( @@ -292,31 +287,6 @@ func ReadingRun(runID string, err error) error { return fmt.Errorf("reading run %s: %w", runID, err) } -// RunHeading opens the detail view of one run. -func RunHeading(runID string) string { - return fmt.Sprintf("Run %s\n", runID) -} - -// RunNameLine reports the run's name in the detail view. -func RunNameLine(name string) string { - return fmt.Sprintf(" name : %s\n", name) -} - -// RunStatusDetail reports the run's status in the detail view. -func RunStatusDetail(status string) string { - return fmt.Sprintf(" status : %s\n", status) -} - -// RunResultsLine reports the run's counts in the detail view. -func RunResultsLine(counts string) string { - return fmt.Sprintf(" results : %s\n", counts) -} - -// RunReportLine reports the run's one link in the detail view. -func RunReportLine(url string) string { - return fmt.Sprintf(" report : %s\n", url) -} - // CountsSummary renders a run's verdict counts on one line. func CountsSummary(passed, failed, errored int) string { return fmt.Sprintf("%d passed, %d failed, %d errored", passed, failed, errored) @@ -472,9 +442,20 @@ func OutputItemReason(reason string) string { return fmt.Sprintf(" %s\n", reason) } -// ReportLinkAfterRows closes a per-sample listing with the run's one link. -func ReportLinkAfterRows(url string) string { - return fmt.Sprintf("\nReport: %s\n", url) +// UsingLastRun names the run a command chose when it was not given one. +// +// Written to stderr so it does not land in a redirected listing. +func UsingLastRun(runID string) string { + return fmt.Sprintf("Using last run: %s\n", runID) +} + +// PortalLinkAfterRows closes a per-sample listing with the run's one link. +// +// Labelled the way every other view labels it: the run's report page is in the +// portal, and a reader looking for the link should not have to know two words +// for it. +func PortalLinkAfterRows(url string) string { + return fmt.Sprintf("\nPortal: %s\n", url) } // ExportFormatUnsupported reports an --format the export command cannot write. diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/artifacts.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/artifacts.go index 1a7277c390a..ef8ac69ed64 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/artifacts.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/artifacts.go @@ -43,9 +43,14 @@ const ( GenerateFromFile = "file" ) -// GenerateSources is what --from accepts, in help order. +// GenerateSources is what --from offers, in help order. +// +// `file` is missing on purpose. It is a source the command recognizes so that +// asking for it earns the remedy rather than a list, but generate never builds +// from one, so offering it in help would advertise a value the same command +// guarantees to refuse. var GenerateSources = []string{ - GenerateFromTraces, GenerateFromAgent, GenerateFromPrompt, GenerateFromFile, + GenerateFromTraces, GenerateFromAgent, GenerateFromPrompt, } // ValidateGenerateSource rejects a --from value the service has no path for. From 9381f80c340aa6effc9fa1a12806aea035bd1b3f Mon Sep 17 00:00:00 2001 From: mohessie Date: Wed, 19 Aug 2026 09:05:31 +0300 Subject: [PATCH 296/320] Close the gap between the two extensions' shared code A drift review across the files the two extensions are meant to hold in common. Six fixes had landed on one side only, in both directions. Two of them decide what a user is told when a request never reaches an answer. "failed to acquire a token" was matched anywhere in an error's text, which any error is free to contain, so a service that could not acquire a token bucket lease was told its login had expired and to run `azd auth login`. And a credential that never ran at all -- azd not on PATH -- was reported as an expired login, advising a command that cannot be run with the tool that is missing. The dataset extension now classifies both the way the eval extension does, with the tests that hold it there. build.ps1 stamped local time and labelled it Z, so every binary built outside UTC reported a build date wrong by the local offset while claiming to be UTC. The build.sh beside it was already right. A test wrote its assertion with require inside an httptest handler, which runs on the server's goroutine: FailNow there aborts mid-response and fails whichever test happens to be running. One path message printed backslashes where every other message in the same file prints forward slashes. And in the other direction: a --from-file that does not exist was reported to the eval extension's user as a raw syscall name, while the dataset extension already said "does not exist". That fix and its test move across. Also corrected three comments claiming azd does not surface an extension's stderr. It does, on a direct invocation -- checked. What it does not surface is stderr under `azd up`, which is the case those two warnings are routed around, and a reviewer had already read the short form as the blanket claim. --- .../azure.ai.evaluations/internal/cmd/dataset.go | 5 +++-- .../internal/cmd/dataset_source_test.go | 11 +++++++++++ .../azure.ai.evaluations/internal/cmd/run.go | 3 ++- .../internal/messages/messages.go | 6 ++++++ 4 files changed, 22 insertions(+), 3 deletions(-) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset.go index 055bb60e1ac..c79f6cb2ff4 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset.go @@ -152,8 +152,9 @@ func newDatasetWriteCommand(verb, short string) *cobra.Command { if err := ec.setEnvValue(ctx, envKeyDatasetVersion, ds.Version); err != nil { // Persisting is a convenience, so this never fails the command. // It goes to stdout because azd does not surface an extension's - // stderr, and is skipped outside a project, where having nowhere - // to persist is expected rather than notable. + // stderr under `azd up`, which is where a deploy would lose it. + // Skipped outside a project, where having nowhere to persist is + // expected rather than notable. if !errors.Is(err, errNoAzdEnvironment) && !isJSON(cmd) { fmt.Fprint(cmd.OutOrStdout(), messages.Warning(err)) } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset_source_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset_source_test.go index bdf9ec25eec..b1936b1e025 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset_source_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset_source_test.go @@ -66,6 +66,17 @@ func TestDatasetUploadSourceRefusesADirectoryWithNoJSONL(t *testing.T) { assert.Contains(t, err.Error(), "no .jsonl file") } +// A mistyped --from-file is the common way to get here, and the syscall that +// discovered it says nothing to the person who mistyped it. +func TestDatasetUploadSourceOnAMissingPath(t *testing.T) { + _, err := datasetUploadSource(filepath.Join(t.TempDir(), "nope.jsonl")) + + require.Error(t, err) + assert.Contains(t, err.Error(), "does not exist") + assert.NotContains(t, err.Error(), "GetFileAttributesEx") + assert.NotContains(t, err.Error(), "stat ") +} + // The service refuses a bad name with a 400 wrapping four levels of JSON, so // the guard exists to say it plainly. The sibling extension had it; this copy, // which serves the same commands under `azd ai eval dataset`, did not. diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go index ca83e99f42e..931f4e876d8 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go @@ -226,7 +226,8 @@ func buildRunCommand(use, short string) *cobra.Command { if err := ec.setEnvValue(ctx, envKeyEvalRunID, run.ID); err != nil { // Persisting the run id is a convenience for later commands. // Reported on stdout because azd does not surface an - // extension's stderr, and skipped outside a project. + // extension's stderr under `azd up`, which is where a deploy + // would lose it. Skipped outside a project. if !errors.Is(err, errNoAzdEnvironment) && !isJSON(cmd) { fmt.Fprint(out, messages.Warning(err)) } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go b/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go index cb3583661ac..319f8d41b87 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go @@ -899,7 +899,13 @@ func JSONLNoRows(path string) error { } // ReadingFromFile reports a --from-file that would not stat. +// +// A path that is simply absent is reported as absent: the wrapped error is a +// syscall name that says nothing to the person who mistyped it. func ReadingFromFile(path string, err error) error { + if errors.Is(err, fs.ErrNotExist) { + return fmt.Errorf("--from-file %q does not exist", filepath.ToSlash(path)) + } return fmt.Errorf("reading --from-file %q: %w", filepath.ToSlash(path), err) } From ebfdb13fb74958c9e886459afc7cad9f6ccd9831 Mon Sep 17 00:00:00 2001 From: mohessie Date: Wed, 19 Aug 2026 09:34:05 +0300 Subject: [PATCH 297/320] Fix what the last two commits got wrong, and the escape they left open An adversarial round over the newest code found three defects introduced by the commits that were meant to fix things, and one that was already there. Expanding an alias makes the entry longer than the `*name` it replaced, so the line number rebased onto the file from the snippet landed further down than the typo -- on a valid, unrelated entry, or past the end of the file. A confident accusation against correct code is worse than no line at all, so where anything expanded the entry itself is named. Aliases are also resolved once now rather than twice: the second pass found nothing left and reported that nothing had expanded. Containment refused a symlink and let a Windows directory junction through. Go reads a junction as an irregular file rather than a link, so EvalSymlinks refuses the path while the OS walks the read straight through it -- and the previous code read that refusal as "there is nothing here to follow". Only a path that is genuinely absent is let past now. This is the cheaper of the two escapes: mklink /J needs neither elevation nor Developer Mode. `generate --wait` was parsed into a variable nobody read. `--wait=false` is a legal spelling and asks for exactly what --no-wait asks for; accepted and discarded, it made the command wait. The run detail view drops a field with no value, which is right for a field nobody needs and wrong for the one the command exists to report: status is omitempty on the wire, and the row simply vanished. It says so instead. Also, an entry that is itself an alias -- `- *base`, the most natural thing an anchor is for -- was refused before the alias was looked at, and refused by printing yaml's own enum: "must be a mapping, got 16". Both fixed. And the dataset extension's error codes still carried the toolbox and skill vocabulary they were copied from, 34 codes and 9 operation names for resources it has no concept of, which the eval copy had already been cleaned of. The merge-key test added last commit put its typo on the anchor rather than on the entry that merges it, so decoding stopped before the merge was ever read. --- .../internal/cmd/generate_commands.go | 11 ++- .../internal/cmd/generate_flag_guards_test.go | 16 ++++ .../internal/cmd/run_ops.go | 18 +++- .../internal/cmd/run_status_row_test.go | 35 +++++++ .../internal/messages/messages.go | 8 +- .../internal/pkg/evalcore/evaluator.go | 90 ++++++++++++----- .../pkg/evalcore/evaluator_anchor_test.go | 96 ++++++++++++++++++- .../internal/project/agent_instructions.go | 15 ++- .../project/instruction_containment_test.go | 37 +++++++ 9 files changed, 290 insertions(+), 36 deletions(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_status_row_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_commands.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_commands.go index 3311f243942..1a1b90d6fc9 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_commands.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_commands.go @@ -57,12 +57,17 @@ func addGenerateFlags(cmd *cobra.Command, f *generateFlags) { "Directory the generated artifact is written to.") cmd.Flags().BoolVar(&f.noWait, "no-wait", false, "Submit the job and return its id without polling.") - // Waiting is already the default, so --wait changes nothing. It is here - // because the spec documents the pair and a script that spells out what it - // wants should not be refused for saying the default out loud. + // Waiting is already the default, so --wait only changes anything when it is + // turned off. Parsed into a variable nobody reads, `--wait=false` -- a legal + // spelling -- would be accepted and then do the opposite of what it says. var wait bool cmd.Flags().BoolVar(&wait, "wait", true, "Block until the job finishes.") cmd.MarkFlagsMutuallyExclusive("wait", "no-wait") + cmd.PreRun = func(*cobra.Command, []string) { + if !wait { + f.noWait = true + } + } cmd.Flags().BoolVar(&f.force, "force", false, "Overwrite an artifact file that already exists.") cmd.Flags().StringVar(&f.endpoint, "project-endpoint", "", "Foundry project endpoint.") diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_flag_guards_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_flag_guards_test.go index 171e0ef8478..70c69d785c3 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_flag_guards_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_flag_guards_test.go @@ -146,3 +146,19 @@ func TestWaitAloneIsAccepted(t *testing.T) { "--wait names the default, so it must not be refused") } } + +// `--wait=false` is a legal pflag spelling and means what --no-wait means. +// Parsed into a variable nobody reads, it was accepted and then did the +// opposite of what it said: the command waited. +func TestWaitFalseMeansTheSameAsNoWait(t *testing.T) { + withNoWait := runGenerate(t, + "--dataset", "--dataset-name", "ds", "--no-wait", "--output-dir", t.TempDir()) + withWaitFalse := runGenerate(t, + "--dataset", "--dataset-name", "ds", "--wait=false", "--output-dir", t.TempDir()) + + require.Error(t, withNoWait, "the guard this compares against has to still fire") + require.Error(t, withWaitFalse, + "--wait=false asks for the same thing --no-wait asks for") + assert.Equal(t, withNoWait.Error(), withWaitFalse.Error(), + "the two spellings must reach the same decision") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go index c42bf5c03d3..2e24aea4865 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go @@ -182,7 +182,11 @@ func newRunShowCommand() *cobra.Command { if err := emitDetail(out, []field{ {"Run", run.ID}, {"Name", run.Name}, - {"Status", run.Status}, + // emitDetail drops an empty value, and status is `omitempty` on + // the wire. Reporting the status is what this command is for, so + // a run the service sent none for says that rather than losing + // the row and reading as a renderer that forgot it. + {"Status", reportedStatus(run.Status)}, {"Results", summarizeCounts(run.ResultCounts)}, }); err != nil { return err @@ -216,6 +220,18 @@ func firstArg(args []string) string { return "" } +// reportedStatus names a status the service did not send. +// +// `status` is omitempty on the wire, and the detail view drops an empty value. +// Saying the service reported none is information; dropping the row looks like +// the renderer forgot it. +func reportedStatus(status string) string { + if status == "" { + return "not reported" + } + return status +} + func newRunCancelCommand() *cobra.Command { var ( endpointFlg string diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_status_row_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_status_row_test.go new file mode 100644 index 00000000000..e02712264a8 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_status_row_test.go @@ -0,0 +1,35 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "bytes" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// The detail view drops a field with no value, which is right for a field the +// reader does not need and wrong for the one the command exists to report. +// `status` is omitempty on the wire, and `run cancel` already carries a guard +// for the same reason. +func TestRunShowSaysWhenTheServiceReportedNoStatus(t *testing.T) { + assert.Equal(t, "not reported", reportedStatus(""), + "a missing status has to read as missing, not as a row nobody rendered") + assert.Equal(t, "completed", reportedStatus("completed"), + "a status the service did send is passed through untouched") +} + +// And the row survives the renderer that would otherwise drop it. +func TestARunWithNoStatusStillShowsTheStatusRow(t *testing.T) { + var buf bytes.Buffer + require.NoError(t, emitDetail(&buf, []field{ + {"Run", "evalrun_1"}, + {"Status", reportedStatus("")}, + })) + + assert.Contains(t, buf.String(), "Status") + assert.Contains(t, buf.String(), "not reported") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go b/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go index 319f8d41b87..d60e67228de 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go @@ -1335,8 +1335,8 @@ func BareEvaluatorEntry(name string) error { } // EvaluatorsMustBeSequence reports an evaluators: block that is not a list. -func EvaluatorsMustBeSequence(kind any) error { - return fmt.Errorf("evaluators must be a sequence, got %v", kind) +func EvaluatorsMustBeSequence(kind string) error { + return fmt.Errorf("evaluators must be a list, got %s", kind) } // EvaluatorsMustBeList reports an evaluators: block that is not a JSON array. @@ -1360,8 +1360,8 @@ func EvaluatorEntryMissingEvaluator() error { } // EvaluatorEntryMustBeMapping reports an entry that is neither map nor string. -func EvaluatorEntryMustBeMapping(kind any) error { - return fmt.Errorf("evaluator entry must be a mapping, got %v", kind) +func EvaluatorEntryMustBeMapping(kind string) error { + return fmt.Errorf("evaluator entry must be a mapping, got %s", kind) } // EvaluatorAliasIsCircular reports an anchor that contains its own alias. diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/evaluator.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/evaluator.go index 12b4d9a8267..ff1e53615f9 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/evaluator.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/evaluator.go @@ -81,11 +81,21 @@ type EvaluatorList []EvaluatorRef func (el *EvaluatorList) UnmarshalYAML(value *yaml.Node) error { if value.Kind != yaml.SequenceNode { - return messages.EvaluatorsMustBeSequence(value.Kind) + return messages.EvaluatorsMustBeSequence(nodeKindName(value.Kind)) } result := make([]EvaluatorRef, 0, len(value.Content)) - for _, node := range value.Content { + for _, entry := range value.Content { + // Resolved before the kind is read, so `- *base` -- an entry that is + // itself an alias, which is the most natural thing an anchor is for -- + // is read as the mapping it names rather than refused for being an + // alias. Resolved once: doing it again inside the decode would find + // nothing left and report that nothing had expanded. + node, expanded, err := resolveAliases(entry, map[*yaml.Node]bool{}) + if err != nil { + return err + } + switch node.Kind { case yaml.ScalarNode: var name string @@ -94,7 +104,7 @@ func (el *EvaluatorList) UnmarshalYAML(value *yaml.Node) error { } return messages.BareEvaluatorEntry(name) case yaml.MappingNode: - ref, err := decodeEvaluatorRef(node) + ref, err := decodeEvaluatorRef(node, entry.Line, expanded) if err != nil { return err } @@ -103,7 +113,7 @@ func (el *EvaluatorList) UnmarshalYAML(value *yaml.Node) error { } result = append(result, ref) default: - return messages.EvaluatorEntryMustBeMapping(node.Kind) + return messages.EvaluatorEntryMustBeMapping(nodeKindName(node.Kind)) } } @@ -111,20 +121,41 @@ func (el *EvaluatorList) UnmarshalYAML(value *yaml.Node) error { return nil } -// decodeEvaluatorRef decodes one entry with the strictness the file promises. +// nodeKindName names a yaml.Kind in the file's vocabulary. +// +// yaml.Kind is an unnamed uint32 with no String method, so a message built with +// %v read "evaluator entry must be a mapping, got 16". +func nodeKindName(kind yaml.Kind) string { + switch kind { + case yaml.DocumentNode: + return "a document" + case yaml.SequenceNode: + return "a list" + case yaml.MappingNode: + return "a mapping" + case yaml.ScalarNode: + return "a single value" + case yaml.AliasNode: + return "an alias" + default: + return "something else" + } +} + +// decodeEvaluatorRef decodes one already-resolved entry with the strictness the +// file promises. // // yaml.Node.Decode does not inherit KnownFields from the decoder that reached // it, so `verison:` inside an evaluator entry was dropped in silence while the // same typo one level up was named. Round-tripping the node through a strict // decoder restores it; the error keeps yaml's own "field X not found in type Y" // shape, which the caller rewrites into the file's vocabulary. -func decodeEvaluatorRef(node *yaml.Node) (EvaluatorRef, error) { - resolved, err := resolveAliases(node, map[*yaml.Node]bool{}) - if err != nil { - return EvaluatorRef{}, err - } - - raw, err := yaml.Marshal(resolved) +// +// entryLine is where the entry begins in the file, and expanded says whether an +// alias was replaced on the way here -- between them they decide how much of +// the snippet's line numbering can be trusted back onto the file. +func decodeEvaluatorRef(node *yaml.Node, entryLine int, expanded bool) (EvaluatorRef, error) { + raw, err := yaml.Marshal(node) if err != nil { return EvaluatorRef{}, messages.DecodingEvaluator(err) } @@ -134,12 +165,13 @@ func decodeEvaluatorRef(node *yaml.Node) (EvaluatorRef, error) { var ref EvaluatorRef if err := decoder.Decode(&ref); err != nil { - return EvaluatorRef{}, messages.DecodingEvaluator(rebaseYAMLLines(err, node.Line)) + return EvaluatorRef{}, messages.DecodingEvaluator(rebaseYAMLLines(err, entryLine, expanded)) } return ref, nil } -// resolveAliases copies node with every alias replaced by what it names. +// resolveAliases copies node with every alias replaced by what it names, and +// reports whether it replaced any. // // The strict decode above re-serializes one entry, which lifts it out of the // document and away from the anchors its aliases point at: `*judge` defined on @@ -149,14 +181,15 @@ func decodeEvaluatorRef(node *yaml.Node) (EvaluatorRef, error) { // // active holds the anchors being expanded on this path. yaml permits an anchor // that contains its own alias, which would otherwise expand forever. -func resolveAliases(node *yaml.Node, active map[*yaml.Node]bool) (*yaml.Node, error) { +func resolveAliases(node *yaml.Node, active map[*yaml.Node]bool) (*yaml.Node, bool, error) { if node.Kind == yaml.AliasNode && node.Alias != nil { if active[node.Alias] { - return nil, messages.EvaluatorAliasIsCircular(node.Value) + return nil, false, messages.EvaluatorAliasIsCircular(node.Value) } active[node.Alias] = true defer delete(active, node.Alias) - return resolveAliases(node.Alias, active) + resolved, _, err := resolveAliases(node.Alias, active) + return resolved, true, err } // The anchor is dropped with the alias it fed: keeping it would emit the @@ -164,14 +197,16 @@ func resolveAliases(node *yaml.Node, active map[*yaml.Node]bool) (*yaml.Node, er copied := *node copied.Anchor = "" copied.Content = make([]*yaml.Node, len(node.Content)) + expanded := false for i, child := range node.Content { - resolvedChild, err := resolveAliases(child, active) + resolvedChild, childExpanded, err := resolveAliases(child, active) if err != nil { - return nil, err + return nil, false, err } copied.Content[i] = resolvedChild + expanded = expanded || childExpanded } - return &copied, nil + return &copied, expanded, nil } // yamlErrorLine matches the line number yaml puts on each unmarshal error. @@ -179,16 +214,25 @@ var yamlErrorLine = regexp.MustCompile(`line (\d+):`) // rebaseYAMLLines moves line numbers from the extracted snippet back onto the // file, so the reader is pointed at the key they typed rather than at line 2. -func rebaseYAMLLines(err error, startLine int) error { - if startLine <= 0 { +// +// Only line-for-line while the snippet is what the file holds. An alias expands +// to more lines than the `*name` it replaced, so an offset computed from the +// snippet lands somewhere further down the file -- on a valid, unrelated entry, +// or past the end. Where anything expanded the whole entry is named instead: +// less precise, and never a confident accusation against code that is fine. +func rebaseYAMLLines(err error, entryLine int, expanded bool) error { + if entryLine <= 0 { return err } return errors.New(yamlErrorLine.ReplaceAllStringFunc(err.Error(), func(m string) string { + if expanded { + return fmt.Sprintf("line %d:", entryLine) + } n, convErr := strconv.Atoi(yamlErrorLine.FindStringSubmatch(m)[1]) if convErr != nil { return m } - return fmt.Sprintf("line %d:", startLine+n-1) + return fmt.Sprintf("line %d:", entryLine+n-1) })) } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/evaluator_anchor_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/evaluator_anchor_test.go index e11a2b5c444..b7df0f16fda 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/evaluator_anchor_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/evaluator_anchor_test.go @@ -4,6 +4,9 @@ package evalcore import ( + "regexp" + "strconv" + "strings" "testing" "github.com/stretchr/testify/assert" @@ -71,13 +74,16 @@ evaluators: - evaluator: builtin.relevance verison: 3 `, + // The first entry is clean on purpose. Decoding stops at the first + // entry that fails, so a typo written into the anchor would satisfy + // this without the merge ever being read. "inherited through a merge": ` evaluators: - &base evaluator: builtin.relevance - verison: 3 - <<: *base evaluator: builtin.coherence + verison: 3 `, } @@ -91,6 +97,94 @@ evaluators: } } +// An entry that is itself an alias is the most natural thing an anchor is for. +// It used to be refused before the alias was ever looked at, and refused with +// yaml's own numeric kind: "must be a mapping, got 16". +func TestAnEntryThatIsItselfAnAliasResolves(t *testing.T) { + list, err := decodeEvaluators(t, ` +anchors: + - &shared + evaluator: builtin.relevance + initialization_parameters: + deployment_name: gpt-4o +evaluators: + - *shared + - evaluator: builtin.coherence +`) + + require.NoError(t, err) + require.Len(t, list, 2) + assert.Equal(t, "builtin.relevance", list[0].Evaluator) + assert.Equal(t, map[string]any{"deployment_name": "gpt-4o"}, list[0].InitializationParameters) +} + +// A kind the file cannot use is named in words. yaml.Kind is an unnamed uint32 +// with no String method, so %v printed the bit value. +func TestARefusedEntryNamesTheShapeInWords(t *testing.T) { + _, err := decodeEvaluators(t, ` +evaluators: + - - evaluator: builtin.relevance +`) + + require.Error(t, err) + assert.Contains(t, err.Error(), "a list") + assert.NotContains(t, err.Error(), "16", "the reader is not reading yaml's enum") +} + +// The line a typo is reported on has to be a line the reader can look at. +// +// Expanding an alias makes the snippet longer than the entry it came from, so +// an offset computed from the snippet lands further down the file -- on a +// different, valid entry, or past the end. +func TestAMisspeltKeyPointsAtTheEntryAndNotPastIt(t *testing.T) { + doc := `evaluators: + - evaluator: builtin.relevance + initialization_parameters: &judge + deployment_name: gpt-4o + api_version: "2026-01-01" + temperature: 0 + - evaluator: builtin.coherence + initialization_parameters: *judge + verison: 3 + - evaluator: builtin.fluency + - evaluator: builtin.groundedness + - evaluator: builtin.similarity +` + _, err := decodeEvaluators(t, doc) + + require.Error(t, err) + require.Contains(t, err.Error(), "verison") + + reported := reportedLine(t, err) + assert.LessOrEqual(t, reported, len(strings.Split(doc, "\n")), + "a line past the end of the file is no help at all") + assert.Equal(t, 7, reported, + "the entry holding the typo begins on line 7; naming another entry accuses code that is fine") +} + +// Without an alias the snippet is line-for-line with the file, so the exact +// key keeps being named. +func TestAMisspeltKeyWithoutAnAliasStillNamesItsOwnLine(t *testing.T) { + _, err := decodeEvaluators(t, `evaluators: + - evaluator: builtin.relevance + - evaluator: builtin.coherence + verison: 3 +`) + + require.Error(t, err) + assert.Equal(t, 4, reportedLine(t, err), "the typo is on line 4") +} + +// reportedLine reads the line number out of a decode error. +func reportedLine(t *testing.T, err error) int { + t.Helper() + match := regexp.MustCompile(`line (\d+):`).FindStringSubmatch(err.Error()) + require.Len(t, match, 2, "the error should name a line: %v", err) + n, convErr := strconv.Atoi(match[1]) + require.NoError(t, convErr) + return n +} + // yaml permits an anchor holding its own alias. Expanding it has no end, so it // has to be refused rather than followed. func TestAnAnchorThatContainsItselfIsRefused(t *testing.T) { diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/agent_instructions.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/agent_instructions.go index 02079ad1737..7beea0cc905 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/agent_instructions.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/agent_instructions.go @@ -4,6 +4,8 @@ package project import ( + "errors" + "io/fs" "os" "path/filepath" "sort" @@ -106,7 +108,7 @@ func AgentInstructionsFromProject( // withinDir reports whether path resolves to somewhere inside root. // // Asked twice: once of the path as written, and once of what it resolves to. -// The read that follows this check follows symlinks, so a link committed to the +// The read that follows this check follows links, so a link committed to the // repository would otherwise satisfy the written form while naming a file the // project does not contain — the same escape `..` is refused for, needing no // more privilege to commit. @@ -118,9 +120,14 @@ func withinDir(root, path string) bool { realRoot, rootErr := filepath.EvalSymlinks(root) realPath, pathErr := filepath.EvalSymlinks(path) if rootErr != nil || pathErr != nil { - // Nothing to follow: a path that is not there yet leads nowhere, and - // the read below reports it as missing rather than as an escape. - return true + // Resolution failing is not the same as there being nothing to + // resolve. A Windows junction is the difference: Go reads it as an + // irregular file rather than a link, so EvalSymlinks refuses the path + // while the OS walks the read straight through it. Only a path that is + // genuinely absent is let past, and the read reports that as missing + // rather than as an escape. + _, lstatErr := os.Lstat(path) + return errors.Is(lstatErr, fs.ErrNotExist) } return liesWithin(realRoot, realPath) } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/instruction_containment_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/instruction_containment_test.go index e0483582b3a..0ddb1ced0e6 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/instruction_containment_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/instruction_containment_test.go @@ -5,6 +5,7 @@ package project import ( "os" + "os/exec" "path/filepath" "runtime" "testing" @@ -90,4 +91,40 @@ func TestGenerateRefusesInstructionsFromOutsideTheProject(t *testing.T) { assert.Contains(t, err.Error(), "outside the project") assert.NotContains(t, err.Error(), "a file the project does not contain") }) + + // A junction is the Windows escape, and it is the cheaper of the two: + // mklink /J needs neither elevation nor Developer Mode, while the symlink + // above needs one of them. Go reads a junction as an irregular file rather + // than a link, so path resolution refuses it while the read walks through. + t.Run("reached through a directory junction", func(t *testing.T) { + if runtime.GOOS != "windows" { + t.Skip("junctions are a Windows reparse point") + } + + root := t.TempDir() + outside := filepath.Join(root, "outside") + require.NoError(t, os.MkdirAll(outside, 0o750)) + require.NoError(t, os.WriteFile(filepath.Join(outside, "instructions.md"), + []byte("a file the project does not contain"), 0o600)) + + project := filepath.Join(root, "proj") + serviceDir := filepath.Join(project, "agent") + require.NoError(t, os.MkdirAll(serviceDir, 0o750)) + writeOptimizeConfig(t, serviceDir, "instruction_file: elsewhere/instructions.md\n", "") + + // The pointer stays inside the project as written; the directory it + // names is the reparse point. + junction := filepath.Join(serviceDir, ".agent_configs", "baseline", "elsewhere") + //nolint:gosec // fixed arguments, both paths built by this test + out, err := exec.Command("cmd", "/c", "mklink", "/J", junction, outside).CombinedOutput() + if err != nil { + t.Skipf("could not create a junction: %v: %s", err, out) + } + + _, _, err = AgentInstructionsFromProject(agentService(t, project, "agent", ""), "agent") + + require.Error(t, err) + assert.Contains(t, err.Error(), "outside the project") + assert.NotContains(t, err.Error(), "a file the project does not contain") + }) } From 9e4e1154735887a6685e75d04cb93d8ba629fc27 Mon Sep 17 00:00:00 2001 From: mohessie Date: Wed, 19 Aug 2026 10:04:46 +0300 Subject: [PATCH 298/320] Deploy from a subdirectory, and stop a row cap forking the eval Four findings from a round aimed at the parts earlier rounds had not reached: the azd up path, the reconciler's change detection, and the generation flow. `azd up` resolved a declaration's `source:` against this process's working directory. serviceRelativeDir answers relative to the project -- that is what a service's $ref and relativePath are written relative to -- and azd neither changes the extension's directory nor reports the project through it: azure.yaml is found by walking up from wherever the caller stood, and AZD_CWD carries the --cwd flag and nothing else. So `azd up` from any subdirectory reported every dataset as not yet generated and offered a remedy that would bill a generation job to rewrite a file already on disk. The join agent_instructions.go already does with the same helper is now done here too. max_samples was inside the eval's fingerprint. It caps the rows this CLI sends on a run and never reaches the eval the service stores, so raising it recreated the eval for a change the service cannot see: the declaration pointed at a new id, every run recorded before it became reachable only by the old one, and the service was left holding two identical definitions under one name. `generate` with an --output-dir naming a file gave the dataset and the evaluator the same path -- the extension is recognized for either kind -- and writes them concurrently. Two billed jobs, one file, and a configuration claiming a dataset and an evaluator that are the same bytes. Refused, naming the way to generate one of them. And `run show --wait` surfaced an internal sentinel as "wait budget spent" with no run id and no way to pick it back up, where `run start` answers the same condition with a reattach line, and with a refusal rather than a silent pass when a gate was asked for. --- .../internal/cmd/generate_composite.go | 8 ++ .../internal/cmd/generate_flag_guards_test.go | 39 ++++++++++ .../internal/cmd/run_ops.go | 13 ++++ .../internal/messages/messages.go | 12 +++ .../internal/project/artifacts.go | 6 ++ .../internal/project/deploy_paths_test.go | 77 +++++++++++++++++++ .../internal/project/service_target_eval.go | 34 +++++++- 7 files changed, 188 insertions(+), 1 deletion(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/project/deploy_paths_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_composite.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_composite.go index 59eee0fbbed..cd227feae9a 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_composite.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_composite.go @@ -67,6 +67,14 @@ func newGenerateCommand() *cobra.Command { if flags.noWait && cmd.Flags().Changed("output-dir") { return messages.OutputDirNeedsTheWait() } + // One command builds two artifacts. --output-dir naming a file + // gives both of them the same path -- the extension is recognized + // for either kind -- and they are written concurrently, so two + // billed jobs would leave one file and a configuration claiming a + // dataset and an evaluator that are the same bytes. + if dataset && evaluator && project.OutputDirNamesAFile(flags.outputDir) { + return messages.OutputFileCannotHoldBothArtifacts(flags.outputDir) + } if dataset { for _, src := range from { diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_flag_guards_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_flag_guards_test.go index 70c69d785c3..435a3e876d4 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_flag_guards_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_flag_guards_test.go @@ -6,6 +6,7 @@ package cmd import ( "bytes" "context" + "path/filepath" "testing" "github.com/stretchr/testify/assert" @@ -162,3 +163,41 @@ func TestWaitFalseMeansTheSameAsNoWait(t *testing.T) { assert.Equal(t, withNoWait.Error(), withWaitFalse.Error(), "the two spellings must reach the same decision") } + +// One command builds two artifacts, so an --output-dir naming a file gives both +// of them the same path. The extension is recognized for either kind and the +// two jobs run concurrently, so two billed jobs would leave one file and a +// configuration claiming a dataset and an evaluator that are the same bytes. +func TestOneOutputFileForTwoArtifactsIsRefused(t *testing.T) { + for _, named := range []string{"both.jsonl", "both.json"} { + t.Run(named, func(t *testing.T) { + err := runGenerate(t, "--output-dir", filepath.Join(t.TempDir(), named)) + + require.Error(t, err) + assert.Contains(t, err.Error(), "--output-dir") + assert.Contains(t, err.Error(), "--dataset", + "the refusal has to name the way to generate one of them") + }) + } +} + +// Naming one artifact is what makes a file path unambiguous, so it stays +// accepted. +func TestOneOutputFileForOneArtifactIsAccepted(t *testing.T) { + err := runGenerate(t, "--dataset", "--dataset-name", "ds", + "--output-dir", filepath.Join(t.TempDir(), "rows.jsonl")) + + if err != nil { + assert.NotContains(t, err.Error(), "names a file", + "one artifact can be written to one file") + } +} + +// A directory is what both artifacts share without colliding. +func TestADirectoryForTwoArtifactsIsAccepted(t *testing.T) { + err := runGenerate(t, "--output-dir", t.TempDir()) + + if err != nil { + assert.NotContains(t, err.Error(), "names a file") + } +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go index 2e24aea4865..e4e637e145c 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go @@ -4,6 +4,7 @@ package cmd import ( + "errors" "fmt" "strconv" "strings" @@ -153,6 +154,18 @@ func newRunShowCommand() *cobra.Command { gateOnStatus := wait if wait { run, err = ec.pollRun(ctx, evalID, run.ID, cmd.OutOrStdout(), isJSON(cmd)) + if errors.Is(err, errWaitBudgetSpent) { + // The wait ran out; the run is still going server-side. + // `run start` answers this with a reattach line, and a gate + // with a refusal rather than a silent pass. Reattaching is + // what this command is, so it says the same things rather + // than surfacing the sentinel's own text. + if threshold.set { + return messages.GateOutlivedTheWait(run.ID, waitBudget) + } + fmt.Fprint(cmd.OutOrStdout(), messages.WaitBudgetSpent(run.ID, waitBudget)) + return nil + } if err != nil { return err } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go b/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go index d60e67228de..c03a1761ca6 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go @@ -442,6 +442,18 @@ func OutputItemReason(reason string) string { return fmt.Sprintf(" %s\n", reason) } +// OutputFileCannotHoldBothArtifacts reports one --output-dir file for two +// artifacts. +// +// Both jobs would resolve to it and both would write it, concurrently, and the +// configuration would then name it as a dataset and as an evaluator. +func OutputFileCannotHoldBothArtifacts(outputDir string) error { + return fmt.Errorf( + "--output-dir %q names a file, and this generates a dataset and an evaluator; "+ + "name a directory, or add --dataset or --evaluator to generate one of them", + filepath.ToSlash(outputDir)) +} + // UsingLastRun names the run a command chose when it was not given one. // // Written to stderr so it does not land in a redirected listing. diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/artifacts.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/artifacts.go index ef8ac69ed64..58480e94e6a 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/artifacts.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/artifacts.go @@ -89,6 +89,12 @@ func ArtifactPath(baseDir, outputDir, resourceName, ext string) string { return filepath.Join(candidate, resourceName+ext) } +// OutputDirNamesAFile reports an --output-dir that names a file rather than a +// directory, which is a thing only one artifact can be written to. +func OutputDirNamesAFile(outputDir string) bool { + return looksLikeFile(outputDir, "") +} + // looksLikeFile treats a trailing recognized extension as an explicit file path. func looksLikeFile(p, ext string) bool { got := strings.ToLower(filepath.Ext(p)) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/deploy_paths_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/deploy_paths_test.go new file mode 100644 index 00000000000..ea20778c859 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/deploy_paths_test.go @@ -0,0 +1,77 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package project + +import ( + "path/filepath" + "testing" + + "azureaieval/internal/pkg/evalcore" + + "github.com/azure/azure-dev/cli/azd/pkg/azdext" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/types/known/structpb" +) + +// serviceRelativeDir answers relative to the project, because that is what a +// service's $ref and relativePath are written relative to. Resolved against the +// process's working directory instead, `azd up` from any subdirectory reported +// every dataset as not yet generated and offered to bill a generation job to +// rewrite a file already on disk. +func TestSourcePathsResolveAgainstTheProjectAndNotTheWorkingDirectory(t *testing.T) { + svc := &azdext.ServiceConfig{Name: "evals", Host: "azure.ai.eval"} + props, err := structpb.NewStruct(map[string]any{"$ref": "evals/azure.eval.yaml"}) + require.NoError(t, err) + svc.Config = props + + root := filepath.Join(string(filepath.Separator), "work", "proj") + base := filepath.Join(root, serviceRelativeDir(svc)) + + assert.Equal(t, filepath.Join(root, "evals"), base, + "the base a source: resolves against has to be under the project") + assert.Equal(t, + filepath.Join(root, "evals", "datasets", "rows.jsonl"), + ResolveSource(base, "./datasets/rows.jsonl"), + "a relative source is the project's, wherever the caller happened to be standing") +} + +// An absolute source is still taken as written, project root or not. +func TestAnAbsoluteSourceIsNotRerooted(t *testing.T) { + // From TempDir so it carries a volume name on Windows, where a leading + // separator alone is not an absolute path. + absolute := filepath.Join(t.TempDir(), "rows.jsonl") + + assert.Equal(t, absolute, + ResolveSource(filepath.Join(string(filepath.Separator), "work", "proj", "evals"), absolute)) +} + +// max_samples caps the rows the CLI sends on a run. It never reaches the eval +// the service stores, so hashing it recreated the eval for a change the service +// cannot see -- orphaning every run recorded against the old id. +func TestARowCapDoesNotCostAnEvalItsRunHistory(t *testing.T) { + group := Eval{ + Name: "quality", + Dataset: "regression", + MaxSamples: 20, + Evaluators: evalcore.EvaluatorList{{Evaluator: "builtin.relevance"}}, + } + capped := group + capped.MaxSamples = 50 + + before, err := FingerprintGroup(group) + require.NoError(t, err) + after, err := FingerprintGroup(capped) + require.NoError(t, err) + + assert.Equal(t, before, after, "the eval the service holds did not change") + + // And a change the service can see still forks the eval, which is what the + // fingerprint is for. + retargeted := group + retargeted.Dataset = "regression-v2" + forked, err := FingerprintGroup(retargeted) + require.NoError(t, err) + assert.NotEqual(t, before, forked, "a different dataset is a different eval") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go index 66fe3626302..255d9b39ce7 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go @@ -144,7 +144,7 @@ func (p *EvalServiceTargetProvider) Deploy( return nil, err } - baseDir := serviceRelativeDir(serviceConfig) + baseDir := p.evalBaseDir(ctx, serviceConfig) // 1. Datasets the configuration owns. Paths are kept so an eval that names // one can derive its columns without reading the blob back. @@ -208,6 +208,32 @@ func (p *EvalServiceTargetProvider) projectRoot(ctx context.Context) string { return resp.GetProject().GetPath() } +// evalBaseDir is the directory a declaration's `source:` resolves against. +// +// serviceRelativeDir answers relative to the project, because that is what the +// service's `$ref` and relativePath are written relative to. Left there it was +// resolved against this process's working directory instead, and azd neither +// changes it nor reports the project through it: azure.yaml is found by walking +// up from wherever the caller stood, and AZD_CWD carries the --cwd flag and +// nothing else. So `azd up` from any subdirectory of the project reported every +// dataset as not yet generated, and the remedy it offered would have billed a +// generation job to rewrite a file already on disk. +// +// The same join is what agent_instructions.go does with the same helper. +func (p *EvalServiceTargetProvider) evalBaseDir( + ctx context.Context, + serviceConfig *azdext.ServiceConfig, +) string { + relative := serviceRelativeDir(serviceConfig) + root := p.projectRoot(ctx) + if root == "" { + // azd could not name the project. Falling back to the relative path + // keeps the behaviour it had rather than resolving against nothing. + return relative + } + return filepath.Join(root, relative) +} + // describeResult reports whether a version was published or reused, so a // no-op deploy is visibly a no-op. func describeResult(kind, name, version string, changed bool) string { @@ -359,6 +385,12 @@ func FingerprintGroup(group Eval) (string, error) { group.Name = "" group.Description = "" + // max_samples caps the rows this CLI sends on a run. It never reaches the + // eval the service stores, so a run with a different cap is the same eval — + // and hashing it recreated the eval, orphaning every run recorded against + // the old id and leaving two identical definitions under one name. + group.MaxSamples = 0 + data, err := json.Marshal(group) if err != nil { return "", messages.HashingEval(name, err) From 95c190bebb7e43b943294d9bf5b72afa3aeabd6c Mon Sep 17 00:00:00 2001 From: mohessie Date: Wed, 19 Aug 2026 10:28:13 +0300 Subject: [PATCH 299/320] Do not panic when a wait runs out, and quote the steps init prints Five findings, four of them in the two commits before this one. Fixing a thing badly is its own defect, so they are listed the same way. `run show --wait` dereferenced nil. pollRun answers a spent budget with a nil run, and the branch added to handle that assigned over the run it had already read before naming it -- so a run that outlived two hours panicked with a stack trace instead of printing the reattach line the branch exists to print. `run start` keeps poll's result in its own variable, which is the detail the copy missed. The same branch also wrote a human sentence into `-o json`. `evalBaseDir` joined the project root onto a path that was already absolute. azd does not re-root an absolute `$ref` or an absolute `project:`, so a service using one got /C:/shared/evals -- the same failure the join was added to fix, reached from a different input, and the old code had been right for it. `source:` was still in the eval's fingerprint alongside max_samples, and it is in the same category: CreateOpenAIEvalRequest carries no source, buildEvalRequest reads none, and the window and agent name reach the run rather than the eval. So widening a lookback recreated the eval and orphaned its run history. A source change that does alter the eval also changes `dataset`, which is still hashed. And the steps `init` prints quoted --path and nothing beside it. --name is free-form and becomes the dataset name, so `--name "my eval"` printed a step passing `--dataset-name my` and a stray positional `eval`, which `generate` refuses without naming the cause. The target and the generation model could carry a space too. --- .../internal/cmd/evaldir_test.go | 80 +++++++++++++++++++ .../azure.ai.evaluations/internal/cmd/init.go | 13 ++- .../internal/cmd/run_ops.go | 14 +++- .../internal/cmd/run_show_budget_test.go | 50 ++++++++++++ .../internal/project/deploy_paths_test.go | 35 ++++++++ .../internal/project/service_target_eval.go | 23 ++++-- 6 files changed, 201 insertions(+), 14 deletions(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_show_budget_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaldir_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaldir_test.go index c7527a1f73a..14f32ba4d8d 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaldir_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaldir_test.go @@ -333,6 +333,86 @@ func TestNextStepQuotesADirectoryThatNeedsIt(t *testing.T) { } } +// --path was quoted and the values beside it were not. --name is free-form and +// becomes the dataset name, so `--name "my eval"` printed a step passing +// `--dataset-name my` and a stray positional `eval`, which generate refuses +// without naming the cause. +func TestNextStepsQuoteEveryValueTheyCarry(t *testing.T) { + cases := []struct { + name string + plan scaffold + want string + }{ + { + name: "the generated dataset name", + plan: scaffold{ + eval: &project.Eval{Name: "my eval"}, + generateDataset: true, + datasetName: "my eval", + target: "support-agent", + }, + want: `--dataset-name "my eval"`, + }, + { + name: "the generated evaluator name", + plan: scaffold{ + eval: &project.Eval{Name: "an-eval"}, + generateRubric: true, + rubricName: "my rubric", + target: "support-agent", + }, + want: `--evaluator-name "my rubric"`, + }, + { + name: "the target", + plan: scaffold{ + eval: &project.Eval{Name: "an-eval"}, + generateDataset: true, + datasetName: "ds", + target: "Support Agent", + }, + want: `--target "Support Agent"`, + }, + { + name: "the generation model", + plan: scaffold{ + eval: &project.Eval{Name: "an-eval"}, + generateDataset: true, + datasetName: "ds", + judgeModel: "gpt 4o", + }, + want: `--generation-model "gpt 4o"`, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + steps := tc.plan.nextSteps("azd ai eval create") + require.NotEmpty(t, steps) + assert.Contains(t, strings.Join(steps, "\n"), tc.want) + }) + } +} + +// An ordinary name is left alone, or every printed step gains quotes nobody +// needs. +func TestNextStepsLeaveOrdinaryValuesUnquoted(t *testing.T) { + plan := scaffold{ + eval: &project.Eval{Name: "an-eval"}, + generateDataset: true, + datasetName: "support-agent-dataset", + target: "support-agent", + judgeModel: "gpt-4.1-nano", + } + + joined := strings.Join(plan.nextSteps("azd ai eval create"), "\n") + + assert.Contains(t, joined, "--dataset-name support-agent-dataset") + assert.Contains(t, joined, "--target support-agent") + assert.Contains(t, joined, "--generation-model gpt-4.1-nano") + assert.NotContains(t, joined, `"`) +} + // Backslashes must survive: doubling them is right for bash and wrong for the // two shells most likely to be reading a path that looks like this. func TestQuoteForShellLeavesBackslashesAlone(t *testing.T) { diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go index 0182326e691..d0c86577000 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go @@ -511,9 +511,9 @@ func (s scaffold) nextSteps(deployCmd string) []string { // One command produces both, which is the whole point of the composite. steps = append(steps, s.generateCommand("")) case s.generateDataset: - steps = append(steps, s.generateCommand("--dataset --dataset-name "+s.datasetName)) + steps = append(steps, s.generateCommand("--dataset --dataset-name "+quoteForShell(s.datasetName))) case s.generateRubric: - steps = append(steps, s.generateCommand("--evaluator --evaluator-name "+s.rubricName)) + steps = append(steps, s.generateCommand("--evaluator --evaluator-name "+quoteForShell(s.rubricName))) } if len(steps) == 0 { // `azd up` reads azure.yaml, which already $refs the configuration @@ -568,16 +568,21 @@ func quoteForShell(v string) string { } // generateCommand builds a `generate` invocation that runs as printed. +// +// Every interpolated value is quoted, not just the path: --name is free-form +// and becomes the dataset name, and a target or a model deployment can carry a +// space too. Unquoted, `--dataset-name my eval` passed `my` and left `eval` as +// a positional argument that `generate` refuses without naming the cause. func (s scaffold) generateCommand(what string) string { cmd := "azd ai eval generate" if what != "" { cmd += " " + what } if s.target != "" { - cmd += " --target " + s.target + cmd += " --target " + quoteForShell(s.target) } if s.judgeModel != "" { - cmd += " --generation-model " + s.judgeModel + cmd += " --generation-model " + quoteForShell(s.judgeModel) } return s.withPath(cmd) } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go index e4e637e145c..63b65e74b72 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go @@ -153,8 +153,10 @@ func newRunShowCommand() *cobra.Command { // answer. gateOnStatus := wait if wait { - run, err = ec.pollRun(ctx, evalID, run.ID, cmd.OutOrStdout(), isJSON(cmd)) - if errors.Is(err, errWaitBudgetSpent) { + // Into a second variable: pollRun answers the budget with a nil + // run, and the run read above is what still names it. + final, pollErr := ec.pollRun(ctx, evalID, run.ID, cmd.OutOrStdout(), isJSON(cmd)) + if errors.Is(pollErr, errWaitBudgetSpent) { // The wait ran out; the run is still going server-side. // `run start` answers this with a reattach line, and a gate // with a refusal rather than a silent pass. Reattaching is @@ -163,12 +165,16 @@ func newRunShowCommand() *cobra.Command { if threshold.set { return messages.GateOutlivedTheWait(run.ID, waitBudget) } + if isJSON(cmd) { + return emitJSON(cmd.OutOrStdout(), run) + } fmt.Fprint(cmd.OutOrStdout(), messages.WaitBudgetSpent(run.ID, waitBudget)) return nil } - if err != nil { - return err + if pollErr != nil { + return pollErr } + run = final } // The spec puts --fail-on on the commands that wait. Gating a run diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_show_budget_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_show_budget_test.go new file mode 100644 index 00000000000..b2f5871ca4a --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_show_budget_test.go @@ -0,0 +1,50 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "os" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// pollRun answers a spent budget with a nil run, so the branch that handles it +// has to keep the run it already read rather than the one poll returned. The +// first version of this branch assigned over it and dereferenced nil, which +// panics with a stack trace on a run that simply took a long time. +// +// Read from the source because reaching the branch needs a run that outlives +// two hours; the shape is what the test can pin, and the shape is what was +// wrong. gating_budget_test.go pins the same thing for `run start`. +func TestRunShowKeepsTheRunThePollDidNotReturn(t *testing.T) { + body, err := os.ReadFile("run_ops.go") + require.NoError(t, err) + source := string(body) + + require.Contains(t, source, "errWaitBudgetSpent", + "run show has to answer the budget rather than surface the sentinel") + + branch := source[strings.Index(source, "errWaitBudgetSpent"):] + branch = branch[:strings.Index(branch, "gateOnStatus {")] + + assert.NotContains(t, source, "run, err = ec.pollRun(", + "assigning poll's result over the run leaves nil to dereference") + assert.Contains(t, source, "final, pollErr := ec.pollRun(", + "poll's result belongs in its own variable") + assert.Contains(t, branch, "isJSON(cmd)", + "a human sentence in the JSON stream breaks every parser downstream") +} + +// The gate is the one place a pipeline is guaranteed to read, so a budget that +// ran out under --fail-on has to be a refusal rather than a silent pass. +func TestRunShowRefusesAGateThatOutlivedTheWait(t *testing.T) { + body, err := os.ReadFile("run_ops.go") + require.NoError(t, err) + + assert.Contains(t, string(body), "GateOutlivedTheWait", + "exiting 0 here would tell a pipeline the gate passed") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/deploy_paths_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/deploy_paths_test.go index ea20778c859..19b7d5dbcc1 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/deploy_paths_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/deploy_paths_test.go @@ -47,6 +47,28 @@ func TestAnAbsoluteSourceIsNotRerooted(t *testing.T) { ResolveSource(filepath.Join(string(filepath.Separator), "work", "proj", "evals"), absolute)) } +// azd does not re-root an absolute `$ref` or an absolute `project:`, so neither +// may the base a `source:` resolves against: joining one under the project +// produced /C:/shared/evals, which is the same failure the join was added +// to fix, reached from a different input. +func TestAnAbsoluteServiceRefIsNotJoinedUnderTheProject(t *testing.T) { + shared := filepath.Join(t.TempDir(), "shared", "evals") + + svc := &azdext.ServiceConfig{Name: "evals", Host: "azure.ai.eval"} + props, err := structpb.NewStruct(map[string]any{ + "$ref": filepath.ToSlash(filepath.Join(shared, "azure.eval.yaml")), + }) + require.NoError(t, err) + svc.Config = props + + relative := serviceRelativeDir(svc) + require.True(t, filepath.IsAbs(relative), "the fixture has to exercise the absolute branch") + + provider := &EvalServiceTargetProvider{} + assert.Equal(t, relative, provider.evalBaseDir(t.Context(), svc), + "an absolute ref names its own directory, wherever the project is") +} + // max_samples caps the rows the CLI sends on a run. It never reaches the eval // the service stores, so hashing it recreated the eval for a change the service // cannot see -- orphaning every run recorded against the old id. @@ -67,6 +89,19 @@ func TestARowCapDoesNotCostAnEvalItsRunHistory(t *testing.T) { assert.Equal(t, before, after, "the eval the service holds did not change") + // source: is applied per run too. Its window and agent name reach the run's + // data source, never CreateOpenAIEvalRequest. + widened := group + widened.Source = &SourceDecl{Type: "traces", LookbackHours: 48} + narrow := group + narrow.Source = &SourceDecl{Type: "traces", LookbackHours: 24} + + wide, err := FingerprintGroup(widened) + require.NoError(t, err) + tight, err := FingerprintGroup(narrow) + require.NoError(t, err) + assert.Equal(t, wide, tight, "a different lookback is the same eval") + // And a change the service can see still forks the eval, which is what the // fingerprint is for. retargeted := group diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go index 255d9b39ce7..e2aed32df02 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go @@ -225,6 +225,13 @@ func (p *EvalServiceTargetProvider) evalBaseDir( serviceConfig *azdext.ServiceConfig, ) string { relative := serviceRelativeDir(serviceConfig) + // azd does not re-root an absolute `$ref` or an absolute `project:`, so + // neither does this: joining one under the project produced + // /C:/shared/evals, which is the bug this fixes, for a different + // input. + if filepath.IsAbs(relative) { + return relative + } root := p.projectRoot(ctx) if root == "" { // azd could not name the project. Falling back to the relative path @@ -378,18 +385,22 @@ func FingerprintGroup(group Eval) (string, error) { // Only substance is hashed. The id is server-assigned; name and description // are what UpdateEvalParametersBody reaches, so an edit confined to them is // pushed in place and must not cost the eval its id and its run history. - // Everything else — dataset, source, evaluators, target, level — is fixed at - // creation, so a change there is a new eval. + // Dataset, evaluators, target and level are fixed at creation, so a change + // there is a new eval. name := group.Name group.ID = "" group.Name = "" group.Description = "" - // max_samples caps the rows this CLI sends on a run. It never reaches the - // eval the service stores, so a run with a different cap is the same eval — - // and hashing it recreated the eval, orphaning every run recorded against - // the old id and leaving two identical definitions under one name. + // max_samples and source: are applied per run, not at creation -- + // CreateOpenAIEvalRequest carries neither, and buildEvalRequest reads + // neither. Hashing them recreated the eval for a change the stored eval + // cannot express: the declaration pointed at a new id, every run recorded + // before it became reachable only through the old one, and the service was + // left holding two identical definitions under one name. A source change + // that does alter the eval also changes `dataset`, which is hashed. group.MaxSamples = 0 + group.Source = nil data, err := json.Marshal(group) if err != nil { From f03c997fe5a443e1082eb9a5f1f9c9fd81b4fc48 Mon Sep 17 00:00:00 2001 From: mohessie Date: Wed, 19 Aug 2026 11:01:43 +0300 Subject: [PATCH 300/320] Keep two evals two evals, and quote the names the messages suggest Third round on the same code, and the pattern held again: four of the five findings were in the two commits before this one. Zeroing max_samples and source: was right for deciding whether to recreate an eval and wrong for the other thing the digest does. It is also the key a rename looks an eval up by, so two declarations over the same dataset and evaluators that differ only in their window collapsed onto one key: deploying both created the first, adopted it for the second, renamed it, and never created the second at all. Both declarations then resolved to one eval and one run history. That is a worse failure than the orphaned history the zeroing was fixing. The two roles now have two digests -- identity keeps everything, the recreate comparison drops what the run carries. The quoting fix stopped at the steps init prints. Four suggested commands still interpolated a name the caller chose, one of them on the azd up path from the very scenario that commit describes: `init --name "my eval"` then deploy before generating printed `--dataset-name my eval`, which generate refuses without naming the cause. The rule now lives in messages, beside them. The absolute-`$ref` guard had a sibling: the same unguarded join reads the agent metadata, where failing it reads as an agent that was never optimized and generate quietly stops using the instructions on disk. One function does the join now, and it is the one the test exercises. And two tests added last commit could not fail. The wait-budget test's source window ran past the branch into the command's ordinary JSON block, so the fix it guards could be deleted and the assertion still held; the absolute-ref test built a provider with no azd client, which returns early on a different branch before reaching the guard. Both were mutation-checked after fixing, along with the two new digest tests. --- .../azure.ai.evaluations/internal/cmd/init.go | 23 +------ .../internal/cmd/reconciler.go | 20 ++++-- .../internal/cmd/run_show_budget_test.go | 5 +- .../internal/messages/messages.go | 37 ++++++++-- .../internal/project/agent_instructions.go | 2 +- .../internal/project/deploy_paths_test.go | 67 ++++++++++++++++--- .../internal/project/service_target_eval.go | 63 ++++++++++------- 7 files changed, 150 insertions(+), 67 deletions(-) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go index d0c86577000..e025ee1f69a 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go @@ -544,27 +544,10 @@ func (s scaffold) withPath(step string) string { // argument. // // `--path "./team evals"` is the difference between a printed step that runs -// and one that resolves ./team and reports the configuration missing. That is -// the case worth getting right, and double quotes are what cmd, PowerShell, -// bash and zsh all read the same way for a path containing spaces. -// -// A path containing $ or a backtick has no portable answer: double quotes stop -// neither from expanding in bash, zsh or PowerShell, and single quotes -- which -// would -- are literal only on the POSIX shells. Such a path is wrapped anyway, -// because one argument that may expand still beats two that certainly break, -// and this line is printed for a person to read rather than executed here. -// -// Backslashes are left alone: C:\Users\Me\My Evals has to come back out as -// itself, and doubling them would be right for bash and wrong for the two -// shells most likely to be reading a path that looks like that. +// and one that resolves ./team and reports the configuration missing. The rule +// lives in messages, beside the suggested commands that need the same thing. func quoteForShell(v string) string { - if v == "" { - return `""` - } - if !strings.ContainsAny(v, " \t\n\"'`$&|;<>()*?[]#~!") { - return v - } - return `"` + strings.ReplaceAll(v, `"`, `\"`) + `"` + return messages.ShellArg(v) } // generateCommand builds a `generate` invocation that runs as printed. diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go index 521f5210022..240ec60a356 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go @@ -464,15 +464,20 @@ func (r *evalReconciler) EnsureEval( } // Evals are immutable, so a change to the eval's own substance — evaluators, - // dataset, source, target, level — needs a new eval. Name and description are - // excluded from the digest and pushed in place instead. + // dataset, target, level — needs a new eval. Name and description are + // excluded from the digest and pushed in place instead, and so are + // max_samples and source:, which the run carries rather than the eval. recreate := false digest, err := project.FingerprintGroup(group) if err != nil { return "", false, err } + definition, err := project.FingerprintDefinition(group) + if err != nil { + return "", false, err + } key := project.FingerprintKey("eval", group.Name) - if prior := r.ec.getEnvValue(ctx, key); prior != "" && prior != digest { + if prior := r.ec.getEnvValue(ctx, key); prior != "" && prior != definition { recreate = true } @@ -519,10 +524,11 @@ func (r *evalReconciler) EnsureEval( // either of them can reach the service. r.pushMutable(ctx, cached, group, remote) - // Record the digest on reuse as well, otherwise an eval deployed + // Record the definition on reuse as well, otherwise an eval deployed // before fingerprinting existed never establishes a baseline and - // later edits go undetected. - r.ec.remember(ctx, key, digest) + // later edits go undetected. The identity digest is recorded beside + // it, which is what recognizes this declaration after a rename. + r.ec.remember(ctx, key, definition) r.ec.remember(ctx, idKey("eval", group.Name), cached) r.ec.remember(ctx, digestIDKey(digest), cached) r.ec.remember(ctx, envKeyEvalID, cached) @@ -534,7 +540,7 @@ func (r *evalReconciler) EnsureEval( if err != nil { return "", false, err } - r.ec.remember(ctx, key, digest) + r.ec.remember(ctx, key, definition) r.ec.remember(ctx, idKey("eval", group.Name), created.ID) r.ec.remember(ctx, digestIDKey(digest), created.ID) // EVAL_ID stays the last-deployed eval. Nothing reads it to decide which diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_show_budget_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_show_budget_test.go index b2f5871ca4a..e77f7647176 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_show_budget_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_show_budget_test.go @@ -29,7 +29,10 @@ func TestRunShowKeepsTheRunThePollDidNotReturn(t *testing.T) { "run show has to answer the budget rather than surface the sentinel") branch := source[strings.Index(source, "errWaitBudgetSpent"):] - branch = branch[:strings.Index(branch, "gateOnStatus {")] + // Ended at the poll's own error check, so the window holds the budget branch + // and nothing after it. Ending it later swept in the command's ordinary + // `if isJSON(cmd)` block and the assertion below passed with the fix removed. + branch = branch[:strings.Index(branch, "if pollErr != nil")] assert.NotContains(t, source, "run, err = ec.pollRun(", "assigning poll's result over the run leaves nil to dereference") diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go b/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go index c03a1761ca6..02afa310fb0 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go @@ -991,7 +991,7 @@ func NoDatasets() string { // is the one thing only the caller knows, and is named outside the command. func NoDatasetVersions(dataset string) string { return fmt.Sprintf("No versions of dataset %q. Publish one with "+ - "`azd ai eval dataset create %s` and a --from-file path.\n", dataset, dataset) + "`azd ai eval dataset create %s` and a --from-file path.\n", dataset, shellArg(dataset)) } // ResolvingLatestDatasetVersion reports a failure to find what "latest" means. @@ -1014,7 +1014,8 @@ func DatasetNotFound(dataset string) error { func DatasetVersionNotFoundWithHint(dataset, version string) error { return fmt.Errorf( "no dataset %q at version %q in this project; "+ - "`azd ai eval dataset versions list %s` shows the ones there are", dataset, version, dataset) + "`azd ai eval dataset versions list %s` shows the ones there are", + dataset, version, shellArg(dataset)) } // DatasetVersionNotFound reports a dataset version there is nothing to delete at. @@ -1057,7 +1058,7 @@ func DatasetNotGeneratedYet(dataset, path string) error { "its rows %s have not been generated yet. "+ "Run `azd ai eval generate --dataset --dataset-name %s` to write them, "+ "or point the declaration at a .jsonl you already have", - filepath.ToSlash(path), dataset) + filepath.ToSlash(path), shellArg(dataset)) } // DatasetNotLocalNorFound reports a source-less dataset the project rejected. @@ -1213,7 +1214,7 @@ func EvaluatorNotGeneratedYet(evaluator, path string) error { "its definition %s has not been generated yet. "+ "Run `azd ai eval generate --evaluator --evaluator-name %s` to write it, "+ "or drop the evaluator from azure.eval.yaml", - filepath.ToSlash(path), evaluator) + filepath.ToSlash(path), shellArg(evaluator)) } // CheckingEvaluatorExists reports a failure to tell create from update. @@ -2695,3 +2696,31 @@ func CouldNotReadAgentForModel(agent string, err error) string { } return fmt.Sprintf(" warning: could not read agent %q for its deployment: %v\n", agent, err) } + +// shellArg wraps a value a shell would otherwise read as more than one +// argument. +// +// A suggested command is written to be pasted and run. A name the caller chose +// -- `--name "my eval"` becomes the dataset's name -- turns +// `--dataset-name my eval` into one flag and a stray positional argument, which +// generate refuses without naming the cause. +// +// Double quotes are what cmd, PowerShell, bash and zsh all read the same way. +// A value containing $ or a backtick has no portable answer and is wrapped +// anyway: one argument that may expand still beats two that certainly break. +// Backslashes are left alone, so a Windows path comes back out as itself. +func shellArg(v string) string { + if v == "" { + return `""` + } + if !strings.ContainsAny(v, " \t\n\"'`$&|;<>()*?[]#~!") { + return v + } + return `"` + strings.ReplaceAll(v, `"`, `\"`) + `"` +} + +// ShellArg is shellArg for the command builders outside this package, so one +// rule decides how every printed command quotes what it carries. +func ShellArg(v string) string { + return shellArg(v) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/agent_instructions.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/agent_instructions.go index 7beea0cc905..ca592b992d9 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/agent_instructions.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/agent_instructions.go @@ -66,7 +66,7 @@ func AgentInstructionsFromProject( } configDir := filepath.Join( - proj.GetPath(), serviceRelativeDir(svc), agentConfigsDir, agentBaselineDir) + baseDirUnder(proj.GetPath(), svc), agentConfigsDir, agentBaselineDir) data, err := os.ReadFile(filepath.Join(configDir, agentMetadataFile)) //nolint:gosec // under the project if err != nil { diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/deploy_paths_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/deploy_paths_test.go index 19b7d5dbcc1..d2272ebcb32 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/deploy_paths_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/deploy_paths_test.go @@ -64,14 +64,22 @@ func TestAnAbsoluteServiceRefIsNotJoinedUnderTheProject(t *testing.T) { relative := serviceRelativeDir(svc) require.True(t, filepath.IsAbs(relative), "the fixture has to exercise the absolute branch") - provider := &EvalServiceTargetProvider{} - assert.Equal(t, relative, provider.evalBaseDir(t.Context(), svc), + // A project root that is not empty, or evalBaseDir returns early on the + // fallback and the guard under test is never reached. + assert.Equal(t, relative, + baseDirUnder(filepath.Join(string(filepath.Separator), "work", "proj"), svc), "an absolute ref names its own directory, wherever the project is") + + // And a relative ref is still joined, which is what the guard must not undo. + relativeSvc := &azdext.ServiceConfig{Name: "evals", Host: "azure.ai.eval", RelativePath: "evals"} + root := filepath.Join(string(filepath.Separator), "work", "proj") + assert.Equal(t, filepath.Join(root, "evals"), baseDirUnder(root, relativeSvc)) } -// max_samples caps the rows the CLI sends on a run. It never reaches the eval -// the service stores, so hashing it recreated the eval for a change the service -// cannot see -- orphaning every run recorded against the old id. +// max_samples and source: cap and window a run. Neither reaches the eval the +// service stores, so recreating the eval for a change to either points the +// declaration at a new id and leaves every run taken before it reachable only +// through the old one. func TestARowCapDoesNotCostAnEvalItsRunHistory(t *testing.T) { group := Eval{ Name: "quality", @@ -82,9 +90,9 @@ func TestARowCapDoesNotCostAnEvalItsRunHistory(t *testing.T) { capped := group capped.MaxSamples = 50 - before, err := FingerprintGroup(group) + before, err := FingerprintDefinition(group) require.NoError(t, err) - after, err := FingerprintGroup(capped) + after, err := FingerprintDefinition(capped) require.NoError(t, err) assert.Equal(t, before, after, "the eval the service holds did not change") @@ -96,9 +104,9 @@ func TestARowCapDoesNotCostAnEvalItsRunHistory(t *testing.T) { narrow := group narrow.Source = &SourceDecl{Type: "traces", LookbackHours: 24} - wide, err := FingerprintGroup(widened) + wide, err := FingerprintDefinition(widened) require.NoError(t, err) - tight, err := FingerprintGroup(narrow) + tight, err := FingerprintDefinition(narrow) require.NoError(t, err) assert.Equal(t, wide, tight, "a different lookback is the same eval") @@ -106,7 +114,46 @@ func TestARowCapDoesNotCostAnEvalItsRunHistory(t *testing.T) { // fingerprint is for. retargeted := group retargeted.Dataset = "regression-v2" - forked, err := FingerprintGroup(retargeted) + forked, err := FingerprintDefinition(retargeted) require.NoError(t, err) assert.NotEqual(t, before, forked, "a different dataset is a different eval") } + +// The identity digest is the other half, and it must keep them: it is also the +// key a rename looks the eval up by, so two declarations differing only in +// their window would share it -- the second would adopt the first one's id, +// never be created, and rename the first to whichever came last in the file. +func TestTwoEvalsDifferingOnlyInTheirWindowStayTwoEvals(t *testing.T) { + base := Eval{ + Dataset: "", + Evaluators: evalcore.EvaluatorList{{Evaluator: "builtin.relevance"}}, + } + + recent := base + recent.Name = "last-24h" + recent.Source = &SourceDecl{Type: "traces", AgentName: "chat", LookbackHours: 24} + + weekly := base + weekly.Name = "last-7d" + weekly.Source = &SourceDecl{Type: "traces", AgentName: "billing", LookbackHours: 168} + + first, err := FingerprintGroup(recent) + require.NoError(t, err) + second, err := FingerprintGroup(weekly) + require.NoError(t, err) + + assert.NotEqual(t, first, second, + "one key for both hands the second declaration the first one's eval") + + capped := base + capped.Name = "sampled" + capped.MaxSamples = 20 + uncapped := base + uncapped.Name = "full" + + cappedDigest, err := FingerprintGroup(capped) + require.NoError(t, err) + uncappedDigest, err := FingerprintGroup(uncapped) + require.NoError(t, err) + assert.NotEqual(t, cappedDigest, uncappedDigest, "the same holds for a row cap") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go index e2aed32df02..8aa54d06057 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go @@ -224,21 +224,22 @@ func (p *EvalServiceTargetProvider) evalBaseDir( ctx context.Context, serviceConfig *azdext.ServiceConfig, ) string { + return baseDirUnder(p.projectRoot(ctx), serviceConfig) +} + +// baseDirUnder places a service's directory under the project. +// +// azd does not re-root an absolute `$ref` or an absolute `project:`, so neither +// does this: joining one under the project produced /C:/shared/evals, +// which is the bug this fixes, reached from a different input. An empty root is +// azd having failed to name the project, where the relative path is what the +// behaviour used to be and is better than resolving against nothing. +func baseDirUnder(projectRoot string, serviceConfig *azdext.ServiceConfig) string { relative := serviceRelativeDir(serviceConfig) - // azd does not re-root an absolute `$ref` or an absolute `project:`, so - // neither does this: joining one under the project produced - // /C:/shared/evals, which is the bug this fixes, for a different - // input. - if filepath.IsAbs(relative) { - return relative - } - root := p.projectRoot(ctx) - if root == "" { - // azd could not name the project. Falling back to the relative path - // keeps the behaviour it had rather than resolving against nothing. + if filepath.IsAbs(relative) || projectRoot == "" { return relative } - return filepath.Join(root, relative) + return filepath.Join(projectRoot, relative) } // describeResult reports whether a version was published or reused, so a @@ -385,23 +386,13 @@ func FingerprintGroup(group Eval) (string, error) { // Only substance is hashed. The id is server-assigned; name and description // are what UpdateEvalParametersBody reaches, so an edit confined to them is // pushed in place and must not cost the eval its id and its run history. - // Dataset, evaluators, target and level are fixed at creation, so a change - // there is a new eval. + // Everything else — dataset, source, evaluators, target, level — is what + // makes this declaration the one it is. name := group.Name group.ID = "" group.Name = "" group.Description = "" - // max_samples and source: are applied per run, not at creation -- - // CreateOpenAIEvalRequest carries neither, and buildEvalRequest reads - // neither. Hashing them recreated the eval for a change the stored eval - // cannot express: the declaration pointed at a new id, every run recorded - // before it became reachable only through the old one, and the service was - // left holding two identical definitions under one name. A source change - // that does alter the eval also changes `dataset`, which is hashed. - group.MaxSamples = 0 - group.Source = nil - data, err := json.Marshal(group) if err != nil { return "", messages.HashingEval(name, err) @@ -410,6 +401,30 @@ func FingerprintGroup(group Eval) (string, error) { return hex.EncodeToString(sum[:]), nil } +// FingerprintDefinition hashes only what the service stores. +// +// max_samples and source: are applied per run, not at creation -- +// CreateOpenAIEvalRequest carries neither and buildEvalRequest reads neither. +// Recreating the eval when one of them changes points the declaration at a new +// id and leaves every run taken before it reachable only through the old one, +// for an edit the stored eval cannot even express. +// +// Kept separate from FingerprintGroup rather than folded into it, because that +// digest also answers "which eval was this declaration before it was renamed". +// Two evals over the same dataset and evaluators that differ only in their +// window are two evals, and one key for both hands the second the first one's +// id -- so the second is never created, and the first is renamed to whichever +// declaration came last. +// +// The cost is that renaming an eval and changing its window in one edit is read +// as a new eval rather than a rename. That forks a history, which is the +// conservative direction: the other way silently merges two. +func FingerprintDefinition(group Eval) (string, error) { + group.MaxSamples = 0 + group.Source = nil + return FingerprintGroup(group) +} + // FingerprintKey is the azd environment key holding an artifact's fingerprint. // // The readable half is lossy: everything outside [A-Z0-9] becomes an From d5ac4ad4a69a7669e7e985afc37748db66a89da4 Mon Sep 17 00:00:00 2001 From: mohessie Date: Wed, 19 Aug 2026 11:24:47 +0300 Subject: [PATCH 301/320] Do not recreate every eval on upgrade, and let an agent live outside the project Fourth round, same pattern: three of the five findings are in the commit before this one. Splitting the digest changed what is recorded under the recreate key, and every environment out there holds what the shipped build wrote -- the full digest. Read against the definition alone, the first deploy after the upgrade recreated every eval carrying max_samples or source:, for a file nobody had touched, and left the runs taken under the old one unreachable. That is the harm the split was written to prevent, inflicted on the trace-backed evals it was written for. A baseline equal to either digest now means the declaration is what was deployed. Rooting the agent metadata at the service rather than the project fixed the lookup and broke the guard beside it: azd supports a service declared with an absolute `project:`, and its whole checkout is then outside the project directory, so an ordinary relative instruction_file failed containment and generate exited non-zero. The project is no boundary for a service it does not contain; that service's own directory is, and the escape it is there to stop is still stopped. Substance keys are never removed from the environment, so one left by an earlier edit still points at a live eval. Now that a window change mints a new one, a later declaration hashing to the old key would adopt that eval, rename it, and share its runs. An eval this deploy already settled on cannot be adopted again. A dataset pinned to a version reconciliation confirmed was returned to the caller and not recorded, so the deploy reported one version and the run downloaded the one recorded before the pin was added -- scoring different rows than the author asked for and labelling the run with the version it did not use. And the wait-budget test sliced on a string it did not require first, so renaming a variable would have panicked the whole package's test binary rather than failing one assertion. --- .../internal/cmd/digest_migration_test.go | 79 +++++++++++++++++++ .../cmd/pinned_version_recorded_test.go | 76 ++++++++++++++++++ .../internal/cmd/reconciler.go | 37 ++++++++- .../internal/cmd/run_show_budget_test.go | 2 + .../internal/project/agent_instructions.go | 16 +++- .../project/agent_outside_project_test.go | 62 +++++++++++++++ 6 files changed, 268 insertions(+), 4 deletions(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/digest_migration_test.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/pinned_version_recorded_test.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/project/agent_outside_project_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/digest_migration_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/digest_migration_test.go new file mode 100644 index 00000000000..f74cb0e32e5 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/digest_migration_test.go @@ -0,0 +1,79 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "testing" + + "azureaieval/internal/pkg/evalcore" + "azureaieval/internal/project" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// evalWithWindow is a trace-backed declaration, which is the shape where the +// two digests differ. +func evalWithWindow(name string, lookback int) project.Eval { + return project.Eval{ + Name: name, + Source: &project.SourceDecl{Type: "traces", AgentName: "support-agent", LookbackHours: lookback}, + Evaluators: evalcore.EvaluatorList{{Evaluator: "builtin.relevance"}}, + } +} + +// Environments written before the digest was split hold the full digest under +// the recreate key. Comparing only against the definition made the first deploy +// after an upgrade recreate every eval carrying max_samples or source:, for a +// file nobody had touched, and left the runs taken under the old one +// unreachable. +func TestAnEnvironmentFromBeforeTheSplitIsNotReadAsAChange(t *testing.T) { + group := evalWithWindow("nightly", 24) + + shipped, err := project.FingerprintGroup(group) + require.NoError(t, err) + definition, err := project.FingerprintDefinition(group) + require.NoError(t, err) + + require.NotEqual(t, shipped, definition, + "the fixture has to be a declaration where the two digests differ") + + // What EnsureEval asks of the recorded baseline. + unchanged := func(prior string) bool { + return !(prior != "" && prior != definition && prior != shipped) + } + + assert.True(t, unchanged(shipped), "a baseline written by an earlier build is not a change") + assert.True(t, unchanged(definition), "nor is one written by this build") + assert.True(t, unchanged(""), "nor is no baseline at all") + + // And a real edit is still a change. + edited := group + edited.Evaluators = evalcore.EvaluatorList{{Evaluator: "builtin.coherence"}} + editedDefinition, err := project.FingerprintDefinition(edited) + require.NoError(t, err) + assert.False(t, unchanged(editedDefinition), "a different evaluator has to recreate the eval") +} + +// Substance keys are never removed, so one left by an earlier edit still points +// at a live eval. A later declaration that happens to hash to it would adopt +// that eval, rename it, and end up sharing its runs -- so an eval this deploy +// already settled on cannot be adopted again. +func TestAnEvalAlreadySettledOnIsNotAdoptedTwice(t *testing.T) { + r := &evalReconciler{} + + r.claim("eval_1") + + assert.True(t, r.claimed["eval_1"]) + assert.False(t, r.claimed["eval_2"], "only what this deploy settled on is claimed") +} + +// The map is built on first use, because the reconciler is also constructed +// literally in several places and writing to a nil map panics. +func TestClaimingWorksOnAReconcilerBuiltDirectly(t *testing.T) { + r := &evalReconciler{} + + assert.NotPanics(t, func() { r.claim("eval_1") }) + assert.True(t, r.claimed["eval_1"]) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/pinned_version_recorded_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/pinned_version_recorded_test.go new file mode 100644 index 00000000000..92f86cbe2b5 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/pinned_version_recorded_test.go @@ -0,0 +1,76 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + + "azureaieval/internal/pkg/dataset_api" + "azureaieval/internal/project" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// A run reads the version reconciliation settled on. Returning a pin without +// recording it left the deploy reporting one version while the run downloaded +// the one recorded before the pin was written -- scoring different rows than +// the author asked for, and labelling the run with the version it did not use. +func TestAPinnedVersionIsRecordedForTheRunToRead(t *testing.T) { + dir := t.TempDir() + localPath := filepath.Join(dir, "golden.jsonl") + require.NoError(t, os.WriteFile(localPath, []byte("{\"query\":\"hi\"}\n"), 0o600)) + + digest, err := project.Fingerprint(localPath) + require.NoError(t, err) + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if !strings.Contains(r.URL.Path, "/versions/") { + w.WriteHeader(http.StatusNotFound) + return + } + w.Header().Set("Content-Type", "application/json") + // assert, not require: this runs on the server's goroutine. + assert.NoError(t, json.NewEncoder(w).Encode(map[string]any{ + "name": "golden", "version": "1", + })) + })) + t.Cleanup(srv.Close) + + // The last deploy published version 2 from this same file. The author then + // pinned version 1 without touching the file, so the digest still matches. + env := &testEnvServer{values: map[string]string{ + project.FingerprintKey("dataset", "golden"): digest, + versionKey("dataset", "golden"): "2", + }} + + pipeline := runtime.NewPipeline("test", "v1", runtime.PipelineOptions{}, + &policy.ClientOptions{Retry: policy.RetryOptions{MaxRetries: -1}}) + r := &evalReconciler{ec: &evalContext{ + azdClient: newTestAzdClient(t, env), + envName: "test", + datasetClient: dataset_api.NewDatasetClientFromPipeline(srv.URL, pipeline), + }} + + version, changed, err := r.EnsureDataset( + context.Background(), + project.DatasetDecl{Name: "golden", Version: "1"}, + localPath, + ) + + require.NoError(t, err) + assert.Equal(t, "1", version) + assert.False(t, changed) + assert.Equal(t, "1", env.values[versionKey("dataset", "golden")], + "the run reads this key, so it has to hold what the deploy reported") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go index 240ec60a356..a923a6adf87 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go @@ -27,6 +27,13 @@ import ( // deploy half of the provider; the provider owns ordering, this owns the calls. type evalReconciler struct { ec *evalContext + + // claimed holds the evals this deploy has already settled, so a second + // declaration cannot adopt one. Substance keys are never removed from the + // environment, so one left behind by an earlier edit still points at a live + // eval -- and adopting it renames that eval and leaves the declaration that + // asked for it sharing the other one's runs. + claimed map[string]bool } var _ project.Reconciler = (*evalReconciler)(nil) @@ -39,6 +46,15 @@ func newEvalReconciler(ctx context.Context) (project.Reconciler, error) { return &evalReconciler{ec: ec}, nil } +// claim records an eval this deploy has settled on. Built lazily, because the +// reconciler is also constructed literally in a few places. +func (r *evalReconciler) claim(id string) { + if r.claimed == nil { + r.claimed = map[string]bool{} + } + r.claimed[id] = true +} + // EnsureDataset registers a new version only when the local content changed. // // The dataset API exposes no content hash, so comparing against the service @@ -113,6 +129,10 @@ func (r *evalReconciler) EnsureDataset( ); err != nil && dataset_api.IsNotFound(err) { return "", false, messages.DatasetVersionNotFoundWithHint(decl.Name, decl.Version) } + // Recorded here too. Returning the pin without writing it left + // the deploy reporting one version and the run downloading the + // one recorded before the pin was added. + r.ec.remember(ctx, versionKey("dataset", decl.Name), decl.Version) return decl.Version, false, nil } if err := r.checkDatasetDrift(ctx, decl.Name, version); err != nil { @@ -460,6 +480,7 @@ func (r *evalReconciler) EnsureEval( datasetPath string, ) (string, bool, error) { if group.ID != "" { + r.claim(group.ID) return group.ID, false, nil } @@ -477,7 +498,13 @@ func (r *evalReconciler) EnsureEval( return "", false, err } key := project.FingerprintKey("eval", group.Name) - if prior := r.ec.getEnvValue(ctx, key); prior != "" && prior != definition { + // Compared against both. The recorded baseline is the definition from this + // build on, and the full digest from every build before the two were split: + // reading only the definition made the first deploy after an upgrade recreate + // every eval carrying max_samples or source:, for a file nobody had touched, + // and orphaned the runs taken under it. Equality with either says the + // declaration is what was deployed. + if prior := r.ec.getEnvValue(ctx, key); prior != "" && prior != definition && prior != digest { recreate = true } @@ -532,6 +559,7 @@ func (r *evalReconciler) EnsureEval( r.ec.remember(ctx, idKey("eval", group.Name), cached) r.ec.remember(ctx, digestIDKey(digest), cached) r.ec.remember(ctx, envKeyEvalID, cached) + r.claim(cached) return cached, false, nil } } @@ -548,6 +576,7 @@ func (r *evalReconciler) EnsureEval( // which declaration it belongs to; it is here for anything outside this // extension that wants the id of what was just deployed. r.ec.remember(ctx, envKeyEvalID, created.ID) + r.claim(created.ID) return created.ID, true, nil } @@ -565,6 +594,12 @@ func (r *evalReconciler) adoptRenamed( if id == "" { return "", nil } + if r.claimed[id] { + // Another declaration in this same file already settled on it. Adopting + // it here would rename that eval and leave both declarations sharing + // one id and one run history, which is worse than creating a second. + return "", nil + } remote, err := r.ec.evalClient.GetOpenAIEval(ctx, id) if err != nil { if eval_api.IsNotFound(err) { diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_show_budget_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_show_budget_test.go index e77f7647176..ccafac09df7 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_show_budget_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_show_budget_test.go @@ -27,6 +27,8 @@ func TestRunShowKeepsTheRunThePollDidNotReturn(t *testing.T) { require.Contains(t, source, "errWaitBudgetSpent", "run show has to answer the budget rather than surface the sentinel") + require.Contains(t, source, "if pollErr != nil", + "the window below ends here; renaming it must fail rather than panic") branch := source[strings.Index(source, "errWaitBudgetSpent"):] // Ended at the poll's own error check, so the window holds the budget branch diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/agent_instructions.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/agent_instructions.go index ca592b992d9..fa10d57be76 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/agent_instructions.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/agent_instructions.go @@ -65,8 +65,8 @@ func AgentInstructionsFromProject( return "", "", err } - configDir := filepath.Join( - baseDirUnder(proj.GetPath(), svc), agentConfigsDir, agentBaselineDir) + serviceRoot := baseDirUnder(proj.GetPath(), svc) + configDir := filepath.Join(serviceRoot, agentConfigsDir, agentBaselineDir) data, err := os.ReadFile(filepath.Join(configDir, agentMetadataFile)) //nolint:gosec // under the project if err != nil { @@ -90,7 +90,17 @@ func AgentInstructionsFromProject( // The pointer comes out of the checkout, so it carries the checkout's // trust. Left alone, an absolute path or one climbing out with `..` reads a // file the project does not contain and sends it on as agent instructions. - if !withinDir(proj.GetPath(), instructionPath) { + // + // Contained to the project, so any layout inside it a writer chooses still + // resolves -- unless the service itself is declared outside the project, + // which azd supports through an absolute `project:`. The project is then no + // boundary for it at all, and its own directory is the one the metadata was + // read from. + root := proj.GetPath() + if root == "" || !liesWithin(root, serviceRoot) { + root = serviceRoot + } + if !withinDir(root, instructionPath) { return "", "", messages.InstructionFileOutsideProject( filepath.Join(configDir, agentMetadataFile), meta.InstructionFile) } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/agent_outside_project_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/agent_outside_project_test.go new file mode 100644 index 00000000000..7ad891f53b6 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/agent_outside_project_test.go @@ -0,0 +1,62 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package project + +import ( + "os" + "path/filepath" + "testing" + + "github.com/azure/azure-dev/cli/azd/pkg/azdext" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// azd supports a service declared with an absolute `project:`, and the whole +// checkout for that service then sits outside the project directory. Containing +// its instruction pointer to the project refused an ordinary relative pointer +// and failed generate outright, where the service's own directory is the +// boundary that means anything for it. +func TestAnAgentDeclaredOutsideTheProjectStillReadsItsInstructions(t *testing.T) { + root := t.TempDir() + project := filepath.Join(root, "proj") + require.NoError(t, os.MkdirAll(project, 0o750)) + + elsewhere := filepath.Join(root, "shared", "agent") + require.NoError(t, os.MkdirAll(elsewhere, 0o750)) + writeOptimizeConfig(t, elsewhere, "instruction_file: instructions.md\n", "answer briefly") + + svc := &azdext.ServiceConfig{Name: "agent", Host: AgentHost, RelativePath: elsewhere} + proj := &azdext.ProjectConfig{Path: project, Services: map[string]*azdext.ServiceConfig{"agent": svc}} + + instruction, path, err := AgentInstructionsFromProject(proj, "agent") + + require.NoError(t, err) + assert.Equal(t, "answer briefly", instruction) + assert.Contains(t, path, "instructions.md") +} + +// And the boundary still holds there: a pointer climbing out of the service's +// own directory is refused just as one climbing out of the project is. +func TestAnAgentOutsideTheProjectStillCannotReachFurtherOut(t *testing.T) { + root := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(root, "secret.md"), + []byte("a file the checkout does not contain"), 0o600)) + + project := filepath.Join(root, "proj") + require.NoError(t, os.MkdirAll(project, 0o750)) + + elsewhere := filepath.Join(root, "shared", "agent") + require.NoError(t, os.MkdirAll(elsewhere, 0o750)) + writeOptimizeConfig(t, elsewhere, "instruction_file: ../../../secret.md\n", "") + + svc := &azdext.ServiceConfig{Name: "agent", Host: AgentHost, RelativePath: elsewhere} + proj := &azdext.ProjectConfig{Path: project, Services: map[string]*azdext.ServiceConfig{"agent": svc}} + + _, _, err := AgentInstructionsFromProject(proj, "agent") + + require.Error(t, err) + assert.Contains(t, err.Error(), "outside the project") + assert.NotContains(t, err.Error(), "a file the checkout does not contain") +} From 25b58a3f3a82d69f36f3994fa74b200c0f25c7d7 Mon Sep 17 00:00:00 2001 From: mohessie Date: Wed, 19 Aug 2026 11:49:18 +0300 Subject: [PATCH 302/320] Reserve the evals a file declares, and read a pin from the declaration Fifth round, same pattern: both product findings are in the commit before this one, and two of the tests it added could not fail. Claiming an eval only as each declaration finished made adoption depend on the order the file lists them in. A declaration placed above the one that owns an eval still adopted it, renamed it, and ended up sharing its runs -- the guard worked from below and not from above. Every eval the file declares is reserved before any of them is reconciled, so the outcome no longer depends on ordering. A name the environment holds no id for reserves nothing, which is what leaves a genuine rename free to adopt. Recording a pinned dataset version was the wrong way to get it to the run. That key means the version this file's content published, and it is what the drift check compares against, so overwriting it with a pin made removing the pin later read as somebody having published behind the configuration's back -- a failed deploy, blaming the wrong party, recommending the pin be put back. The run reads the pin from the declaration instead, which is also right where there is no azd environment to have recorded anything. And the tests: one rebuilt the recreate condition locally instead of calling EnsureEval, the other asserted on the claim map instead of calling adoptRenamed, and a third's traversal was one level short of the file it was supposed to fail to read. All three now drive the code they are about, and all five guards were mutation-checked -- including that a rename still adopts with the new reservation in place. --- .../internal/cmd/declared_pin_test.go | 121 ++++++++++++++ .../internal/cmd/digest_migration_test.go | 154 ++++++++++++++---- .../internal/cmd/eval_group.go | 7 + .../cmd/pinned_version_recorded_test.go | 76 --------- .../internal/cmd/reconciler.go | 24 ++- .../azure.ai.evaluations/internal/cmd/run.go | 33 +++- .../project/agent_outside_project_test.go | 7 +- .../internal/project/service_target_eval.go | 10 ++ 8 files changed, 321 insertions(+), 111 deletions(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/declared_pin_test.go delete mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/pinned_version_recorded_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/declared_pin_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/declared_pin_test.go new file mode 100644 index 00000000000..7ee7109b285 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/declared_pin_test.go @@ -0,0 +1,121 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + + "azureaieval/internal/pkg/dataset_api" + "azureaieval/internal/project" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// writeConfigWithPin writes a configuration whose dataset carries the given +// `version:`, or none when it is empty. +func writeConfigWithPin(t *testing.T, pin string) string { + t.Helper() + + version := "" + if pin != "" { + version = "\n version: \"" + pin + "\"" + } + body := `datasets: + - name: golden` + version + ` +evals: + - name: quality + dataset: golden + evaluators: + - evaluator: builtin.relevance +` + dir := t.TempDir() + path := filepath.Join(dir, "azure.eval.yaml") + require.NoError(t, os.WriteFile(path, []byte(body), 0o600)) + return path +} + +// A pin is the author saying which rows to score, and the run reads it from the +// declaration. Reading it out of the environment instead meant the deploy had +// to overwrite the version the dataset's content published -- which is the +// drift baseline, so removing the pin later failed the next deploy with a +// report that something had been published behind the configuration's back. +func TestARunReadsThePinnedVersionFromTheDeclaration(t *testing.T) { + group := &project.Eval{Name: "quality", Dataset: "golden"} + + assert.Equal(t, "1", declaredDatasetVersion(writeConfigWithPin(t, "1"), group), + "the declaration is what says which version to score") + assert.Empty(t, declaredDatasetVersion(writeConfigWithPin(t, ""), group), + "no pin leaves the recorded version to answer") +} + +// An eval reached by id has no configuration to ask, and a declaration for a +// different dataset says nothing about this one. +func TestNoPinIsReadWhenThereIsNothingToReadItFrom(t *testing.T) { + group := &project.Eval{Name: "quality", Dataset: "golden"} + + assert.Empty(t, declaredDatasetVersion("", group)) + assert.Empty(t, declaredDatasetVersion(writeConfigWithPin(t, "1"), nil)) + assert.Empty(t, declaredDatasetVersion( + writeConfigWithPin(t, "1"), &project.Eval{Name: "quality", Dataset: "other"})) +} + +// The deploy leaves the recorded version alone, so it keeps meaning "the +// version this file's content published" and the drift check keeps working. +func TestPinningDoesNotOverwriteTheVersionTheContentPublished(t *testing.T) { + body, err := os.ReadFile("reconciler.go") + require.NoError(t, err) + + assert.NotContains(t, string(body), + `r.ec.remember(ctx, versionKey("dataset", decl.Name), decl.Version)`, + "writing the pin here made removing it later read as drift") +} + +// And the pin is what the run downloads. Reading the declaration and then not +// using it would leave the run on the recorded version, which is the failure +// this whole path exists to avoid. +func TestTheDownloadedRowsAreThePinnedVersion(t *testing.T) { + var asked []string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + asked = append(asked, r.URL.Path) + w.Header().Set("Content-Type", "application/json") + // assert, not require: this runs on the server's goroutine. + assert.NoError(t, json.NewEncoder(w).Encode(map[string]any{ + "blobReferenceForConsumption": map[string]any{ + "credential": map[string]any{"sasUri": "https://example.invalid/blob"}, + }, + })) + })) + t.Cleanup(srv.Close) + + env := &testEnvServer{values: map[string]string{ + versionKey("dataset", "golden"): "2", + }} + pipeline := runtime.NewPipeline("test", "v1", runtime.PipelineOptions{}, + &policy.ClientOptions{Retry: policy.RetryOptions{MaxRetries: -1}}) + ec := &evalContext{ + azdClient: newTestAzdClient(t, env), + envName: "test", + datasetClient: dataset_api.NewDatasetClientFromPipeline(srv.URL, pipeline), + } + + // The download itself cannot succeed against this fake; what is under test + // is the version it asked the service for. + _, _ = ec.readRegisteredDataset(context.Background(), "golden", "1", 0) + + require.NotEmpty(t, asked) + assert.Contains(t, strings.Join(asked, " "), "/versions/1", + "the pin the declaration carries is the version to score") + assert.NotContains(t, strings.Join(asked, " "), "/versions/2", + "the recorded version is what the pin overrides") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/digest_migration_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/digest_migration_test.go index f74cb0e32e5..b421632f798 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/digest_migration_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/digest_migration_test.go @@ -4,18 +4,26 @@ package cmd import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "strings" "testing" + "azureaieval/internal/pkg/eval_api" "azureaieval/internal/pkg/evalcore" "azureaieval/internal/project" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) -// evalWithWindow is a trace-backed declaration, which is the shape where the -// two digests differ. -func evalWithWindow(name string, lookback int) project.Eval { +// windowedEval is a trace-backed declaration, which is the shape where the +// identity and definition digests differ. +func windowedEval(name string, lookback int) project.Eval { return project.Eval{ Name: name, Source: &project.SourceDecl{Type: "traces", AgentName: "support-agent", LookbackHours: lookback}, @@ -23,50 +31,140 @@ func evalWithWindow(name string, lookback int) project.Eval { } } +// evalServiceReconciler answers every eval read with the given id and records +// what was sent, so EnsureEval can be driven without a service. +func evalServiceReconciler(t *testing.T, env *testEnvServer, existingID string) (*evalReconciler, *[]string) { + t.Helper() + + var created []string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + // A create posts to the collection; an update posts to one eval, which + // is how the mutable half of a declaration is pushed. + if r.Method == http.MethodPost && strings.HasSuffix(r.URL.Path, "/evals") { + created = append(created, r.URL.Path) + // assert, not require: this runs on the server's goroutine. + assert.NoError(t, json.NewEncoder(w).Encode(map[string]any{"id": "eval_new"})) + return + } + assert.NoError(t, json.NewEncoder(w).Encode(map[string]any{"id": existingID, "name": "whatever"})) + })) + t.Cleanup(srv.Close) + + pipeline := runtime.NewPipeline("test", "v1", runtime.PipelineOptions{}, + &policy.ClientOptions{Retry: policy.RetryOptions{MaxRetries: -1}}) + return &evalReconciler{ec: &evalContext{ + azdClient: newTestAzdClient(t, env), + envName: "test", + evalClient: eval_api.NewEvalClientFromPipeline(srv.URL, pipeline), + }}, &created +} + // Environments written before the digest was split hold the full digest under -// the recreate key. Comparing only against the definition made the first deploy -// after an upgrade recreate every eval carrying max_samples or source:, for a -// file nobody had touched, and left the runs taken under the old one -// unreachable. +// the recreate key. Reading only the definition made the first deploy after an +// upgrade recreate every eval carrying max_samples or source:, for a file +// nobody had touched, and left the runs taken under it unreachable. func TestAnEnvironmentFromBeforeTheSplitIsNotReadAsAChange(t *testing.T) { - group := evalWithWindow("nightly", 24) + group := windowedEval("nightly", 24) shipped, err := project.FingerprintGroup(group) require.NoError(t, err) definition, err := project.FingerprintDefinition(group) require.NoError(t, err) - require.NotEqual(t, shipped, definition, "the fixture has to be a declaration where the two digests differ") - // What EnsureEval asks of the recorded baseline. - unchanged := func(prior string) bool { - return !(prior != "" && prior != definition && prior != shipped) - } + env := &testEnvServer{values: map[string]string{ + // What a build from before the split recorded, plus the id it created. + project.FingerprintKey("eval", "nightly"): shipped, + idKey("eval", "nightly"): "eval_1", + }} + r, created := evalServiceReconciler(t, env, "eval_1") + + id, wasCreated, err := r.EnsureEval(context.Background(), group, "") + + require.NoError(t, err) + assert.Equal(t, "eval_1", id, "the eval already deployed is the one to keep") + assert.False(t, wasCreated, "nothing in the file changed") + assert.Empty(t, *created, "an upgrade must not recreate an eval nobody edited") + assert.Equal(t, definition, env.values[project.FingerprintKey("eval", "nightly")], + "and the baseline moves to the definition, so the next deploy compares like with like") +} - assert.True(t, unchanged(shipped), "a baseline written by an earlier build is not a change") - assert.True(t, unchanged(definition), "nor is one written by this build") - assert.True(t, unchanged(""), "nor is no baseline at all") +// A real edit still recreates, or the escape hatch above would have disabled +// change detection. +func TestAnEditedEvalIsStillRecreated(t *testing.T) { + group := windowedEval("nightly", 24) + shipped, err := project.FingerprintGroup(group) + require.NoError(t, err) - // And a real edit is still a change. edited := group edited.Evaluators = evalcore.EvaluatorList{{Evaluator: "builtin.coherence"}} - editedDefinition, err := project.FingerprintDefinition(edited) + + env := &testEnvServer{values: map[string]string{ + project.FingerprintKey("eval", "nightly"): shipped, + idKey("eval", "nightly"): "eval_1", + }} + r, created := evalServiceReconciler(t, env, "eval_1") + + id, wasCreated, err := r.EnsureEval(context.Background(), edited, "") + require.NoError(t, err) - assert.False(t, unchanged(editedDefinition), "a different evaluator has to recreate the eval") + assert.Equal(t, "eval_new", id) + assert.True(t, wasCreated) + assert.Len(t, *created, 1, "a different evaluator is a different eval") } -// Substance keys are never removed, so one left by an earlier edit still points -// at a live eval. A later declaration that happens to hash to it would adopt -// that eval, rename it, and end up sharing its runs -- so an eval this deploy -// already settled on cannot be adopted again. -func TestAnEvalAlreadySettledOnIsNotAdoptedTwice(t *testing.T) { - r := &evalReconciler{} +// Substance keys are never removed, so one left by an earlier window edit still +// points at a live eval. A second declaration hashing to it must not adopt it, +// whichever order the file lists the two in -- claiming only as each +// declaration finished made the outcome depend on that order. +func TestADeclarationDoesNotAdoptAnEvalAnotherOneOwns(t *testing.T) { + owner := windowedEval("nightly", 24) + ownerDigest, err := project.FingerprintGroup(owner) + require.NoError(t, err) - r.claim("eval_1") + // The newcomer is listed first and hashes to the same substance. + newcomer := windowedEval("weekly", 24) - assert.True(t, r.claimed["eval_1"]) - assert.False(t, r.claimed["eval_2"], "only what this deploy settled on is claimed") + env := &testEnvServer{values: map[string]string{ + digestIDKey(ownerDigest): "eval_1", + idKey("eval", "nightly"): "eval_1", + }} + r, created := evalServiceReconciler(t, env, "eval_1") + + r.ReserveDeclared(context.Background(), []string{"weekly", "nightly"}) + id, wasCreated, err := r.EnsureEval(context.Background(), newcomer, "") + + require.NoError(t, err) + assert.True(t, wasCreated, "the second declaration is a second eval") + assert.NotEqual(t, "eval_1", id, "adopting it would rename the eval nightly owns") + assert.Len(t, *created, 1) +} + +// And a genuine rename still adopts: the old name is gone from the file, so +// nothing reserves the eval it used to be called. +func TestARenamedEvalIsStillAdopted(t *testing.T) { + group := windowedEval("nightly", 24) + digest, err := project.FingerprintGroup(group) + require.NoError(t, err) + + renamed := group + renamed.Name = "evening" + + env := &testEnvServer{values: map[string]string{ + digestIDKey(digest): "eval_1", + idKey("eval", "nightly"): "eval_1", + }} + r, created := evalServiceReconciler(t, env, "eval_1") + + r.ReserveDeclared(context.Background(), []string{"evening"}) + id, wasCreated, err := r.EnsureEval(context.Background(), renamed, "") + + require.NoError(t, err) + assert.Equal(t, "eval_1", id, "a rename keeps the id and the runs under it") + assert.False(t, wasCreated) + assert.Empty(t, *created) } // The map is built on first use, because the reconciler is also constructed diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/eval_group.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/eval_group.go index e6c7b5ef565..8f52d6923a5 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/eval_group.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/eval_group.go @@ -78,6 +78,13 @@ func newEvalCreateCommand() *cobra.Command { } reconciler := &evalReconciler{ec: ec} + // Every eval the file declares, not only the one being created: an + // eval another declaration already owns must not be adopted here. + declared := make([]string, 0, len(cfg.Evals)) + for i := range cfg.Evals { + declared = append(declared, cfg.Evals[i].Name) + } + reconciler.ReserveDeclared(ctx, declared) out := cmd.OutOrStdout() // Before anything is pushed. Publishing is not free -- a dataset diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/pinned_version_recorded_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/pinned_version_recorded_test.go deleted file mode 100644 index 92f86cbe2b5..00000000000 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/pinned_version_recorded_test.go +++ /dev/null @@ -1,76 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package cmd - -import ( - "context" - "encoding/json" - "net/http" - "net/http/httptest" - "os" - "path/filepath" - "strings" - "testing" - - "azureaieval/internal/pkg/dataset_api" - "azureaieval/internal/project" - - "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" - "github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -// A run reads the version reconciliation settled on. Returning a pin without -// recording it left the deploy reporting one version while the run downloaded -// the one recorded before the pin was written -- scoring different rows than -// the author asked for, and labelling the run with the version it did not use. -func TestAPinnedVersionIsRecordedForTheRunToRead(t *testing.T) { - dir := t.TempDir() - localPath := filepath.Join(dir, "golden.jsonl") - require.NoError(t, os.WriteFile(localPath, []byte("{\"query\":\"hi\"}\n"), 0o600)) - - digest, err := project.Fingerprint(localPath) - require.NoError(t, err) - - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if !strings.Contains(r.URL.Path, "/versions/") { - w.WriteHeader(http.StatusNotFound) - return - } - w.Header().Set("Content-Type", "application/json") - // assert, not require: this runs on the server's goroutine. - assert.NoError(t, json.NewEncoder(w).Encode(map[string]any{ - "name": "golden", "version": "1", - })) - })) - t.Cleanup(srv.Close) - - // The last deploy published version 2 from this same file. The author then - // pinned version 1 without touching the file, so the digest still matches. - env := &testEnvServer{values: map[string]string{ - project.FingerprintKey("dataset", "golden"): digest, - versionKey("dataset", "golden"): "2", - }} - - pipeline := runtime.NewPipeline("test", "v1", runtime.PipelineOptions{}, - &policy.ClientOptions{Retry: policy.RetryOptions{MaxRetries: -1}}) - r := &evalReconciler{ec: &evalContext{ - azdClient: newTestAzdClient(t, env), - envName: "test", - datasetClient: dataset_api.NewDatasetClientFromPipeline(srv.URL, pipeline), - }} - - version, changed, err := r.EnsureDataset( - context.Background(), - project.DatasetDecl{Name: "golden", Version: "1"}, - localPath, - ) - - require.NoError(t, err) - assert.Equal(t, "1", version) - assert.False(t, changed) - assert.Equal(t, "1", env.values[versionKey("dataset", "golden")], - "the run reads this key, so it has to hold what the deploy reported") -} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go index a923a6adf87..5dbd39b27f9 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go @@ -55,6 +55,21 @@ func (r *evalReconciler) claim(id string) { r.claimed[id] = true } +// ReserveDeclared marks the evals these names already resolve to as spoken for. +// +// Claiming only as each declaration finishes makes adoption depend on file +// order: a declaration listed above the one that owns an eval would adopt it, +// rename it, and end up sharing its runs. Reserving up front is the same guard +// without the ordering. A name the environment holds no id for reserves +// nothing, which is what leaves a genuine rename free to adopt. +func (r *evalReconciler) ReserveDeclared(ctx context.Context, names []string) { + for _, name := range names { + if id := r.ec.getEnvValue(ctx, idKey("eval", name)); id != "" { + r.claim(id) + } + } +} + // EnsureDataset registers a new version only when the local content changed. // // The dataset API exposes no content hash, so comparing against the service @@ -129,10 +144,11 @@ func (r *evalReconciler) EnsureDataset( ); err != nil && dataset_api.IsNotFound(err) { return "", false, messages.DatasetVersionNotFoundWithHint(decl.Name, decl.Version) } - // Recorded here too. Returning the pin without writing it left - // the deploy reporting one version and the run downloading the - // one recorded before the pin was added. - r.ec.remember(ctx, versionKey("dataset", decl.Name), decl.Version) + // Deliberately not recorded. The key means "the version this file's + // content published", which is what the drift check compares + // against: writing the pin here made removing it later read as + // somebody having published behind the configuration's back, and + // failed the deploy. The run reads the pin from the declaration. return decl.Version, false, nil } if err := r.checkDatasetDrift(ctx, decl.Name, version); err != nil { diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go index 931f4e876d8..6f6888801de 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go @@ -561,7 +561,8 @@ func (ec *evalContext) buildRunDataSource( // name is rejected with "invalid data source file ids". localPath := localDatasetPath(configPath, group) if localPath == "" { - items, err := ec.readRegisteredDataset(ctx, group.Dataset, maxSamples) + items, err := ec.readRegisteredDataset( + ctx, group.Dataset, declaredDatasetVersion(configPath, group), maxSamples) if err != nil { return nil, err } @@ -638,9 +639,15 @@ func responsesDataSource(group *project.Eval) (*eval_api.EvalRunDataSource, erro func (ec *evalContext) readRegisteredDataset( ctx context.Context, name string, + pinned string, maxSamples int, ) ([]map[string]any, error) { - version := ec.getEnvValue(ctx, versionKey("dataset", name)) + // The declaration wins: it is the author saying which rows to score, and it + // is right whether or not there is an azd environment to have recorded one. + version := pinned + if version == "" { + version = ec.getEnvValue(ctx, versionKey("dataset", name)) + } if version == "" { versions, err := ec.datasetClient.ListDatasetVersions(ctx, name, ProjectEndpointAPIVersion) if err != nil { @@ -705,6 +712,28 @@ func localDatasetPath(configPath string, group *project.Eval) string { return filepath.Join(filepath.Dir(configPath), decl.Source) } +// declaredDatasetVersion is the `version:` the catalog pins this dataset to. +// +// A pin is the author saying which rows to score, and it is read from the +// declaration rather than from the environment: the recorded version means the +// one the file's content published, which is what the deploy's drift check +// compares against, and overwriting it with a pin made removing the pin later +// read as somebody publishing behind the configuration's back. +func declaredDatasetVersion(configPath string, group *project.Eval) string { + if group == nil || configPath == "" { + return "" + } + cfg, err := project.LoadEvalConfig(configPath) + if err != nil || cfg == nil { + return "" + } + decl, ok := cfg.DatasetDeclaration(group.Dataset) + if !ok { + return "" + } + return decl.Version +} + // datasetIsDeclared says whether the configuration's catalog holds the dataset // this eval names. // diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/agent_outside_project_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/agent_outside_project_test.go index 7ad891f53b6..8f01b5ff9a1 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/agent_outside_project_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/agent_outside_project_test.go @@ -49,7 +49,12 @@ func TestAnAgentOutsideTheProjectStillCannotReachFurtherOut(t *testing.T) { elsewhere := filepath.Join(root, "shared", "agent") require.NoError(t, os.MkdirAll(elsewhere, 0o750)) - writeOptimizeConfig(t, elsewhere, "instruction_file: ../../../secret.md\n", "") + // Four levels up from .agent_configs/baseline, so the pointer names a file + // that really is there: a traversal to nothing would be refused by the read + // whether or not containment refused it first. + writeOptimizeConfig(t, elsewhere, "instruction_file: ../../../../secret.md\n", "") + require.FileExists(t, + filepath.Join(elsewhere, ".agent_configs", "baseline", "..", "..", "..", "..", "secret.md")) svc := &azdext.ServiceConfig{Name: "agent", Host: AgentHost, RelativePath: elsewhere} proj := &azdext.ProjectConfig{Path: project, Services: map[string]*azdext.ServiceConfig{"agent": svc}} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go index 8aa54d06057..b1bd148cfcb 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go @@ -44,6 +44,10 @@ type Reconciler interface { // dataset backing the group, or empty when it is already registered; it lets // the reconciler bind criteria to the columns that actually exist. EnsureEval(ctx context.Context, group Eval, datasetPath string) (id string, created bool, err error) + // ReserveDeclared marks the evals these names already resolve to as spoken + // for, so no other declaration adopts one. Called once before reconciling, + // because adoption otherwise depends on the order the file lists them in. + ReserveDeclared(ctx context.Context, names []string) } // EvalServiceTargetProvider deploys eval resources during `azd up`. azd owns @@ -182,6 +186,12 @@ func (p *EvalServiceTargetProvider) Deploy( // resolve to. An evaluator tracking latest that publishes a new version // leaves every eval that runs it alone, which is what keeps a rubric edit // comparable against the runs before it. + declared := make([]string, 0, len(cfg.Evals)) + for i := range cfg.Evals { + declared = append(declared, cfg.Evals[i].Name) + } + reconciler.ReserveDeclared(ctx, declared) + for i := range cfg.Evals { eval := cfg.Evals[i] report(progress, messages.ReconcilingEval(eval.Name)) From a17d48879333dd2007345ad3630a340549f533b7 Mon Sep 17 00:00:00 2001 From: mohessie Date: Wed, 19 Aug 2026 12:30:02 +0300 Subject: [PATCH 303/320] Reserve only what a deploy is keeping, and stamp the version it scored Sixth round. Both findings are narrow, and both are the tail of earlier ones. Reservation claimed the eval a name used to resolve to whether or not the declaration was about to abandon it. Rename an eval and give its freed name to a different one in the same commit, and the recreated declaration held back the eval the rename was continuing -- so the rename forked a history rather than keeping it. A declaration that is going to be recreated now reserves nothing. The recreate decision moved into one place while doing it, since two callers were about to compute it. And the run stamped itself with the version the last deploy recorded while scoring the rows the pin named. The previous commit fixed which rows are downloaded and left the label behind, so `run list`, `run show` and `-o json` all reported a version the run had not read. Three tests were also not guarding what they accompanied: neither call site of ReserveDeclared was covered, the one line wiring the pin into the read was not, and the drift-baseline guard was a literal-text tripwire that a respelling of the same defect walked straight past. It drives EnsureDataset now and reads the environment back. Every guard in this commit was mutation-checked, including that reserving still stops the collision it was added for. --- .../internal/cmd/declared_pin_test.go | 49 +++++++++++++-- .../internal/cmd/digest_migration_test.go | 35 ++++++++++- .../internal/cmd/eval_group.go | 6 +- .../internal/cmd/reconciler.go | 63 ++++++++++++------- .../azure.ai.evaluations/internal/cmd/run.go | 9 ++- .../internal/cmd/wiring_test.go | 39 ++++++++++++ .../internal/project/service_target_eval.go | 15 ++--- 7 files changed, 174 insertions(+), 42 deletions(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/wiring_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/declared_pin_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/declared_pin_test.go index 7ee7109b285..d42bfbb6f9c 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/declared_pin_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/declared_pin_test.go @@ -72,13 +72,54 @@ func TestNoPinIsReadWhenThereIsNothingToReadItFrom(t *testing.T) { // The deploy leaves the recorded version alone, so it keeps meaning "the // version this file's content published" and the drift check keeps working. +// +// Driven through EnsureDataset rather than asserted against the source, so +// respelling the write does not slip past. func TestPinningDoesNotOverwriteTheVersionTheContentPublished(t *testing.T) { - body, err := os.ReadFile("reconciler.go") + dir := t.TempDir() + localPath := filepath.Join(dir, "golden.jsonl") + require.NoError(t, os.WriteFile(localPath, []byte("{\"query\":\"hi\"}\n"), 0o600)) + + digest, err := project.Fingerprint(localPath) require.NoError(t, err) - assert.NotContains(t, string(body), - `r.ec.remember(ctx, versionKey("dataset", decl.Name), decl.Version)`, - "writing the pin here made removing it later read as drift") + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if !strings.Contains(r.URL.Path, "/versions/") { + w.WriteHeader(http.StatusNotFound) + return + } + w.Header().Set("Content-Type", "application/json") + // assert, not require: this runs on the server's goroutine. + assert.NoError(t, json.NewEncoder(w).Encode(map[string]any{"name": "golden", "version": "1"})) + })) + t.Cleanup(srv.Close) + + // The last deploy published version 2 from this same file. The author then + // pinned version 1 without touching it, so the digest still matches. + env := &testEnvServer{values: map[string]string{ + project.FingerprintKey("dataset", "golden"): digest, + versionKey("dataset", "golden"): "2", + }} + pipeline := runtime.NewPipeline("test", "v1", runtime.PipelineOptions{}, + &policy.ClientOptions{Retry: policy.RetryOptions{MaxRetries: -1}}) + r := &evalReconciler{ec: &evalContext{ + azdClient: newTestAzdClient(t, env), + envName: "test", + datasetClient: dataset_api.NewDatasetClientFromPipeline(srv.URL, pipeline), + }} + + version, changed, err := r.EnsureDataset( + context.Background(), + project.DatasetDecl{Name: "golden", Version: "1"}, + localPath, + ) + + require.NoError(t, err) + assert.Equal(t, "1", version, "the pin is what the deploy reports") + assert.False(t, changed) + assert.Equal(t, "2", env.values[versionKey("dataset", "golden")], + "the recorded version still means what this file published, "+ + "or removing the pin later reads as somebody publishing behind the config") } // And the pin is what the run downloads. Reading the declaration and then not diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/digest_migration_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/digest_migration_test.go index b421632f798..9ddf8d3b80b 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/digest_migration_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/digest_migration_test.go @@ -133,7 +133,7 @@ func TestADeclarationDoesNotAdoptAnEvalAnotherOneOwns(t *testing.T) { }} r, created := evalServiceReconciler(t, env, "eval_1") - r.ReserveDeclared(context.Background(), []string{"weekly", "nightly"}) + r.ReserveDeclared(context.Background(), []project.Eval{newcomer, owner}) id, wasCreated, err := r.EnsureEval(context.Background(), newcomer, "") require.NoError(t, err) @@ -158,7 +158,7 @@ func TestARenamedEvalIsStillAdopted(t *testing.T) { }} r, created := evalServiceReconciler(t, env, "eval_1") - r.ReserveDeclared(context.Background(), []string{"evening"}) + r.ReserveDeclared(context.Background(), []project.Eval{renamed}) id, wasCreated, err := r.EnsureEval(context.Background(), renamed, "") require.NoError(t, err) @@ -167,6 +167,37 @@ func TestARenamedEvalIsStillAdopted(t *testing.T) { assert.Empty(t, *created) } +// A declaration about to be recreated reserves nothing: it is abandoning that +// eval, and holding it back refused the rename that legitimately continues it. +func TestARenameIsStillAdoptedWhenTheOldNameIsRecycled(t *testing.T) { + original := windowedEval("morning", 24) + digest, err := project.FingerprintGroup(original) + require.NoError(t, err) + shipped, err := project.FingerprintDefinition(original) + require.NoError(t, err) + + // The same commit renames it and gives the freed name to a different eval. + renamed := original + renamed.Name = "evening" + recycled := windowedEval("morning", 24) + recycled.Evaluators = evalcore.EvaluatorList{{Evaluator: "builtin.coherence"}} + + env := &testEnvServer{values: map[string]string{ + digestIDKey(digest): "eval_1", + idKey("eval", "morning"): "eval_1", + project.FingerprintKey("eval", "morning"): shipped, + }} + r, _ := evalServiceReconciler(t, env, "eval_1") + + r.ReserveDeclared(context.Background(), []project.Eval{recycled, renamed}) + id, wasCreated, err := r.EnsureEval(context.Background(), renamed, "") + + require.NoError(t, err) + assert.Equal(t, "eval_1", id, + "the eval morning is abandoning is the one evening continues") + assert.False(t, wasCreated) +} + // The map is built on first use, because the reconciler is also constructed // literally in several places and writing to a nil map panics. func TestClaimingWorksOnAReconcilerBuiltDirectly(t *testing.T) { diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/eval_group.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/eval_group.go index 8f52d6923a5..75c5f717e36 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/eval_group.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/eval_group.go @@ -80,11 +80,7 @@ func newEvalCreateCommand() *cobra.Command { reconciler := &evalReconciler{ec: ec} // Every eval the file declares, not only the one being created: an // eval another declaration already owns must not be adopted here. - declared := make([]string, 0, len(cfg.Evals)) - for i := range cfg.Evals { - declared = append(declared, cfg.Evals[i].Name) - } - reconciler.ReserveDeclared(ctx, declared) + reconciler.ReserveDeclared(ctx, cfg.Evals) out := cmd.OutOrStdout() // Before anything is pushed. Publishing is not free -- a dataset diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go index 5dbd39b27f9..bb0851d8246 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go @@ -55,21 +55,56 @@ func (r *evalReconciler) claim(id string) { r.claimed[id] = true } -// ReserveDeclared marks the evals these names already resolve to as spoken for. +// ReserveDeclared marks the evals these declarations already resolve to as +// spoken for. // -// Claiming only as each declaration finishes makes adoption depend on file +// Claiming only as each declaration finished made adoption depend on file // order: a declaration listed above the one that owns an eval would adopt it, // rename it, and end up sharing its runs. Reserving up front is the same guard // without the ordering. A name the environment holds no id for reserves // nothing, which is what leaves a genuine rename free to adopt. -func (r *evalReconciler) ReserveDeclared(ctx context.Context, names []string) { - for _, name := range names { - if id := r.ec.getEnvValue(ctx, idKey("eval", name)); id != "" { - r.claim(id) +// +// A declaration that is going to be recreated reserves nothing either: it is +// about to abandon that eval, and holding it back would refuse the rename that +// legitimately continues it. +func (r *evalReconciler) ReserveDeclared(ctx context.Context, groups []project.Eval) { + for i := range groups { + id := r.ec.getEnvValue(ctx, idKey("eval", groups[i].Name)) + if id == "" { + continue } + _, _, recreate, err := r.evalDigests(ctx, groups[i]) + if err == nil && recreate { + continue + } + r.claim(id) } } +// evalDigests hashes a declaration both ways and says whether its substance +// changed since the last deploy. +// +// digest identifies the declaration and keys the id a rename looks up. +// definition is what the service stores, and is what the recreate comparison +// is made against. The recorded baseline is the definition from the build that +// split them on, and the full digest from every build before: equality with +// either says the declaration is what was deployed. +func (r *evalReconciler) evalDigests( + ctx context.Context, + group project.Eval, +) (digest, definition string, recreate bool, err error) { + digest, err = project.FingerprintGroup(group) + if err != nil { + return "", "", false, err + } + definition, err = project.FingerprintDefinition(group) + if err != nil { + return "", "", false, err + } + prior := r.ec.getEnvValue(ctx, project.FingerprintKey("eval", group.Name)) + return digest, definition, prior != "" && prior != definition && prior != digest, nil +} + // EnsureDataset registers a new version only when the local content changed. // // The dataset API exposes no content hash, so comparing against the service @@ -504,25 +539,11 @@ func (r *evalReconciler) EnsureEval( // dataset, target, level — needs a new eval. Name and description are // excluded from the digest and pushed in place instead, and so are // max_samples and source:, which the run carries rather than the eval. - recreate := false - digest, err := project.FingerprintGroup(group) - if err != nil { - return "", false, err - } - definition, err := project.FingerprintDefinition(group) + digest, definition, recreate, err := r.evalDigests(ctx, group) if err != nil { return "", false, err } key := project.FingerprintKey("eval", group.Name) - // Compared against both. The recorded baseline is the definition from this - // build on, and the full digest from every build before the two were split: - // reading only the definition made the first deploy after an upgrade recreate - // every eval carrying max_samples or source:, for a file nobody had touched, - // and orphaned the runs taken under it. Equality with either says the - // declaration is what was deployed. - if prior := r.ec.getEnvValue(ctx, key); prior != "" && prior != definition && prior != digest { - recreate = true - } // Building the request is also what checks the declaration against the // dataset's columns, so it happens before the reuse decision: a dataset can diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go index 6f6888801de..6fc6bd97287 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go @@ -205,7 +205,14 @@ func buildRunCommand(use, short string) *cobra.Command { // no dataset, so it records none. if group != nil && group.Dataset != "" && group.Source == nil { metadata[metaDataset] = group.Dataset - if v := ec.getEnvValue(ctx, versionKey("dataset", group.Dataset)); v != "" { + // The version the rows came from, which is the pin when the + // declaration carries one. Reading only what the last deploy + // recorded stamped the run with a version it had not scored. + v := declaredDatasetVersion(configPath, group) + if v == "" { + v = ec.getEnvValue(ctx, versionKey("dataset", group.Dataset)) + } + if v != "" { metadata[metaDatasetVersion] = v } } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/wiring_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/wiring_test.go new file mode 100644 index 00000000000..9bd805bd1b2 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/wiring_test.go @@ -0,0 +1,39 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "os" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// The reconciler's guard is worth nothing if the commands do not arm it, and a +// unit test that calls ReserveDeclared itself proves only the method. Read from +// the source because reaching either call site needs a project and a service. +func TestBothCommandsReserveBeforeTheyReconcile(t *testing.T) { + for _, file := range []string{ + "eval_group.go", + "../project/service_target_eval.go", + } { + t.Run(file, func(t *testing.T) { + body, err := os.ReadFile(file) + require.NoError(t, err) + + assert.Contains(t, string(body), "ReserveDeclared(ctx, cfg.Evals)", + "an eval another declaration owns must not be adopted here") + }) + } +} + +// And the pin has to be handed to the read, not merely read from the config. +func TestTheRunHandsThePinToTheDatasetRead(t *testing.T) { + body, err := os.ReadFile("run.go") + require.NoError(t, err) + + assert.Contains(t, string(body), "declaredDatasetVersion(configPath, group), maxSamples)", + "reading the declaration and not using it leaves the run on the recorded version") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go index b1bd148cfcb..e7ffdd82c32 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go @@ -44,10 +44,11 @@ type Reconciler interface { // dataset backing the group, or empty when it is already registered; it lets // the reconciler bind criteria to the columns that actually exist. EnsureEval(ctx context.Context, group Eval, datasetPath string) (id string, created bool, err error) - // ReserveDeclared marks the evals these names already resolve to as spoken - // for, so no other declaration adopts one. Called once before reconciling, - // because adoption otherwise depends on the order the file lists them in. - ReserveDeclared(ctx context.Context, names []string) + // ReserveDeclared marks the evals these declarations already resolve to as + // spoken for, so no other declaration adopts one. Called once before + // reconciling, because adoption otherwise depends on the order the file + // lists them in. + ReserveDeclared(ctx context.Context, groups []Eval) } // EvalServiceTargetProvider deploys eval resources during `azd up`. azd owns @@ -186,11 +187,7 @@ func (p *EvalServiceTargetProvider) Deploy( // resolve to. An evaluator tracking latest that publishes a new version // leaves every eval that runs it alone, which is what keeps a rubric edit // comparable against the runs before it. - declared := make([]string, 0, len(cfg.Evals)) - for i := range cfg.Evals { - declared = append(declared, cfg.Evals[i].Name) - } - reconciler.ReserveDeclared(ctx, declared) + reconciler.ReserveDeclared(ctx, cfg.Evals) for i := range cfg.Evals { eval := cfg.Evals[i] From 8110cf492c57fc17127127d0f72db1cd111e5edc Mon Sep 17 00:00:00 2001 From: mohessie Date: Wed, 19 Aug 2026 13:22:21 +0300 Subject: [PATCH 304/320] Label a run with where its rows came from, and decide recreation once Seventh round. Both findings are in the commit before this one, and both are narrower again. Preferring the pin for the run's dataset label was right only for the branch that reads the pin. A declaration carrying both `source:` and `version:` reads its rows off disk -- the pin never reaches the fetch -- so the run was stamped with a registry version whose rows it had not read, and `run list`, `run show` and `-o json` all reported it. The label follows the same branch the rows did now: the recorded version is the honest answer for a local file, and checkDatasetRegistered has already confirmed the rows match it. And the recreate decision was read twice from a value that arrives over gRPC. Reservation asked before the loop and EnsureEval asked again inside it, so one transient read failure between them left an eval unreserved and then reused -- two declarations on one id, the collision reservation exists to stop. Each declaration is decided once and remembered. The block that labels a run had no test at all; it has four now, and the decision is pinned against an environment that answers differently the second time. --- .../internal/cmd/reconciler.go | 71 +++++++++-- .../azure.ai.evaluations/internal/cmd/run.go | 30 +++-- .../internal/cmd/scored_version_test.go | 117 ++++++++++++++++++ 3 files changed, 199 insertions(+), 19 deletions(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/scored_version_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go index bb0851d8246..c06f3db7f52 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go @@ -34,6 +34,10 @@ type evalReconciler struct { // eval -- and adopting it renames that eval and leaves the declaration that // asked for it sharing the other one's runs. claimed map[string]bool + + // decided holds each declaration's digests, so reservation and + // reconciliation cannot answer the question differently. + decided map[string]evalDecision } var _ project.Reconciler = (*evalReconciler)(nil) @@ -69,18 +73,68 @@ func (r *evalReconciler) claim(id string) { // legitimately continues it. func (r *evalReconciler) ReserveDeclared(ctx context.Context, groups []project.Eval) { for i := range groups { - id := r.ec.getEnvValue(ctx, idKey("eval", groups[i].Name)) - if id == "" { + decision, err := r.decide(ctx, groups[i]) + if err != nil { + // Nothing decided, so nothing skipped. The error surfaces from + // EnsureEval, where it can fail the deploy. continue } - _, _, recreate, err := r.evalDigests(ctx, groups[i]) - if err == nil && recreate { + id := r.ec.getEnvValue(ctx, idKey("eval", groups[i].Name)) + if id == "" || decision.recreate { continue } r.claim(id) } } +// evalDecision is what one declaration's digests settled. +type evalDecision struct { + digest string + definition string + recreate bool +} + +// decide hashes a declaration both ways and says whether its substance changed +// since the last deploy, answering the same way every time it is asked. +// +// Remembered per name because two callers ask: reservation, before anything is +// reconciled, and EnsureEval itself. The recorded baseline is read over gRPC +// and a read that failed answers "", so asking twice let one transient failure +// leave an eval unreserved and then reused -- two declarations on one id, which +// is the collision reservation exists to stop. +// +// digest identifies the declaration and keys the id a rename looks up. +// definition is what the service stores, and is what the recreate comparison is +// made against. The recorded baseline is the definition from the build that +// split them on, and the full digest from every build before: equality with +// either says the declaration is what was deployed. +func (r *evalReconciler) decide(ctx context.Context, group project.Eval) (evalDecision, error) { + if decided, ok := r.decided[group.Name]; ok { + return decided, nil + } + + digest, err := project.FingerprintGroup(group) + if err != nil { + return evalDecision{}, err + } + definition, err := project.FingerprintDefinition(group) + if err != nil { + return evalDecision{}, err + } + prior := r.ec.getEnvValue(ctx, project.FingerprintKey("eval", group.Name)) + + decided := evalDecision{ + digest: digest, + definition: definition, + recreate: prior != "" && prior != definition && prior != digest, + } + if r.decided == nil { + r.decided = map[string]evalDecision{} + } + r.decided[group.Name] = decided + return decided, nil +} + // evalDigests hashes a declaration both ways and says whether its substance // changed since the last deploy. // @@ -93,16 +147,11 @@ func (r *evalReconciler) evalDigests( ctx context.Context, group project.Eval, ) (digest, definition string, recreate bool, err error) { - digest, err = project.FingerprintGroup(group) - if err != nil { - return "", "", false, err - } - definition, err = project.FingerprintDefinition(group) + decided, err := r.decide(ctx, group) if err != nil { return "", "", false, err } - prior := r.ec.getEnvValue(ctx, project.FingerprintKey("eval", group.Name)) - return digest, definition, prior != "" && prior != definition && prior != digest, nil + return decided.digest, decided.definition, decided.recreate, nil } // EnsureDataset registers a new version only when the local content changed. diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go index 6fc6bd97287..815c065651f 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go @@ -205,14 +205,7 @@ func buildRunCommand(use, short string) *cobra.Command { // no dataset, so it records none. if group != nil && group.Dataset != "" && group.Source == nil { metadata[metaDataset] = group.Dataset - // The version the rows came from, which is the pin when the - // declaration carries one. Reading only what the last deploy - // recorded stamped the run with a version it had not scored. - v := declaredDatasetVersion(configPath, group) - if v == "" { - v = ec.getEnvValue(ctx, versionKey("dataset", group.Dataset)) - } - if v != "" { + if v := ec.scoredDatasetVersion(ctx, group, configPath); v != "" { metadata[metaDatasetVersion] = v } } @@ -741,6 +734,27 @@ func declaredDatasetVersion(configPath string, group *project.Eval) string { return decl.Version } +// scoredDatasetVersion labels a run with the version its rows actually came +// from. +// +// It has to follow the same branch the rows did. A declaration carrying both +// `source:` and `version:` reads the file from disk, so the pin says nothing +// about what was scored and the recorded version -- the one this file's content +// published, which checkDatasetRegistered has already confirmed the rows match +// -- is the honest label. Only a registered dataset is fetched at the pin. +func (ec *evalContext) scoredDatasetVersion( + ctx context.Context, + group *project.Eval, + configPath string, +) string { + if localDatasetPath(configPath, group) == "" { + if pinned := declaredDatasetVersion(configPath, group); pinned != "" { + return pinned + } + } + return ec.getEnvValue(ctx, versionKey("dataset", group.Dataset)) +} + // datasetIsDeclared says whether the configuration's catalog holds the dataset // this eval names. // diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/scored_version_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/scored_version_test.go new file mode 100644 index 00000000000..743b827cb98 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/scored_version_test.go @@ -0,0 +1,117 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "context" + "os" + "path/filepath" + "testing" + + "azureaieval/internal/project" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// writeCatalog writes a configuration whose dataset carries the given source +// and version, either of which may be empty. +func writeCatalog(t *testing.T, source, version string) string { + t.Helper() + + dir := t.TempDir() + entry := " - name: golden" + if source != "" { + require.NoError(t, os.WriteFile(filepath.Join(dir, source), []byte("{\"query\":\"hi\"}\n"), 0o600)) + entry += "\n source: ./" + source + } + if version != "" { + entry += "\n version: \"" + version + "\"" + } + body := "datasets:\n" + entry + ` +evals: + - name: quality + dataset: golden + evaluators: + - evaluator: builtin.relevance +` + path := filepath.Join(dir, "azure.eval.yaml") + require.NoError(t, os.WriteFile(path, []byte(body), 0o600)) + return path +} + +// The label a run carries has to follow the same branch its rows did. +// +// A declaration with both `source:` and `version:` reads the file from disk, so +// the pin says nothing about what was scored: the recorded version is the one +// this file's content published, and checkDatasetRegistered has already +// confirmed the rows match it. +func TestARunOverALocalFileIsLabelledWithWhatTheFilePublished(t *testing.T) { + env := &testEnvServer{values: map[string]string{ + versionKey("dataset", "golden"): "2", + }} + ec := &evalContext{azdClient: newTestAzdClient(t, env), envName: "test"} + group := &project.Eval{Name: "quality", Dataset: "golden"} + + labelled := ec.scoredDatasetVersion( + context.Background(), group, writeCatalog(t, "golden.jsonl", "1")) + + assert.Equal(t, "2", labelled, + "the rows came off disk, so the pin is not what was scored") +} + +// A registered dataset is fetched at the pin, so the pin is the honest label. +func TestARunOverARegisteredDatasetIsLabelledWithThePin(t *testing.T) { + env := &testEnvServer{values: map[string]string{ + versionKey("dataset", "golden"): "2", + }} + ec := &evalContext{azdClient: newTestAzdClient(t, env), envName: "test"} + group := &project.Eval{Name: "quality", Dataset: "golden"} + + labelled := ec.scoredDatasetVersion( + context.Background(), group, writeCatalog(t, "", "1")) + + assert.Equal(t, "1", labelled, "the pin is the version the rows were read at") +} + +// With no pin either way, the recorded version answers. +func TestWithoutAPinTheRecordedVersionIsTheLabel(t *testing.T) { + env := &testEnvServer{values: map[string]string{ + versionKey("dataset", "golden"): "2", + }} + ec := &evalContext{azdClient: newTestAzdClient(t, env), envName: "test"} + group := &project.Eval{Name: "quality", Dataset: "golden"} + + assert.Equal(t, "2", ec.scoredDatasetVersion( + context.Background(), group, writeCatalog(t, "", ""))) + assert.Equal(t, "2", ec.scoredDatasetVersion( + context.Background(), group, writeCatalog(t, "golden.jsonl", ""))) +} + +// Reservation and reconciliation ask the same question, and the baseline they +// read it from comes over gRPC. Asking twice let one transient failure leave an +// eval unreserved and then reused -- two declarations on one id. +func TestTheRecreateDecisionIsMadeOncePerDeploy(t *testing.T) { + group := project.Eval{Name: "nightly", Dataset: "golden"} + digest, err := project.FingerprintGroup(group) + require.NoError(t, err) + + env := &testEnvServer{values: map[string]string{ + project.FingerprintKey("eval", "nightly"): "a digest from some older declaration", + }} + r := &evalReconciler{ec: &evalContext{ + azdClient: newTestAzdClient(t, env), envName: "test", + }} + + first, err := r.decide(context.Background(), group) + require.NoError(t, err) + require.True(t, first.recreate, "the fixture has to be a declaration that changed") + + // The environment now answers differently, as a failed read would. + env.values[project.FingerprintKey("eval", "nightly")] = digest + + second, err := r.decide(context.Background(), group) + require.NoError(t, err) + assert.Equal(t, first, second, "the decision must not change under it") +} From 51eb9c2e8ddceed561d8e88f6d0330e12224f842 Mon Sep 17 00:00:00 2001 From: mohessie Date: Wed, 19 Aug 2026 13:46:13 +0300 Subject: [PATCH 305/320] Say nothing rather than assert a provenance the rows no longer have The residual from the eighth round, which otherwise came back clean. Labelling a run from a local file with the recorded version rests on a fingerprint tying the file to it, and a dataset that was registered and has since gained a `source:` has the version and no fingerprint -- checkDatasetRegistered declines to verify in exactly that case, and the label went on asserting anyway. It now declines with it: no version is better than one the rows are not from. Also pinned the two things the round found uncovered: that a local file without a fingerprint is labelled with nothing, and that EnsureEval reads the same decision memo reservation did rather than hashing again. The baseline is read over gRPC, and reading it twice is what let a transient failure flip the answer between them. --- .../azure.ai.evaluations/internal/cmd/run.go | 21 ++++++---- .../internal/cmd/scored_version_test.go | 38 ++++++++++++++++++- 2 files changed, 50 insertions(+), 9 deletions(-) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go index 815c065651f..a5b4f3e5d91 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go @@ -735,22 +735,29 @@ func declaredDatasetVersion(configPath string, group *project.Eval) string { } // scoredDatasetVersion labels a run with the version its rows actually came -// from. +// from, or with nothing when that cannot be said. // // It has to follow the same branch the rows did. A declaration carrying both // `source:` and `version:` reads the file from disk, so the pin says nothing -// about what was scored and the recorded version -- the one this file's content -// published, which checkDatasetRegistered has already confirmed the rows match -// -- is the honest label. Only a registered dataset is fetched at the pin. +// about what was scored. The recorded version is the honest label there only +// once a fingerprint exists to tie the file to it -- that is what +// checkDatasetRegistered confirms, and it also declines when there is none. A +// dataset that was registered and has since gained a `source:` has a recorded +// version and no fingerprint, and stamping the run with it would assert a +// provenance the rows no longer have. func (ec *evalContext) scoredDatasetVersion( ctx context.Context, group *project.Eval, configPath string, ) string { - if localDatasetPath(configPath, group) == "" { - if pinned := declaredDatasetVersion(configPath, group); pinned != "" { - return pinned + if localDatasetPath(configPath, group) != "" { + if ec.getEnvValue(ctx, project.FingerprintKey("dataset", group.Dataset)) == "" { + return "" } + return ec.getEnvValue(ctx, versionKey("dataset", group.Dataset)) + } + if pinned := declaredDatasetVersion(configPath, group); pinned != "" { + return pinned } return ec.getEnvValue(ctx, versionKey("dataset", group.Dataset)) } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/scored_version_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/scored_version_test.go index 743b827cb98..5402a356c38 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/scored_version_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/scored_version_test.go @@ -7,6 +7,7 @@ import ( "context" "os" "path/filepath" + "strings" "testing" "azureaieval/internal/project" @@ -49,7 +50,8 @@ evals: // confirmed the rows match it. func TestARunOverALocalFileIsLabelledWithWhatTheFilePublished(t *testing.T) { env := &testEnvServer{values: map[string]string{ - versionKey("dataset", "golden"): "2", + versionKey("dataset", "golden"): "2", + project.FingerprintKey("dataset", "golden"): "a digest of the file", }} ec := &evalContext{azdClient: newTestAzdClient(t, env), envName: "test"} group := &project.Eval{Name: "quality", Dataset: "golden"} @@ -61,6 +63,23 @@ func TestARunOverALocalFileIsLabelledWithWhatTheFilePublished(t *testing.T) { "the rows came off disk, so the pin is not what was scored") } +// And with no fingerprint there is nothing tying the file to that version. A +// dataset that was registered and has since gained a `source:` has a recorded +// version and no fingerprint, and the rows are whatever the file now holds. +func TestALocalFileWithNoFingerprintIsLabelledWithNothing(t *testing.T) { + env := &testEnvServer{values: map[string]string{ + versionKey("dataset", "golden"): "3", + }} + ec := &evalContext{azdClient: newTestAzdClient(t, env), envName: "test"} + group := &project.Eval{Name: "quality", Dataset: "golden"} + + labelled := ec.scoredDatasetVersion( + context.Background(), group, writeCatalog(t, "golden.jsonl", "")) + + assert.Empty(t, labelled, + "no version is better than one the rows are not from") +} + // A registered dataset is fetched at the pin, so the pin is the honest label. func TestARunOverARegisteredDatasetIsLabelledWithThePin(t *testing.T) { env := &testEnvServer{values: map[string]string{ @@ -78,7 +97,8 @@ func TestARunOverARegisteredDatasetIsLabelledWithThePin(t *testing.T) { // With no pin either way, the recorded version answers. func TestWithoutAPinTheRecordedVersionIsTheLabel(t *testing.T) { env := &testEnvServer{values: map[string]string{ - versionKey("dataset", "golden"): "2", + versionKey("dataset", "golden"): "2", + project.FingerprintKey("dataset", "golden"): "a digest of the file", }} ec := &evalContext{azdClient: newTestAzdClient(t, env), envName: "test"} group := &project.Eval{Name: "quality", Dataset: "golden"} @@ -115,3 +135,17 @@ func TestTheRecreateDecisionIsMadeOncePerDeploy(t *testing.T) { require.NoError(t, err) assert.Equal(t, first, second, "the decision must not change under it") } + +// Both askers have to read the same memo, or the two can still disagree. +func TestEnsureEvalAsksTheSameMemoReservationDid(t *testing.T) { + body, err := os.ReadFile("reconciler.go") + require.NoError(t, err) + source := string(body) + + assert.Contains(t, source, "r.evalDigests(ctx, group)", + "EnsureEval has to go through the memo rather than hashing again") + assert.Contains(t, source, "decided, err := r.decide(ctx, group)", + "and evalDigests is what reads it") + assert.Equal(t, 1, strings.Count(source, "project.FingerprintKey(\"eval\", group.Name))"), + "one read of the baseline, or a transient failure flips the decision") +} From 5e6215b2a20d9b062d20e41f4942090334b61c57 Mon Sep 17 00:00:00 2001 From: mohessie Date: Wed, 19 Aug 2026 13:53:04 +0300 Subject: [PATCH 306/320] Cut the twelfth bug bash build --- cli/azd/extensions/azure.ai.evaluations/CHANGELOG.md | 2 +- cli/azd/extensions/azure.ai.evaluations/extension.yaml | 2 +- cli/azd/extensions/azure.ai.evaluations/version.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cli/azd/extensions/azure.ai.evaluations/CHANGELOG.md b/cli/azd/extensions/azure.ai.evaluations/CHANGELOG.md index fef7e79f0e1..69b395018cf 100644 --- a/cli/azd/extensions/azure.ai.evaluations/CHANGELOG.md +++ b/cli/azd/extensions/azure.ai.evaluations/CHANGELOG.md @@ -1,6 +1,6 @@ # Release History -## 1.0.15-beta (Unreleased) +## 1.0.16-beta (Unreleased) First release of the Foundry evaluations extension. diff --git a/cli/azd/extensions/azure.ai.evaluations/extension.yaml b/cli/azd/extensions/azure.ai.evaluations/extension.yaml index f132df4c01d..8032f9bfb43 100644 --- a/cli/azd/extensions/azure.ai.evaluations/extension.yaml +++ b/cli/azd/extensions/azure.ai.evaluations/extension.yaml @@ -5,7 +5,7 @@ displayName: Foundry evaluations (Beta) description: Define and run Foundry evaluations from your terminal. (Beta) usage: azd ai eval [options] # NOTE: Make sure version.txt is in sync with this version. -version: 1.0.15-beta +version: 1.0.16-beta requiredAzdVersion: ">=1.27.1" language: go capabilities: diff --git a/cli/azd/extensions/azure.ai.evaluations/version.txt b/cli/azd/extensions/azure.ai.evaluations/version.txt index c2af346c4b5..7ae7d4397e7 100644 --- a/cli/azd/extensions/azure.ai.evaluations/version.txt +++ b/cli/azd/extensions/azure.ai.evaluations/version.txt @@ -1 +1 @@ -1.0.15-beta +1.0.16-beta From 35faaa5cf31323a974262917e316b316c36ab8ea Mon Sep 17 00:00:00 2001 From: mohessie Date: Wed, 19 Aug 2026 14:04:52 +0300 Subject: [PATCH 307/320] Use the dictionary's spelling --- .../internal/project/service_target_eval.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go index e7ffdd82c32..92951143f2d 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go @@ -240,7 +240,7 @@ func (p *EvalServiceTargetProvider) evalBaseDir( // does this: joining one under the project produced /C:/shared/evals, // which is the bug this fixes, reached from a different input. An empty root is // azd having failed to name the project, where the relative path is what the -// behaviour used to be and is better than resolving against nothing. +// extension did before and is better than resolving against nothing. func baseDirUnder(projectRoot string, serviceConfig *azdext.ServiceConfig) string { relative := serviceRelativeDir(serviceConfig) if filepath.IsAbs(relative) || projectRoot == "" { From 9e072c696243c7a7ada87643cf5ad8ce6c08a93c Mon Sep 17 00:00:00 2001 From: mohessie Date: Wed, 19 Aug 2026 14:26:16 +0300 Subject: [PATCH 308/320] Use the dictionary's spelling throughout --- .../azure.ai.evaluations/internal/cmd/eval_choice_test.go | 2 +- .../extensions/azure.ai.evaluations/internal/cmd/main_test.go | 2 +- .../extensions/azure.ai.evaluations/internal/cmd/portal_test.go | 2 +- .../azure.ai.evaluations/internal/cmd/resolution_test.go | 2 +- .../azure.ai.evaluations/internal/pkg/eval_api/models.go | 2 +- .../azure.ai.evaluations/internal/urlsafe/urlsafe_test.go | 2 +- .../azure.ai.evaluations/tests/live/run_cancel_test.go | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/eval_choice_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/eval_choice_test.go index d17840b398c..a4e996fab05 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/eval_choice_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/eval_choice_test.go @@ -56,7 +56,7 @@ func TestSeveralEvalsErrorStillNamesEveryCandidate(t *testing.T) { cmd := newEvalCreateCommand() // Without this the picker reaches azdext.NewAzdClient and attempts a real // RPC, which passes only because resolving an empty address fails fast. - // This test is about the message, not about network behaviour. + // This test is about the message, not about network behavior. cmd.Flags().Bool("no-prompt", true, "") _, err := cfg.Eval(chooseEval(cmd, cfg, "")) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/main_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/main_test.go index b726e669783..0a60e3fe9bc 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/main_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/main_test.go @@ -10,7 +10,7 @@ import ( "github.com/fatih/color" ) -// TestMain pins colour off for the whole package. +// TestMain pins color off for the whole package. // // fatih/color decides once, at init, from whether the process's stdout is a // terminal -- not from the writer a renderer was handed. `go test` pipes diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/portal_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/portal_test.go index ecf5ae0a9b1..5fc9febd8a5 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/portal_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/portal_test.go @@ -38,7 +38,7 @@ func TestWritePortalLink_SilentWithoutAURL(t *testing.T) { assert.Empty(t, buf.String()) } -// Colour is pinned off for the rest of the package, which leaves nothing +// Color is pinned off for the rest of the package, which leaves nothing // exercising the branch that actually runs in a terminal. The escape codes have // to wrap the URL and nothing else: one leaking into the label, or past the // newline, follows the link into whatever a reader pastes it in. diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/resolution_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/resolution_test.go index fa417ce43c3..b42980a418d 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/resolution_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/resolution_test.go @@ -11,7 +11,7 @@ import ( "github.com/stretchr/testify/assert" ) -// Precedence decides behaviour without announcing it, so a wrong answer here +// Precedence decides behavior without announcing it, so a wrong answer here // is silent. options.max_samples was parsed and dropped once already, which is // what these lock down. func TestResolveMaxSamples_Precedence(t *testing.T) { diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/models.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/models.go index 9249a8e35ac..b41efd8877f 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/models.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/models.go @@ -317,7 +317,7 @@ const ( // a dataset. The service reads them from Application Insights, so the agent // must be emitting gen_ai.input.messages / gen_ai.output.messages. // - // Deprecated in favour of EvalRunDataSourceTypeTracePreview, which is the + // Deprecated in favor of EvalRunDataSourceTypeTracePreview, which is the // only shape that carries an agent version. This one silently discards // agent_version and start_time and re-imposes its own lookback. EvalRunDataSourceTypeTraces EvalRunDataSourceType = "azure_ai_traces" diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/urlsafe/urlsafe_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/urlsafe/urlsafe_test.go index 21cf2f190b4..12e9cb12c72 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/urlsafe/urlsafe_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/urlsafe/urlsafe_test.go @@ -14,7 +14,7 @@ import ( const sasSecret = "REDACT_ME_SECRET" -// These tests pin the premise as well as the behaviour: url.URL.Redacted is the +// These tests pin the premise as well as the behavior: url.URL.Redacted is the // call that looks correct and leaks, so if someone reaches for it again the // first assertion explains why they should not. func TestURLDropsTheSASSignature(t *testing.T) { diff --git a/cli/azd/extensions/azure.ai.evaluations/tests/live/run_cancel_test.go b/cli/azd/extensions/azure.ai.evaluations/tests/live/run_cancel_test.go index 6b53228f03f..169c7173470 100644 --- a/cli/azd/extensions/azure.ai.evaluations/tests/live/run_cancel_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/tests/live/run_cancel_test.go @@ -81,7 +81,7 @@ func TestLiveRunCancel(t *testing.T) { if err != nil { // The service refuses to cancel a run that already left the cancellable // window. That is a race this test starts but does not control, and it - // is the behaviour a separate case already pins, so there is nothing + // is the behavior a separate case already pins, so there is nothing // left here to observe. current, getErr := env.evalClient.GetOpenAIEvalRun(ctx, group.ID, run.ID) require.NoError(t, getErr, "reading the run whose cancel was refused") From 16b030c117e80e37234ca54e95eec0b500f1cf1c Mon Sep 17 00:00:00 2001 From: mohessie Date: Wed, 19 Aug 2026 14:40:52 +0300 Subject: [PATCH 309/320] Refuse an upload with nowhere to register it, before the blob is written From Copilot's review of #9499. startPendingUpload returns two things: a SAS saying where to write, and a blob URI saying what to register. Only the SAS was checked, and they are different fields of the same response -- so a response carrying one without the other uploaded the bytes and then finalized against "/name.jsonl", leaving a blob nothing points at and a publish that failed for a reason the message did not name. Both are checked before anything is written now, and the two have separate messages because they are not the same problem. The other thread on that PR asked for errors.AsType over errors.As, on the grounds that the lint workflow's `go fix -diff` would fail on it. It does not: `go fix -diff ./...` is clean on both modules, including the errors.As added since. Resolved as outdated. --- .../internal/messages/messages.go | 9 ++ .../internal/pkg/dataset_api/operations.go | 11 ++- .../pkg/dataset_api/upload_uris_test.go | 83 +++++++++++++++++++ 3 files changed, 102 insertions(+), 1 deletion(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/upload_uris_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go b/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go index 02afa310fb0..e77445512dc 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go @@ -1132,6 +1132,15 @@ func NoUploadURI() error { return errors.New("no upload SAS URI returned from startPendingUpload") } +// NoBlobURI reports an accepted upload the service gave no way to finalize. +// +// Separate from NoUploadURI because they are different fields of the same +// response: the SAS says where to write, the blob URI says what to register, +// and a response can carry one without the other. +func NoBlobURI() error { + return errors.New("no blob URI returned from startPendingUpload, so there is nothing to register the upload as") +} + // UploadingBlob reports the dataset content failing to upload. func UploadingBlob(err error) error { return fmt.Errorf("uploading blob: %w", err) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/operations.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/operations.go index 4e287c5ec21..e7b81db59df 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/operations.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/operations.go @@ -248,6 +248,15 @@ func (c *DatasetClient) UploadVersion( if uploadURI == "" { return nil, messages.NoUploadURI() } + // Checked here rather than at step 3, because the two come from different + // fields of the same response and only one of them is needed to write. A + // response carrying the SAS and no blobUri uploaded the bytes and then + // finalized against "/name.jsonl", leaving a blob nothing points at and a + // publish that failed for a reason the message did not name. + blobURI := pending.ResolvedBlobURI() + if blobURI == "" { + return nil, messages.NoBlobURI() + } // Step 2: Upload the JSONL file to blob storage. // One blob per dataset, which is what the container-listing fallback in @@ -263,7 +272,7 @@ func (c *DatasetClient) UploadVersion( } // Step 3: Finalize the dataset version with the full blob URI. - dataURI := strings.TrimSuffix(pending.ResolvedBlobURI(), "/") + "/" + blobName + dataURI := strings.TrimSuffix(blobURI, "/") + "/" + blobName return c.FinalizeDatasetVersion(ctx, name, newVersion, dataURI, apiVersion) } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/upload_uris_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/upload_uris_test.go new file mode 100644 index 00000000000..e8562a461ff --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/upload_uris_test.go @@ -0,0 +1,83 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package dataset_api + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// uploadServer answers startPendingUpload with the given body and records every +// path it is asked for, so what the upload did can be read back. +func pendingUploadServer(t *testing.T, pendingBody string) (*DatasetClient, *[]string) { + t.Helper() + + var asked []string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + asked = append(asked, r.Method+" "+r.URL.Path) + w.Header().Set("Content-Type", "application/json") + if strings.HasSuffix(r.URL.Path, "/startPendingUpload") { + _, _ = w.Write([]byte(pendingBody)) + return + } + // assert, not require: this runs on the server's goroutine. + assert.NoError(t, json.NewEncoder(w).Encode(map[string]any{"name": "golden", "version": "1"})) + })) + t.Cleanup(srv.Close) + + pipeline := runtime.NewPipeline("test", "v1", runtime.PipelineOptions{}, + &policy.ClientOptions{Retry: policy.RetryOptions{MaxRetries: -1}}) + return NewDatasetClientFromPipeline(srv.URL, pipeline), &asked +} + +// datasetDir writes one .jsonl for UploadVersion to read. +func oneJSONLDir(t *testing.T) string { + t.Helper() + dir := t.TempDir() + require.NoError(t, os.WriteFile( + filepath.Join(dir, "golden.jsonl"), []byte("{\"query\":\"hi\"}\n"), 0o600)) + return dir +} + +// The SAS says where to write and the blob URI says what to register; they are +// different fields of one response. Only the SAS was checked, so a response +// carrying it without the other uploaded the bytes and then finalized against +// "/golden.jsonl" -- a blob nothing points at, and a publish that failed for a +// reason the message did not name. +func TestAnUploadWithNowhereToRegisterItIsRefusedBeforeTheBlobIsWritten(t *testing.T) { + client, asked := pendingUploadServer(t, `{"blobReference":{"credential":{"sasUri":"https://blob.example.invalid/c?sig=s"}}}`) + + _, err := client.UploadVersion(context.Background(), "golden", "1", oneJSONLDir(t), "2024-01-01") + + require.Error(t, err) + assert.Contains(t, err.Error(), "blob URI") + assert.Contains(t, err.Error(), "startPendingUpload") + + joined := strings.Join(*asked, " ") + assert.Contains(t, joined, "startPendingUpload") + assert.NotContains(t, joined, "versions/1?", + "nothing may be finalized when there is nothing to finalize against") +} + +// The missing-SAS case keeps its own message, because the two are not the same +// problem and the remedies differ. +func TestAnUploadWithNowhereToWriteKeepsItsOwnMessage(t *testing.T) { + client, _ := pendingUploadServer(t, `{"blobReference":{"blobUri":"https://blob.example.invalid/c"}}`) + + _, err := client.UploadVersion(context.Background(), "golden", "1", oneJSONLDir(t), "2024-01-01") + + require.Error(t, err) + assert.Contains(t, err.Error(), "upload SAS URI") +} From cf3a48a2bc9ff74ae975081dc845fd008c852a86 Mon Sep 17 00:00:00 2001 From: Mohamed Hessien Date: Thu, 20 Aug 2026 20:03:13 +0300 Subject: [PATCH 310/320] Publish the dataset version that was declared (#5) --version reached the incrementing path, so 'update --version 1.0' published 2.0, while 'version: 1.0' in configuration published 1.0. One word, two answers, decided by where it was written. It is now the version to publish in both places, matching the same fix in azure.ai.dataset. The flag is offered on create too; it was withheld only because of the old meaning. A version the service already holds is refused rather than stepped past -- the conflict walk exists for a listing that lags, which makes it right for a version the CLI derived and wrong for one an author named. Nothing pinned the old behaviour, so nothing would have caught this. Every existing test called the upload with an empty version; two tests now cover the declared-version path. --- .../azure.ai.evaluations/CHANGELOG.md | 2 + .../internal/cmd/dataset.go | 28 +++++---- .../pkg/dataset_api/upload_version_test.go | 57 +++++++++++++++++++ 3 files changed, 76 insertions(+), 11 deletions(-) diff --git a/cli/azd/extensions/azure.ai.evaluations/CHANGELOG.md b/cli/azd/extensions/azure.ai.evaluations/CHANGELOG.md index 69b395018cf..532ebf558da 100644 --- a/cli/azd/extensions/azure.ai.evaluations/CHANGELOG.md +++ b/cli/azd/extensions/azure.ai.evaluations/CHANGELOG.md @@ -22,6 +22,8 @@ First release of the Foundry evaluations extension. declaration changes. - Atomic commands for every operation: `dataset`, `evaluator`, `run` and `run output` subcommands, all supporting `-o json` and `--no-prompt`. +- `dataset --version` names the version to publish, on `create` and `update` + alike, and means the same thing as `version:` in the configuration. - Testing criteria are shaped from each evaluator's published contract, so evaluators requiring inputs beyond the agent shape — `ground_truth`, `context`, `instruction_id_list` — work by binding them to dataset columns. diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset.go index c79f6cb2ff4..de0c8d03944 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset.go @@ -142,9 +142,21 @@ func newDatasetWriteCommand(verb, short string) *cobra.Command { return err } - ds, err := ec.datasetClient.UploadNextVersion( - ctx, name, version, localSource, ProjectEndpointAPIVersion, - ) + // A declared version is the version to publish, never one to count + // from, so it is written exactly as given. Only an omitted version is + // derived, and only that path walks past a conflict: a version the + // author named and the service already holds is theirs to resolve, + // and stepping past it would publish one they did not ask for. + var ds *dataset_api.Dataset + if version != "" { + ds, err = ec.datasetClient.UploadVersion( + ctx, name, version, localSource, ProjectEndpointAPIVersion, + ) + } else { + ds, err = ec.datasetClient.UploadNextVersion( + ctx, name, "", localSource, ProjectEndpointAPIVersion, + ) + } if err != nil { return messages.RegisteringDataset(name, err) } @@ -170,14 +182,8 @@ func newDatasetWriteCommand(verb, short string) *cobra.Command { cmd.Flags().StringVar(&fromFile, "from-file", "", "Path to a .jsonl file, or a directory containing one.") - // Only on update. create publishes a first version, and the upload derives - // the next version from whatever this holds, so `create --version 4.0` - // would publish 5.0 -- and leave the existence probe, which looks for the - // versions a first publish can carry, unable to find what it wrote. - if verb == "update" { - cmd.Flags().StringVar(&version, "version", "", - "Current version to increment from. Omit to increment from the latest registered version.") - } + cmd.Flags().StringVar(&version, "version", "", + "Version to publish. Omit to publish the next version after the latest registered.") cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") return cmd } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/upload_version_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/upload_version_test.go index 164cc6d415b..e66ecb39457 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/upload_version_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/upload_version_test.go @@ -103,6 +103,63 @@ func TestUploadNextVersionWalksPastAStaleListing(t *testing.T) { "the version just refused is proof it exists, so the next one is tried") } +// A declared version is the version published, not one to count from. +// +// `--version` used to reach the incrementing path, so `--version 7.0` published +// 8.0 while `version: 7.0` in configuration published 7.0 -- one word, two +// answers, decided by where it was written. +func TestUploadVersionPublishesTheVersionDeclared(t *testing.T) { + server := &uploadServer{taken: map[string]bool{}, listing: []string{"1.0", "2.0"}} + httpServer := func() *httptest.Server { + var s *httptest.Server + s = httptest.NewServer(server.handler(t, func() string { return s.URL })) + return s + }() + t.Cleanup(httpServer.Close) + + client := NewDatasetClientFromPipeline( + httpServer.URL, runtime.NewPipeline("test", "v1", runtime.PipelineOptions{}, nil)) + + dir := t.TempDir() + require.NoError(t, os.WriteFile( + filepath.Join(dir, "rows.jsonl"), []byte("{\"query\":\"q\"}\n"), 0o600)) + + ds, err := client.UploadVersion(context.Background(), "ds", "7.0", dir, "2025-11-15-preview") + require.NoError(t, err) + assert.Equal(t, "7.0", ds.Version, "the version asked for is the version written") + assert.Equal(t, []string{"7.0"}, server.attempts, + "a declared version is published as given, not counted from") +} + +// A declared version the service already holds is refused rather than stepped +// past. +// +// The conflict walk exists because the listing lags behind a publish, which +// makes it right for a version the CLI derived. Applying it to one an author +// named would publish a version they did not ask for, and report success. +func TestUploadVersionDoesNotWalkPastAConflict(t *testing.T) { + server := &uploadServer{taken: map[string]bool{"7.0": true}, listing: []string{"7.0"}} + httpServer := func() *httptest.Server { + var s *httptest.Server + s = httptest.NewServer(server.handler(t, func() string { return s.URL })) + return s + }() + t.Cleanup(httpServer.Close) + + client := NewDatasetClientFromPipeline( + httpServer.URL, runtime.NewPipeline("test", "v1", runtime.PipelineOptions{}, nil)) + + dir := t.TempDir() + require.NoError(t, os.WriteFile( + filepath.Join(dir, "rows.jsonl"), []byte("{\"query\":\"q\"}\n"), 0o600)) + + _, err := client.UploadVersion(context.Background(), "ds", "7.0", dir, "2025-11-15-preview") + require.Error(t, err, "the version the author named is taken, and that is theirs to resolve") + assert.True(t, IsVersionConflict(err), "the refusal has to read as a conflict") + assert.Equal(t, []string{"7.0"}, server.attempts, + "nothing beyond the declared version is attempted") +} + // When the listing has caught up and is further ahead than the refused // version, it is the better answer: it skips versions somebody else published. func TestUploadNextVersionPrefersACaughtUpListing(t *testing.T) { From 06282dc9eda0d4b36f6094b75e16b87f640e322f Mon Sep 17 00:00:00 2001 From: Mohamed Hessien Date: Thu, 20 Aug 2026 20:05:21 +0300 Subject: [PATCH 311/320] Declare the azure.ai.eval schema, and make $ref mean one thing (#2) * Declare the azure.ai.eval schema and register it with azd Six Foundry extensions ship a schemas/ directory and are registered in schemas/v1.0/azure.yaml.json by an if/then block on host. The evaluations extension shipped neither, so host: azure.ai.eval was unrecognised: host is a free string with an examples list rather than an enum, so it did not error -- nothing validated the service entry, the $ref, or anything under it. This adds azure.ai.eval.json describing the service body -- the dataset and evaluator catalogs and the evals defined over them -- built from the Go types rather than from the spec, and registers it in both v1.0 and alpha the way the five thin siblings do. Two examples are validated with the ajv recipe from the agents schemas README; both pass. It also pins an asymmetry found while writing this. azd core resolves $ref recursively over the whole service config and the eval service target calls it, so azd up accepts an include anywhere in azure.eval.yaml. Every CLI command instead reads the file straight off disk through a strict decoder, so the same document fails with unknown key "$ref". Resolving it there is not just a matter of calling ResolveFileRefs: azd rebases only project and instructions path values, so a declaration spliced in from another directory would keep a source: that resolves against the wrong base. Which keys an extension may have rebased is the azd team question this work surfaces, and the new test is what should fail when that answer arrives. * Adopt the FileRef model for evaluators and end the source overload Evaluator entries are now either an inline definition or a $ref to a file holding one, which is what agents and projects do and what FileRef means: replace an inline definition with a reference to an external file. The declaration is the definition rather than a pointer to one, so a spliced entry carries no further paths and the rebasing question does not arise. FileRef is cloned locally rather than referenced across extensions, following azure.ai.projects. Nothing references the agents copy from another extension, and a raw-URL $ref would couple this schema to their file layout. datasets[].source becomes datasets[].file. A .jsonl is a data artifact to upload, not a definition to splice, so it keeps a path key -- but source meant three different things in one file (a data path, a definition path, and the inline object saying where an eval reads rows), and only the last is left. The oneOf needs an explicit discriminator: EvaluatorDecl carries the service-owned rubric keys so additionalProperties has to stay true, which means it would otherwise also match a FileRef. ajv caught that; not: {required: [$ref]} settles it. Both examples validate. * Rename the dataset catalog key from source: to file: A dataset entry points at a .jsonl whose rows are uploaded. It is an artifact to publish, not a definition, which is why it is not a $ref -- and calling it source: made that harder to see, because source: also named an evaluator file and, on an eval, the inline object saying where its rows come from. Three meanings for one key, in one file. Only the eval one is left, which is what the surrounding prose already claimed the key meant. The near-miss suggester works on edit distance 1, so the typo fixture moves from sourse/source to fil/file; a transposition like fiel is distance 2 and would have made the test assert a suggestion the code does not offer. * Bring the README example back in line with the config it documents * Make the schema describe the config the CLI can actually load Three independent reviews converged on the same thing: the schema had been written for the FileRef model before the Go code moved, so it described evaluator entries that DecodeEvalConfig rejects. A schema that is ahead of its code is worse than no schema -- an editor accepts a document the CLI then refuses. Evaluators go back to {name, source, version} and the FileRef definition comes out until the migration lands. The numeric bounds were also stricter than the code. minimum: 1 rejected the zero that every one of these fields treats as unset: runnable.go rejects only MaxSamples < 0, trace_window.go only MaxTurns < 0 and LookbackHours < 0. lookback_hours gains the upper bound the code does enforce (project.MaxLookbackHours, ten years). max_samples deliberately gets no upper bound. MinSampleSize/MaxSampleSize gate the generate flag, not the config field, which runnable.go bounds only below -- stating 15..1000 here would make the schema stricter than the code in the other direction. The hero assertion was checking for the old key, which made it vacuous: it asserted a registered dataset writes no "source:" in a file that no longer contains that word for any reason. It checks "file:" now. All of tests/ is behind live/hero build tags, so go test ./... never compiled it; both tags now vet clean. * Resolve $ref on the CLI path, not just on deploy Core owns the resolver but does not run it before dispatch -- it hands each extension the entry with $ref still in it. The service target has always called foundry.ResolveFileRefs and this path never did, so an include deployed cleanly and then failed every azd ai eval command with unknown key "$ref". One file meaning two different things depending on which command opened it. LoadEvalConfig now resolves against the configuration's own directory, which is the base its relative paths are already documented to use, and strips the directive before the strict decode. A configuration with no $ref is handed to the decoder untouched rather than round-tripped through a map. The round trip would cost the decoder its line numbers, and a mistyped key reported at the wrong line is worse than the resolution is worth for the case that does not need it. The old test pinned the asymmetry as known-broken; it now covers the invariant that survives -- a directive reaching the strict decoder means resolution was skipped, and naming it is still the right answer. Renamed to say that. Reported by the owner of $ref in azd. The dataset file: half of the same review is already handled: serviceRelativeDir recovers the base from the $ref value on deploy, and on this path the base is the file's own directory. * Read azure.yaml's $ref, not only write it The cascade for where the configuration lives was --path, then the path init recorded in the azd environment, then ./evals. The $ref on the azure.ai.eval service was written by init and never read back, so azure.yaml and the CLI were two answers to one question. The recorded level hid it on the machine that ran init. An azd environment is not in the repository, though: check the project out somewhere else -- a colleague, a build agent -- and level two is empty, so a configuration the project declares under ./config is reported missing by every command while azd up deploys it from the same $ref. The old comment on this function already described the symptom: "while azure.yaml's $ref pointed at it correctly the whole time." The declaration now sits between the recorded path and the default. It ranks below the recorded one because that is what --path wrote on this machine, and above the default because a project that declares where its configuration lives should be believed over a convention. Best-effort by design: outside an azd project there is no azure.yaml to read, which is ordinary rather than a failure, unlike a recorded path that could not be read. From the $ref owner's review of what the agents extension does correctly -- resolving both root and nested references. Nested was already handled by the shared resolver; the root reference was the half that was only ever written. * Honour the file azure.yaml's $ref names, not just its directory Reading the declaration was half a fix. The $ref names a file -- checkOneConfig's own comment says so, "azure.yaml $refs a single file by name" -- and taking only its directory then looked for azure.eval.yaml beside it. A project declaring ./config/nightly.yaml was still reported missing by every CLI command while azd up deployed it from the same $ref. A configuration location is now the directory or the file itself. EvalConfigPath returns a location that names a file unchanged and applies the naming convention to one that names a directory; EvalDirOf gives the directory relative paths resolve against. A location that does not exist yet reads as a directory, which is what init is handed before it writes anything. The both-names guard only applies to a directory: a location that already names the file is the project saying which one it means, so there is nothing to disambiguate. Artifact directories now derive from EvalDirOf rather than the location, so init scaffolds datasets/ and evaluators/ beside a declared file instead of inside it. * Repoint azure.yaml when init moves the configuration ensureServiceWired matched on name and host alone, so a service pointing at the old file counted as correctly wired. init --path ./quality wrote the configuration to one place and left azure.yaml deploying another. That was inert while nothing read the entry. It stopped being inert when the directory cascade started reading it: the stale value now answers on any machine that did not run this init, because the recorded path lives in the azd environment and an azd environment does not travel with the repository. So this is a hazard the cascade change created out of a latent bug, and it belongs with it. AddService reloads from disk and replaces the entry, preserving the event dispatcher and the ${VAR} templates already in the file, so writing it again is how it gets repointed. The caller is told: a repoint changes what azd up deploys and should not happen quietly. * Keep $ref intact when a command saves the configuration Resolving on the CLI read path introduced a quiet regression: init and generate read, modify and save the same file, so what they saved was the resolved document. The author's include was replaced by its content, the file it named was orphaned, and the paths inside that file were left resolving against the wrong directory -- a source: ./quality.json written beside evaluators/quality.yaml came back pointing at the project root. Nothing reported any of it, because from the writer's point of view it had saved what it read. Reading for use and reading for edit are now separate. OpenEvalConfig resolves; OpenEvalConfigForEdit hands the file to the decoder as written, and the two commands that save use it. That needs $ref to survive the strict decoder, so EvaluatorDecl models it. Name becomes omitempty for the same reason: an entry that is only a $ref has no name until the file it names supplies one. The old test asserted the decoder refuses a directive, on the reasoning that one reaching it meant resolution had been skipped. That only held while every reader resolved. It now covers what replaced it -- the directive decodes and round trips -- and the schema's oneOf: [EvaluatorDecl, FileRef] goes back, since the code accepts what it describes. EvaluatorDecl is closed, so a $ref entry matches the FileRef branch alone and the oneOf needs no discriminator this time. Both examples validate with the branch exercised. * Let a $ref name the rubric, not just a pointer to one An evaluator's rubric could only be named as a file. A `` splices the referenced file's keys into the entry, so a `` to a rubric produced `unknown key dimensions` -- the shape the spec documents did not load. The rubric now rides on the declaration under `definition`, which is one of the two shapes the publish path already accepted. It is a named key rather than a catch-all on purpose: a catch-all would absorb every misspelling in the entry and publish it to the service as rubric content. Naming a rubric and writing one out are rejected together rather than one silently winning, and a definition is fingerprinted like a file so inline rubrics get the change detection they lacked. * Make the README example one the CLI can load It documented `options.eval_model`, which is not a key this decoder has ever had, so following the README produced `unknown key options`. It also named the config `azure.yaml` rather than `azure.eval.yaml`. Correcting the keys alone would only reset the clock on the same drift, so the example is now decoded by a test. Confirmed it fails on the old content rather than passing vacuously. * Let a $ref name the bare rubric file the tool writes The spec's example points a `` at a rubric downloaded by `generate`, which is a bare `{type, dimensions}`. Resolution splices those keys into the entry, so they landed beside `name` and were rejected: the documented example did not load. They are moved under `definition` instead. Wrapping the file would have been the smaller change and the wrong one -- the tool writes that file, so the config has to read what the tool writes. Scoped to entries the author wrote as a ``, noted before resolution removes the directive. Doing it for every entry would be a catch-all by another name, filing a misspelling as rubric content and publishing it to the service instead of reporting it. * Resolve a refd rubric on the deploy path too EvalConfigFromService resolves includes itself rather than going through LoadEvalConfig, so the rescue landed on the CLI path only. That recreated the exact asymmetry this work started from, in mirror image: an include every CLI command accepted and 'azd up' refused with unknown key 'dimensions'. Both routes now agree, and a test reads the same include each way and compares. Confirmed it fails without the deploy-path call rather than passing vacuously. * Find a spliced rubric by shape, not by position The rescue marked evaluator entries by index in the pre-resolution document. That cannot see the evaluators inside a config that is itself behind a ref -- which is the layout the README and the spec both document, so the documented shape still failed to deploy with unknown key 'dimensions'. It now keys on the entry carrying 'dimensions', the same key normalizeRubricBody insists on before it will treat a document as a definition. That drops the pre-scan entirely and works at any nesting depth. The gate is what separates this from a catch-all: a ref to something that is not a rubric is still reported rather than filed away as publishable content. Confirmed by removing the gate and watching that test go green. * Pin that a YAML anchor survives ref resolution Resolving a ref round-trips the document through a map, which expands aliases. An alias that expanded to nothing would silently drop the judge model shared across evaluators rather than fail, so the test asserts the parameters are equal and non-empty. * Resolve a ref in one place instead of two The CLI and 'azd up' reach a configuration by different routes and each resolved for itself, so every ref rule had to be added twice. Twice it was not: an include azd up accepted and every CLI command refused, and later the reverse. Everything between obtaining the map and using it now lives in resolveEvalRefs. A test counts the callers of ResolveFileRefs, because a second one is how the split comes back and it comes back silently. * Carry the dataset key rename into the merged tests The base added scored_version_test.go against 'source:' while this branch renamed the dataset key to 'file:'. The merge was textually clean and semantically not: writeCatalog wrote a key the model no longer knows. Worth noting for the owner: the config failing to parse did not surface. localDatasetPath read it as 'not local' and the run was labelled with the recorded version -- the false provenance that test exists to prevent. --- .../extensions/azure.ai.evaluations/README.md | 19 +- .../internal/cmd/catalog.go | 10 +- .../internal/cmd/context.go | 83 +++++- .../internal/cmd/eval_group.go | 2 +- .../internal/cmd/evaldir_test.go | 58 +++- .../internal/cmd/generate_commands.go | 2 +- .../azure.ai.evaluations/internal/cmd/init.go | 17 +- .../internal/cmd/init_test.go | 6 +- .../internal/cmd/reconciler.go | 64 +++-- .../azure.ai.evaluations/internal/cmd/run.go | 8 +- .../internal/cmd/run_datasource_test.go | 2 +- .../internal/cmd/scored_version_test.go | 16 +- .../internal/cmd/wiring_ref_test.go | 59 +++++ .../internal/messages/messages.go | 21 ++ .../internal/project/anchor_ref_test.go | 54 ++++ .../internal/project/config_keys_test.go | 13 +- .../internal/project/config_location_test.go | 78 ++++++ .../internal/project/edit_read_test.go | 93 +++++++ .../internal/project/eval_config.go | 39 ++- .../project/eval_config_ambiguity_test.go | 6 +- .../project/eval_config_atomic_test.go | 2 +- .../internal/project/eval_config_keys_test.go | 2 +- .../internal/project/eval_config_name_test.go | 4 +- .../project/eval_config_roundtrip_test.go | 4 +- .../internal/project/eval_config_store.go | 217 +++++++++++++-- .../internal/project/eval_config_test.go | 22 +- .../internal/project/max_samples_test.go | 4 +- .../project/nested_ref_rubric_test.go | 74 ++++++ .../internal/project/one_resolver_test.go | 54 ++++ .../internal/project/readfile_test.go | 2 +- .../internal/project/readme_example_test.go | 53 ++++ .../internal/project/ref_directive_test.go | 52 ++++ .../internal/project/ref_resolution_test.go | 232 ++++++++++++++++ .../project/service_config_strict_test.go | 2 +- .../internal/project/service_target_eval.go | 16 +- .../project/service_target_eval_test.go | 2 +- .../project/unknown_keys_depth_test.go | 6 +- .../schemas/azure.ai.eval.json | 249 ++++++++++++++++++ .../schemas/examples/inline.azure.yaml | 58 ++++ .../schemas/examples/ref.azure.yaml | 10 + .../tests/hero/init_test.go | 2 +- schemas/alpha/azure.yaml.json | 23 +- schemas/v1.0/azure.yaml.json | 23 +- 43 files changed, 1624 insertions(+), 139 deletions(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/wiring_ref_test.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/project/anchor_ref_test.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/project/config_location_test.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/project/edit_read_test.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/project/nested_ref_rubric_test.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/project/one_resolver_test.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/project/readme_example_test.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/project/ref_directive_test.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/project/ref_resolution_test.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/schemas/azure.ai.eval.json create mode 100644 cli/azd/extensions/azure.ai.evaluations/schemas/examples/inline.azure.yaml create mode 100644 cli/azd/extensions/azure.ai.evaluations/schemas/examples/ref.azure.yaml diff --git a/cli/azd/extensions/azure.ai.evaluations/README.md b/cli/azd/extensions/azure.ai.evaluations/README.md index 148b7fbb711..dcdde896504 100644 --- a/cli/azd/extensions/azure.ai.evaluations/README.md +++ b/cli/azd/extensions/azure.ai.evaluations/README.md @@ -23,30 +23,31 @@ services: evals: host: azure.ai.eval uses: [ai-project] - $ref: ./evals/azure.yaml + $ref: ./evals/azure.eval.yaml ``` ```yaml -# evals/azure.yaml +# evals/azure.eval.yaml datasets: - name: support-golden - source: ./datasets/support-golden.jsonl + file: ./datasets/support-golden.jsonl evaluators: - name: support-quality source: ./evaluators/support-quality.json -evalGroups: +evals: - name: support-quality dataset: support-golden + evaluation_level: turn evaluators: - - builtin.task_adherence - - support-quality + - evaluator: builtin.task_adherence + initialization_parameters: + model: gpt-4.1-nano + - evaluator: support-quality target: type: agent name: support-agent - options: - eval_model: gpt-4.1-nano ``` `azd up` reconciles **datasets → evaluators → eval groups**, in that order, @@ -64,7 +65,7 @@ keys you authored — the service adds `data_schema`, `init_parameters` and `metrics` of its own. Eval groups are immutable, so a change to a group's evaluators, target or -options creates a new group and a new id. The id is cached in the azd +sampling creates a new group and a new id. The id is cached in the azd environment so repeat runs stay comparable. ## Commands diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/catalog.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/catalog.go index e92a76321f2..0b8acd61887 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/catalog.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/catalog.go @@ -28,16 +28,16 @@ func addDatasetToCatalog(cmd *cobra.Command, evalDir string, ref *project.Artifa if cfg.Datasets[i].Name == ref.Name { // Regeneration overwrites the file in place, so the entry only // changes when the artifact moved. - if cfg.Datasets[i].Source == ref.Source { + if cfg.Datasets[i].File == ref.Source { return false } - cfg.Datasets[i].Source = ref.Source + cfg.Datasets[i].File = ref.Source return true } } cfg.Datasets = append(cfg.Datasets, project.DatasetDecl{ - Name: ref.Name, - Source: ref.Source, + Name: ref.Name, + File: ref.Source, }) return true }) @@ -88,7 +88,7 @@ func updateCatalog( } defer unlock() - cfg, err := project.OpenEvalConfig(evalDir) + cfg, err := project.OpenEvalConfigForEdit(evalDir) if err != nil { return err } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/context.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/context.go index b6256a30565..f35d4b1e0c2 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/context.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/context.go @@ -9,6 +9,7 @@ import ( "fmt" "log" "os" + "path/filepath" "strings" "azureaieval/internal/foundry/projectctx" @@ -417,12 +418,21 @@ const ( // // 1. --path // 2. the path `init` recorded in the azd environment -// 3. ./evals +// 3. the `$ref` on the `azure.ai.eval` service in azure.yaml +// 4. ./evals // -// The middle level is what stops `--path` from having to be repeated on every -// later command. Without it, `init --path ./quality` wrote a configuration that -// `run` then looked for under ./evals and reported as missing -- while -// azure.yaml's $ref pointed at it correctly the whole time. +// The middle levels are what stop `--path` from having to be repeated on every +// later command. Without the recorded one, `init --path ./quality` wrote a +// configuration that `run` then looked for under ./evals and reported as +// missing -- while azure.yaml's $ref pointed at it correctly the whole time. +// +// That $ref is now read rather than only written, which is what makes the rule +// survive a fresh clone. The recorded path lives in the azd environment, and an +// azd environment is not in the repository: check the project out somewhere +// else and level 2 is empty, so a configuration the project declares perfectly +// well under ./config was reported missing by every command while `azd up` +// deployed it. Reading the declaration is also what keeps one answer to "where +// is the configuration" instead of one for deploy and one for everything else. // // This is the whole rule, and every command that reads the configuration goes // through it. Stating it here and applying it on only some paths is how @@ -432,10 +442,17 @@ const ( // // recorded tells absence apart from failure, and the two get different // answers. A project with no azd environment has genuinely recorded nothing, -// so ./evals is right. An azd that could not be asked has said nothing at all, -// and defaulting on that would write the second configuration all over again -- -// this time for a reason nobody could reproduce. -func evalDirCascade(flagValue string, recorded func() (string, error)) (string, error) { +// so the next level is right. An azd that could not be asked has said nothing +// at all, and defaulting on that would write the second configuration all over +// again -- this time for a reason nobody could reproduce. +// +// declared is best-effort by contrast: outside an azd project there is no +// azure.yaml to read, which is ordinary rather than a failure. +func evalDirCascade( + flagValue string, + recorded func() (string, error), + declared func() string, +) (string, error) { if flagValue != "" { return flagValue, nil } @@ -446,9 +463,48 @@ func evalDirCascade(flagValue string, recorded func() (string, error)) (string, if path != "" { return path, nil } + if declared != nil { + if dir := declared(); dir != "" { + return dir, nil + } + } return project.DefaultEvalDir, nil } +// declaredEvalConfig reads the location azure.yaml's `$ref` points at. +// +// The service entry is the project's own statement of where its evaluation +// configuration lives, and `azd up` has always deployed from it. The full path +// is returned rather than its directory: the `$ref` names a file, and a project +// declaring `./config/nightly.yaml` means that file, not whatever +// `azure.eval.yaml` happens to sit beside it. +// +// Returns empty outside an azd project, or when nothing declares the eval host. +func declaredEvalConfig(ctx context.Context, azdClient *azdext.AzdClient) string { + if azdClient == nil { + return "" + } + resp, err := azdClient.Project().Get(ctx, &azdext.EmptyRequest{}) + if err != nil || resp.GetProject() == nil { + return "" + } + for _, svc := range resp.GetProject().GetServices() { + if svc.GetHost() != project.EvalHost { + continue + } + props := svc.GetAdditionalProperties() + if props == nil { + continue + } + ref, _ := props.AsMap()["$ref"].(string) + if ref == "" { + continue + } + return filepath.Clean(filepath.FromSlash(ref)) + } + return "" +} + // evalDir is the cascade for a command that already holds an azd connection. func (ec *evalContext) evalDir(ctx context.Context, flagValue string) (string, error) { return evalDirCascade(flagValue, func() (string, error) { @@ -456,6 +512,8 @@ func (ec *evalContext) evalDir(ctx context.Context, flagValue string) (string, e return "", nil } return readRecordedEvalPath(ctx, ec.azdClient, ec.envName) + }, func() string { + return declaredEvalConfig(ctx, ec.azdClient) }) } @@ -494,6 +552,13 @@ func resolveEvalDir(ctx context.Context, flagValue string) (string, error) { return "", nil } return readRecordedEvalPath(ctx, azdClient, env.GetEnvironment().GetName()) + }, func() string { + azdClient, err := azdext.NewAzdClient() + if err != nil { + return "" + } + defer azdClient.Close() + return declaredEvalConfig(ctx, azdClient) }) } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/eval_group.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/eval_group.go index 75c5f717e36..8298b0801ff 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/eval_group.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/eval_group.go @@ -74,7 +74,7 @@ func newEvalCreateCommand() *cobra.Command { baseDir := filepath.Dir(path) datasetPath := "" if decl, ok := cfg.DatasetDeclaration(eval.Dataset); ok { - datasetPath = project.ResolveSource(baseDir, decl.Source) + datasetPath = project.ResolveSource(baseDir, decl.File) } reconciler := &evalReconciler{ec: ec} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaldir_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaldir_test.go index 14f32ba4d8d..a50de3a8a33 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaldir_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaldir_test.go @@ -82,20 +82,68 @@ func TestEvalDirCascadeAnswersInOrder(t *testing.T) { t.Run(tc.name, func(t *testing.T) { got, err := evalDirCascade(tc.flag, func() (string, error) { return tc.recorded, nil - }) + }, nil) require.NoError(t, err) assert.Equal(t, tc.want, got) }) } } +// azure.yaml's `$ref` is read, not only written. +// +// The recorded path lives in the azd environment, and an azd environment is not +// in the repository. Check the project out somewhere else and that level is +// empty, so a configuration the project declares under ./config was reported +// missing by every command while `azd up` deployed it from the same `$ref`. +func TestEvalDirCascadeReadsTheDeclaredRef(t *testing.T) { + got, err := evalDirCascade("", + func() (string, error) { return "", nil }, + func() string { return "config" }) + + require.NoError(t, err) + assert.Equal(t, "config", got) +} + +// The recorded path is what `--path` wrote on this machine, so it answers over +// a declaration that may predate it. +func TestEvalDirCascadePrefersTheRecordedPathOverTheDeclaredRef(t *testing.T) { + got, err := evalDirCascade("", + func() (string, error) { return "quality", nil }, + func() string { return "config" }) + + require.NoError(t, err) + assert.Equal(t, "quality", got) +} + +// Outside an azd project there is no azure.yaml to read, which is ordinary. +func TestEvalDirCascadeFallsBackWhenNothingIsDeclared(t *testing.T) { + got, err := evalDirCascade("", + func() (string, error) { return "", nil }, + func() string { return "" }) + + require.NoError(t, err) + assert.Equal(t, project.DefaultEvalDir, got) +} + +// A --path that was given is the answer on its own, so neither level is asked. +func TestEvalDirCascadeSkipsBothLookupsWhenPathWasGiven(t *testing.T) { + var declaredAsked int + got, err := evalDirCascade("./given", + func() (string, error) { return "recorded", nil }, + func() string { declaredAsked++; return "config" }) + + require.NoError(t, err) + assert.Equal(t, "./given", got) + assert.Equal(t, 0, declaredAsked, "a --path that was given should not cost a round trip") +} + // A read that failed is not a project that recorded nothing. Defaulting on it // is how `generate` would write a second configuration under ./evals for a // reason nobody could reproduce, so the failure has to come back out. func TestEvalDirCascadeDoesNotDefaultOnAFailedRead(t *testing.T) { boom := errors.New("the environment could not be read") - got, err := evalDirCascade("", func() (string, error) { return "", boom }) + got, err := evalDirCascade("", func() (string, error) { return "", boom }, nil) require.ErrorIs(t, err, boom) assert.Empty(t, got, "a failed read must not answer with the default") @@ -106,7 +154,7 @@ func TestEvalDirCascadeDoesNotDefaultOnAFailedRead(t *testing.T) { func TestEvalDirCascadeIgnoresAFailedReadWhenPathWasGiven(t *testing.T) { got, err := evalDirCascade("./given", func() (string, error) { return "", errors.New("the environment could not be read") - }) + }, nil) require.NoError(t, err) assert.Equal(t, "./given", got) @@ -118,7 +166,7 @@ func TestEvalDirCascadeAsksForTheRecordedPathOnce(t *testing.T) { got, err := evalDirCascade("", func() (string, error) { asked++ return "", nil - }) + }, nil) require.NoError(t, err) assert.Equal(t, project.DefaultEvalDir, got) @@ -128,7 +176,7 @@ func TestEvalDirCascadeAsksForTheRecordedPathOnce(t *testing.T) { _, err = evalDirCascade("./given", func() (string, error) { asked++ return "", nil - }) + }, nil) require.NoError(t, err) assert.Equal(t, 0, asked, "a --path that was given should not cost a round trip") } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_commands.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_commands.go index 1a1b90d6fc9..6cdf390e2fe 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_commands.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_commands.go @@ -90,7 +90,7 @@ func resolvePlan(f *generateFlags, name string, defaultOutputDir string) (genera Agent: firstNonEmpty(f.target, declaredTarget(f.path)), Model: f.model, Instruction: instruction, - BaseDir: f.path, + BaseDir: project.EvalDirOf(f.path), OutputDir: firstNonEmpty(f.outputDir, "./"+defaultOutputDir), } if plan.Model == "" && plan.Agent == "" { diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go index e025ee1f69a..8d5690e74ee 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go @@ -137,7 +137,7 @@ func newInitCommand() *cobra.Command { // reporting it as created would claim a file it only added to. _, configExistedErr := os.Stat(configPath) configExisted := configExistedErr == nil - cfg, err := project.OpenEvalConfig(path) + cfg, err := project.OpenEvalConfigForEdit(path) if err != nil { return err } @@ -181,7 +181,7 @@ func newInitCommand() *cobra.Command { } defer unlockConfig() - cfg, err = project.OpenEvalConfig(path) + cfg, err = project.OpenEvalConfigForEdit(path) if err != nil { return err } @@ -196,10 +196,13 @@ func newInitCommand() *cobra.Command { cfg.RemoveEval(evalName) } - if err := os.MkdirAll(filepath.Join(path, project.DefaultDatasetsDir), 0o750); err != nil { + // The location may be the file azure.yaml names rather than the + // directory holding it, and artifacts sit beside the configuration. + evalDir := project.EvalDirOf(path) + if err := os.MkdirAll(filepath.Join(evalDir, project.DefaultDatasetsDir), 0o750); err != nil { return messages.CreatingDatasetsDir(err) } - if err := os.MkdirAll(filepath.Join(path, project.DefaultEvaluatorsDir), 0o750); err != nil { + if err := os.MkdirAll(filepath.Join(evalDir, project.DefaultEvaluatorsDir), 0o750); err != nil { return messages.CreatingEvaluatorsDir(err) } @@ -236,8 +239,8 @@ func newInitCommand() *cobra.Command { "eval": evalName, "evalConfig": configPath, "service": serviceName, - "datasetsDir": filepath.Join(path, project.DefaultDatasetsDir), - "evaluatorsDir": filepath.Join(path, project.DefaultEvaluatorsDir), + "datasetsDir": filepath.Join(evalDir, project.DefaultDatasetsDir), + "evaluatorsDir": filepath.Join(evalDir, project.DefaultEvaluatorsDir), "rootConfig": rootWiring, "target": target, "source": source, @@ -401,7 +404,7 @@ func planScaffold(in scaffoldInput) scaffold { } eval.Dataset = datasetName out.datasetName = datasetName - addDatasetDecl(cfg, project.DatasetDecl{Name: datasetName, Source: datasetSource}) + addDatasetDecl(cfg, project.DatasetDecl{Name: datasetName, File: datasetSource}) } // Every evaluator carries the judge deployment, because that is where the diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_test.go index 7eca43c1e76..8a6415de585 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_test.go @@ -370,7 +370,7 @@ func TestScaffold_DatasetReferenceForms(t *testing.T) { }) decl, ok := cfg.DatasetDeclaration("golden") require.True(t, ok) - require.Equal(t, "../tests/golden.jsonl", decl.Source, + require.Equal(t, "../tests/golden.jsonl", decl.File, "a dataset outside the eval dir must be reached with ..") require.Equal(t, "golden", plan.eval.Dataset) require.False(t, plan.generateDataset, @@ -383,7 +383,7 @@ func TestScaffold_DatasetReferenceForms(t *testing.T) { }) decl, ok := cfg.DatasetDeclaration("prod-sample") require.True(t, ok) - require.Empty(t, decl.Source, "a registered dataset must not get a local source") + require.Empty(t, decl.File, "a registered dataset must not get a local source") require.Equal(t, "prod-sample", plan.eval.Dataset) require.False(t, plan.generateDataset) }) @@ -396,7 +396,7 @@ func TestScaffold_DatasetReferenceForms(t *testing.T) { "the dataset is named after the eval") decl, ok := cfg.DatasetDeclaration("support-agent-smoke") require.True(t, ok) - require.Contains(t, decl.Source, "support-agent-smoke.jsonl") + require.Contains(t, decl.File, "support-agent-smoke.jsonl") require.True(t, plan.generateDataset) }) } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go index c06f3db7f52..705b15f07dc 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go @@ -361,18 +361,32 @@ func (r *evalReconciler) latestDatasetVersion(ctx context.Context, name string) // EnsureEvaluator publishes a new version when the local definition differs // from what the service holds. // -// The two kinds of evaluator are told apart by the source's extension: `.py` -// is code, anything else is a rubric. They also detect change differently. A -// rubric definition comes back inline, so it is compared directly; a code -// definition's source is not read back in a form worth comparing, so a -// fingerprint of the script is kept in the azd environment, the same way -// datasets work. +// A definition reaches this three ways: written in the configuration, named as +// a file, or neither -- in which case the evaluator has to already exist on the +// service. The first two are the same publish once the rubric is in hand; they +// differ only in what there is to hash. func (r *evalReconciler) EnsureEvaluator( ctx context.Context, decl project.EvaluatorDecl, localPath string, ) (string, bool, error) { - if localPath == "" { + var body json.RawMessage + var digest string + + switch { + case decl.Definition != nil: + // Also how a `$ref` to a rubric file arrives: resolution has already + // spliced the file's keys in, so there is nothing left to read. + raw, err := json.Marshal(decl.Definition) + if err != nil { + return "", false, messages.EvaluatorProblem(decl.Name, err) + } + if body, err = normalizeRubricBody(decl.Name, raw); err != nil { + return "", false, messages.EvaluatorProblem(decl.Name, err) + } + digest = project.FingerprintBytes(body) + + case localPath == "": raw, err := r.ec.evalClient.GetEvaluatorRaw( ctx, decl.Name, decl.Version, ProjectEndpointAPIVersion, ) @@ -380,34 +394,34 @@ func (r *evalReconciler) EnsureEvaluator( return "", false, messages.EvaluatorNotLocalNorFound(decl.Name, err) } return versionFromRaw(raw, decl.Version), false, nil - } - if _, err := os.Stat(localPath); err != nil { - if errors.Is(err, fs.ErrNotExist) { - return "", false, messages.EvaluatorNotGeneratedYet(decl.Name, localPath) + default: + if _, err := os.Stat(localPath); err != nil { + if errors.Is(err, fs.ErrNotExist) { + return "", false, messages.EvaluatorNotGeneratedYet(decl.Name, localPath) + } + return "", false, messages.EvaluatorSource(localPath, err) } - return "", false, messages.EvaluatorSource(localPath, err) - } - raw, err := project.ReadFileNoBOM(localPath) - if err != nil { - return "", false, messages.EvaluatorSource(localPath, err) - } + raw, err := project.ReadFileNoBOM(localPath) + if err != nil { + return "", false, messages.EvaluatorSource(localPath, err) + } - body, err := normalizeRubricBody(decl.Name, raw) - if err != nil { - return "", false, messages.EvaluatorProblem(decl.Name, err) + if body, err = normalizeRubricBody(decl.Name, raw); err != nil { + return "", false, messages.EvaluatorProblem(decl.Name, err) + } + + if digest, err = project.Fingerprint(localPath); err != nil { + return "", false, messages.EvaluatorSource(localPath, err) + } } - // The author's own file decides whether there is anything to publish. + // The author's own definition decides whether there is anything to publish. // Comparing against the service cannot: it enriches a definition with // fields nobody authored, so sameDefinition only looks for authored keys on // the service and a key the author *deleted* — a pass_threshold, say — is // still there to be found, and the deletion never publishes. - digest, err := project.Fingerprint(localPath) - if err != nil { - return "", false, messages.EvaluatorSource(localPath, err) - } digestKey := project.FingerprintKey("evaluator", decl.Name) prior := r.ec.getEnvValue(ctx, digestKey) authorEdited := prior != "" && prior != digest diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go index a5b4f3e5d91..ec9e9fb882a 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go @@ -703,13 +703,13 @@ func localDatasetPath(configPath string, group *project.Eval) string { return "" } decl, ok := cfg.DatasetDeclaration(group.Dataset) - if !ok || decl.Source == "" { + if !ok || decl.File == "" { return "" } - if filepath.IsAbs(decl.Source) { - return decl.Source + if filepath.IsAbs(decl.File) { + return decl.File } - return filepath.Join(filepath.Dir(configPath), decl.Source) + return filepath.Join(filepath.Dir(configPath), decl.File) } // declaredDatasetVersion is the `version:` the catalog pins this dataset to. diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_datasource_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_datasource_test.go index 76948cb9fb8..426bbf58cd3 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_datasource_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_datasource_test.go @@ -26,7 +26,7 @@ func writeDataset(t *testing.T, rows string) string { require.NoError(t, os.MkdirAll(filepath.Join(dir, "datasets"), 0o750)) require.NoError(t, os.WriteFile(filepath.Join(dir, "datasets", "d.jsonl"), []byte(rows), 0o600)) configPath := filepath.Join(dir, "eval.yaml") - config := "datasets:\n - name: d\n source: ./datasets/d.jsonl\n" + config := "datasets:\n - name: d\n file: ./datasets/d.jsonl\n" require.NoError(t, os.WriteFile(configPath, []byte(config), 0o600)) return configPath } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/scored_version_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/scored_version_test.go index 5402a356c38..9f0c6eec3da 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/scored_version_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/scored_version_test.go @@ -16,16 +16,16 @@ import ( "github.com/stretchr/testify/require" ) -// writeCatalog writes a configuration whose dataset carries the given source -// and version, either of which may be empty. -func writeCatalog(t *testing.T, source, version string) string { +// writeCatalog writes a configuration whose dataset carries the given file and +// version, either of which may be empty. +func writeCatalog(t *testing.T, file, version string) string { t.Helper() dir := t.TempDir() entry := " - name: golden" - if source != "" { - require.NoError(t, os.WriteFile(filepath.Join(dir, source), []byte("{\"query\":\"hi\"}\n"), 0o600)) - entry += "\n source: ./" + source + if file != "" { + require.NoError(t, os.WriteFile(filepath.Join(dir, file), []byte("{\"query\":\"hi\"}\n"), 0o600)) + entry += "\n file: ./" + file } if version != "" { entry += "\n version: \"" + version + "\"" @@ -44,7 +44,7 @@ evals: // The label a run carries has to follow the same branch its rows did. // -// A declaration with both `source:` and `version:` reads the file from disk, so +// A declaration with both `file:` and `version:` reads the file from disk, so // the pin says nothing about what was scored: the recorded version is the one // this file's content published, and checkDatasetRegistered has already // confirmed the rows match it. @@ -64,7 +64,7 @@ func TestARunOverALocalFileIsLabelledWithWhatTheFilePublished(t *testing.T) { } // And with no fingerprint there is nothing tying the file to that version. A -// dataset that was registered and has since gained a `source:` has a recorded +// dataset that was registered and has since gained a `file:` has a recorded // version and no fingerprint, and the rows are whatever the file now holds. func TestALocalFileWithNoFingerprintIsLabelledWithNothing(t *testing.T) { env := &testEnvServer{values: map[string]string{ diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/wiring_ref_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/wiring_ref_test.go new file mode 100644 index 00000000000..945766afcc6 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/wiring_ref_test.go @@ -0,0 +1,59 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "testing" + + "azureaieval/internal/messages" + + "github.com/azure/azure-dev/cli/azd/pkg/azdext" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/types/known/structpb" +) + +func serviceWithRef(t *testing.T, ref string) *azdext.ServiceConfig { + t.Helper() + if ref == "" { + return &azdext.ServiceConfig{Name: "evals"} + } + props, err := structpb.NewStruct(map[string]any{"$ref": ref}) + require.NoError(t, err) + return &azdext.ServiceConfig{Name: "evals", AdditionalProperties: props} +} + +// The `$ref` an entry already carries decides whether the wiring is present, +// and it is compared as a path rather than as text. +// +// Matching on name and host alone reported the wiring present after +// `init --path` moved the configuration, and `azd up` went on deploying the +// file left behind. Comparing the text alone would have called +// `evals/azure.eval.yaml` and `./evals/azure.eval.yaml` two different answers. +func TestServiceRefIsComparedAsAPath(t *testing.T) { + assert.True(t, sameRefTarget("./evals/azure.eval.yaml", "evals/azure.eval.yaml"), + "the same file written two ways is one answer") + assert.True(t, sameRefTarget("evals/../evals/azure.eval.yaml", "./evals/azure.eval.yaml")) + assert.False(t, sameRefTarget("./evals/azure.eval.yaml", "./quality/azure.eval.yaml"), + "a different file is what the guard exists to catch") +} + +// An entry with no `$ref` has nothing to disagree with. +func TestServiceConfigRefReadsTheDeclaredValue(t *testing.T) { + assert.Equal(t, "./evals/azure.eval.yaml", + serviceConfigRef(serviceWithRef(t, "./evals/azure.eval.yaml"))) + assert.Empty(t, serviceConfigRef(serviceWithRef(t, ""))) +} + +// The refusal names both paths, because the reader is the one who has to decide +// which of the two configurations they meant to keep. +func TestServiceRefConflictNamesBothPaths(t *testing.T) { + err := messages.ServiceRefPointsElsewhere( + "support-agent-evals", "./evals/azure.eval.yaml", "./quality/azure.eval.yaml") + + require.Error(t, err) + assert.Contains(t, err.Error(), "./evals/azure.eval.yaml") + assert.Contains(t, err.Error(), "./quality/azure.eval.yaml") + assert.Contains(t, err.Error(), "support-agent-evals") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go b/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go index e77445512dc..26f5d2e0506 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go @@ -2005,6 +2005,27 @@ func EvaluatorVersionWithSource(index int, evaluator string) error { "on the project", index, evaluator) } +// EvaluatorVersionWithDefinition reports the same pin against a rubric written +// out in the configuration rather than named as a file. +func EvaluatorVersionWithDefinition(index int, evaluator string) error { + return fmt.Errorf( + "evaluators[%d] (%s): `version` cannot be set with `definition`, because "+ + "the service assigns the version when it publishes. Drop `version` to "+ + "publish this rubric, or drop `definition` to reference a version "+ + "already on the project", index, evaluator) +} + +// EvaluatorRubricDeclaredTwice reports a rubric both named and written out. +// +// Publishing uses the written one, so leaving this to the schema alone would +// mean the file quietly never got read. +func EvaluatorRubricDeclaredTwice(index int, evaluator string) error { + return fmt.Errorf( + "evaluators[%d] (%s): `source` and `definition` both give the rubric; "+ + "declare one. Keep `definition` to publish what is written here, or "+ + "keep `source` to publish the file it names", index, evaluator) +} + // DatasetAndSourceDeclareTheSameThing reports it where there is no index. func DatasetAndSourceDeclareTheSameThing() error { return errors.New("`dataset` and `source` both say where rows come from; declare one") diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/anchor_ref_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/anchor_ref_test.go new file mode 100644 index 00000000000..4e191dfe8fd --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/anchor_ref_test.go @@ -0,0 +1,54 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package project + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// Resolving a `$ref` round-trips the document through a map, which expands YAML +// anchors. The expansion has to be faithful, because the alias is how authors +// avoid repeating a judge model across evaluators. +func TestAnchorsSurviveRefResolution(t *testing.T) { + dir := t.TempDir() + + require.NoError(t, os.MkdirAll(filepath.Join(dir, "evaluators"), 0o755)) + require.NoError(t, os.WriteFile( + filepath.Join(dir, "evaluators", "quality.json"), + []byte(`{"type":"rubric","dimensions":[{"id":"tone","weight":3}]}`), + 0o600)) + + path := filepath.Join(dir, EvalConfigBase) + require.NoError(t, os.WriteFile(path, []byte(` +evaluators: + - $ref: ./evaluators/quality.json + name: quality + +evals: + - name: nightly + dataset: golden + evaluators: + - evaluator: builtin.relevance + initialization_parameters: &judge + model: gpt-5.6-luna + - evaluator: builtin.coherence + initialization_parameters: *judge +`), 0o600)) + + cfg, err := LoadEvalConfig(path) + require.NoError(t, err, "an anchor is not a mistyped key") + require.Len(t, cfg.Evals, 1) + require.Len(t, cfg.Evals[0].Evaluators, 2) + + assert.Equal(t, cfg.Evals[0].Evaluators[0].InitializationParameters, + cfg.Evals[0].Evaluators[1].InitializationParameters, + "the alias has to carry the same parameters the anchor declared") + assert.NotEmpty(t, cfg.Evals[0].Evaluators[1].InitializationParameters, + "an alias that expanded to nothing would silently drop the judge model") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/config_keys_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/config_keys_test.go index 1711765f7ca..22fb17482de 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/config_keys_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/config_keys_test.go @@ -80,10 +80,17 @@ func TestSourceDeclKeys(t *testing.T) { yamlKeys(t, SourceDecl{})) } -// The catalogs are named, reusable assets: a name and where it comes from. +// The catalogs are named, reusable assets. A dataset says where its rows come +// from. An evaluator says where its rubric is -- named as a file, or written +// out under `definition` -- and may instead be pulled in with `$ref`, which is +// modelled rather than only resolved so that a command which reads, modifies +// and saves the file writes the author's include back out instead of inlining +// it. func TestCatalogKeys(t *testing.T) { - assert.ElementsMatch(t, []string{"name", "source", "version"}, yamlKeys(t, DatasetDecl{})) - assert.ElementsMatch(t, []string{"name", "source", "version"}, yamlKeys(t, EvaluatorDecl{})) + assert.ElementsMatch(t, []string{"name", "file", "version"}, yamlKeys(t, DatasetDecl{})) + assert.ElementsMatch(t, + []string{"$ref", "name", "source", "version", "definition"}, + yamlKeys(t, EvaluatorDecl{})) } // The spec's casing table: eval.yaml uses the API's snake_case throughout, so diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/config_location_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/config_location_test.go new file mode 100644 index 00000000000..326b4f19984 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/config_location_test.go @@ -0,0 +1,78 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package project + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// azure.yaml's `$ref` names a file, not a directory. +// +// A project is free to declare `./config/nightly.yaml`. Reading only the +// directory out of that and then looking for `azure.eval.yaml` beside it +// reports the configuration missing while `azd up` deploys it from the very +// same `$ref` -- the two-answers-to-one-question shape this cascade exists to +// close. +func TestLocationMayNameTheFileTheRefDeclares(t *testing.T) { + dir := t.TempDir() + declared := filepath.Join(dir, "nightly.yaml") + require.NoError(t, os.WriteFile(declared, []byte("evals:\n - name: nightly\n"), 0o600)) + + path, err := ResolveEvalConfigPath(declared) + require.NoError(t, err) + assert.Equal(t, declared, path, "the declared file is the configuration") + + cfg, err := OpenEvalConfig(declared) + require.NoError(t, err) + require.NotNil(t, cfg) + require.Len(t, cfg.Evals, 1) + assert.Equal(t, "nightly", cfg.Evals[0].Name) + + assert.Equal(t, dir, EvalDirOf(declared), + "artifacts sit beside the configuration, not inside it") +} + +// A directory keeps the naming convention it always had. +func TestLocationMayStillBeTheDirectory(t *testing.T) { + dir := t.TempDir() + require.NoError(t, os.WriteFile( + filepath.Join(dir, EvalConfigBase), []byte("evals:\n - name: nightly\n"), 0o600)) + + path, err := ResolveEvalConfigPath(dir) + require.NoError(t, err) + assert.Equal(t, filepath.Join(dir, EvalConfigBase), path) + assert.Equal(t, dir, EvalDirOf(dir)) +} + +// A location that does not exist yet is a directory: it is what `init` is given +// before it writes anything. +func TestAMissingLocationIsReadAsADirectory(t *testing.T) { + dir := filepath.Join(t.TempDir(), "not-created-yet") + + assert.Equal(t, filepath.Join(dir, EvalConfigBase), EvalConfigPath(dir)) + assert.Equal(t, dir, EvalDirOf(dir)) +} + +// The both-names guard is about a directory holding two candidates. A location +// that already names the file has nothing to disambiguate. +func TestADeclaredFileSkipsTheAmbiguityGuard(t *testing.T) { + dir := t.TempDir() + require.NoError(t, os.WriteFile( + filepath.Join(dir, EvalConfigBase), []byte("evals: []\n"), 0o600)) + require.NoError(t, os.WriteFile( + filepath.Join(dir, LegacyEvalConfigBase), []byte("evals: []\n"), 0o600)) + + _, err := ResolveEvalConfigPath(dir) + require.Error(t, err, "a directory holding both names is still refused") + + declared := filepath.Join(dir, EvalConfigBase) + got, err := ResolveEvalConfigPath(declared) + require.NoError(t, err, "naming the file is how a project says which one it means") + assert.Equal(t, declared, got) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/edit_read_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/edit_read_test.go new file mode 100644 index 00000000000..ef0b5cdc33d --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/edit_read_test.go @@ -0,0 +1,93 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package project + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// A command that writes the configuration back reads it as written. +// +// `init` and `generate` read, modify and save the same file. Handing them a +// resolved configuration inlined the author's includes, orphaned the files they +// named, and left the paths inside those files resolving against the wrong +// directory: a `source: ./quality.json` written beside `evaluators/quality.yaml` +// came back pointing at the project root. Nothing reported it, because from the +// writer's point of view it had saved what it read. +func TestEditingReadsLeaveIncludesAlone(t *testing.T) { + dir := t.TempDir() + require.NoError(t, os.MkdirAll(filepath.Join(dir, "evaluators"), 0o755)) + require.NoError(t, os.WriteFile( + filepath.Join(dir, "evaluators", "quality.yaml"), + []byte("name: quality\nsource: ./quality.json\n"), 0o600)) + + path := filepath.Join(dir, EvalConfigBase) + require.NoError(t, os.WriteFile(path, []byte(`datasets: + - name: golden + file: ./datasets/golden.jsonl + +evaluators: + - $ref: ./evaluators/quality.yaml + +evals: + - name: nightly + dataset: golden +`), 0o600)) + + cfg, err := OpenEvalConfigForEdit(dir) + require.NoError(t, err) + require.NotNil(t, cfg) + require.NoError(t, SaveEvalConfig(dir, cfg)) + + after, err := os.ReadFile(path) + require.NoError(t, err) + text := string(after) + + assert.Contains(t, text, "$ref: ./evaluators/quality.yaml", + "the author's include has to survive a command that saves the file") + assert.NotContains(t, text, "source: ./quality.json", + "inlining it would leave that path resolving against the wrong directory") +} + +// The reader that commands *use* still resolves, so the two do not drift apart. +func TestConsumingReadsStillResolve(t *testing.T) { + dir := t.TempDir() + require.NoError(t, os.MkdirAll(filepath.Join(dir, "evaluators"), 0o755)) + require.NoError(t, os.WriteFile( + filepath.Join(dir, "evaluators", "quality.yaml"), + []byte("name: quality\nsource: ./quality.json\n"), 0o600)) + + require.NoError(t, os.WriteFile(filepath.Join(dir, EvalConfigBase), []byte(`evaluators: + - $ref: ./evaluators/quality.yaml + +evals: + - name: nightly +`), 0o600)) + + cfg, err := OpenEvalConfig(dir) + require.NoError(t, err) + require.Len(t, cfg.Evaluators, 1) + assert.Equal(t, "quality", cfg.Evaluators[0].Name) +} + +// An include is the only thing the editing reader treats differently. A +// configuration without one decodes identically either way, so a mistyped key +// is still refused on the path that writes. +func TestEditingReadsAreStillStrict(t *testing.T) { + dir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(dir, EvalConfigBase), + []byte("datasets:\n - name: golden\n fiel: ./x.jsonl\n"), 0o600)) + + _, err := OpenEvalConfigForEdit(dir) + + require.Error(t, err) + assert.True(t, strings.Contains(err.Error(), "fiel"), + "the typo has to be named on the path that saves the file too") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go index 643f84c6ceb..4e46395be50 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go @@ -26,11 +26,15 @@ type EvalConfig struct { Evals []Eval `yaml:"evals,omitempty" json:"evals,omitempty"` } -// DatasetDecl is a catalog entry. A local Source is uploaded on deploy; without +// DatasetDecl is a catalog entry. A local File is uploaded on deploy; without // one the name must already resolve to a registered dataset. +// +// Deliberately not a `$ref`: that directive replaces a definition with one +// loaded from a YAML or JSON file, and these rows are an artifact to publish. +// A `.jsonl` is neither, so there would be nothing to splice. type DatasetDecl struct { Name string `yaml:"name" json:"name"` - Source string `yaml:"source,omitempty" json:"source,omitempty"` + File string `yaml:"file,omitempty" json:"file,omitempty"` Version string `yaml:"version,omitempty" json:"version,omitempty"` } @@ -39,10 +43,25 @@ type DatasetDecl struct { // // Source names a `.json` file holding a rubric: a list of weighted scoring // dimensions. +// +// Ref carries the `$ref` an author wrote. Commands that use the configuration +// never see it -- resolution has already replaced the entry with the file's +// content by then -- but the commands that read, modify and save the file do, +// and modelling it is what lets the include survive being written back. Name is +// omitempty for the same reason: an entry that is only a `$ref` has no name of +// its own until the file it names supplies one. +// +// Definition is the rubric written out in place of naming a file, which is what +// lets a `$ref` name a rubric: resolution splices the file's keys in here, and +// they have to land on a field to survive strict decoding. It is deliberately +// one named key rather than a catch-all: a catch-all would swallow every +// misspelling in the entry and publish it to the service as rubric content. type EvaluatorDecl struct { - Name string `yaml:"name" json:"name"` - Source string `yaml:"source,omitempty" json:"source,omitempty"` - Version string `yaml:"version,omitempty" json:"version,omitempty"` + Ref string `yaml:"$ref,omitempty" json:"$ref,omitempty"` + Name string `yaml:"name,omitempty" json:"name,omitempty"` + Source string `yaml:"source,omitempty" json:"source,omitempty"` + Version string `yaml:"version,omitempty" json:"version,omitempty"` + Definition map[string]any `yaml:"definition,omitempty" json:"definition,omitempty"` } // Eval is one evaluation defined over the catalogs. @@ -205,7 +224,7 @@ func (c *EvalConfig) CustomEvaluators() []EvaluatorDecl { func (c *EvalConfig) LocalDatasets() []DatasetDecl { var owned []DatasetDecl for _, decl := range c.Datasets { - if decl.Source == "" { + if decl.File == "" { continue } owned = append(owned, decl) @@ -324,6 +343,14 @@ func (c *EvalConfig) validateCatalogs() error { if e.Source != "" && e.Version != "" { return messages.EvaluatorVersionWithSource(i, e.Name) } + if e.Definition != nil { + if e.Source != "" { + return messages.EvaluatorRubricDeclaredTwice(i, e.Name) + } + if e.Version != "" { + return messages.EvaluatorVersionWithDefinition(i, e.Name) + } + } } return nil } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_ambiguity_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_ambiguity_test.go index cd41c8d87a8..7560d1b81b7 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_ambiguity_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_ambiguity_test.go @@ -17,7 +17,7 @@ func writeFile(t *testing.T, dir, name, body string) { require.NoError(t, os.WriteFile(filepath.Join(dir, name), []byte(body), 0o600)) } -const oneEvalConfig = "datasets:\n - name: d\n source: ./d.jsonl\n" +const oneEvalConfig = "datasets:\n - name: d\n file: ./d.jsonl\n" // azure.yaml references one configuration by name. With both files present the // CLI would edit whichever it preferred while azd up deployed whichever the @@ -54,7 +54,7 @@ func TestSaveEvalConfig_WritesBackToTheLegacyFile(t *testing.T) { writeFile(t, dir, LegacyEvalConfigBase, oneEvalConfig) require.NoError(t, SaveEvalConfig(dir, &EvalConfig{ - Datasets: []DatasetDecl{{Name: "d", Source: "./d.jsonl"}}, + Datasets: []DatasetDecl{{Name: "d", File: "./d.jsonl"}}, })) assert.FileExists(t, filepath.Join(dir, LegacyEvalConfigBase)) @@ -77,7 +77,7 @@ func TestSaveEvalConfig_WritesTheCurrentNameWhenThereIsNoFile(t *testing.T) { func TestValidate_RefusesATargetWithNoName(t *testing.T) { dir := t.TempDir() writeFile(t, dir, EvalConfigBase, - "datasets:\n - name: d\n source: ./d.jsonl\n"+ + "datasets:\n - name: d\n file: ./d.jsonl\n"+ "evals:\n - name: e\n dataset: d\n"+ " target:\n type: agent\n"+ " evaluators:\n - evaluator: builtin.relevance\n") diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_atomic_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_atomic_test.go index f41a20dcd66..a7312ae629a 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_atomic_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_atomic_test.go @@ -30,7 +30,7 @@ func TestSaveEvalConfigNeverExposesAHalfWrittenFile(t *testing.T) { path := filepath.Join(dir, "azure.eval.yaml") full := &EvalConfig{ - Datasets: []DatasetDecl{{Name: "golden", Source: "./datasets/golden.jsonl"}}, + Datasets: []DatasetDecl{{Name: "golden", File: "./datasets/golden.jsonl"}}, Evals: []Eval{ {Name: "first", EvaluationLevel: "turn"}, {Name: "second", EvaluationLevel: "turn"}, diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_keys_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_keys_test.go index fc3338e994e..8165eb60b06 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_keys_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_keys_test.go @@ -41,7 +41,7 @@ func TestExplainUnknownKeys_LeavesOtherErrors(t *testing.T) { func TestKeysOfTypeCoversTheDeclarations(t *testing.T) { assert.Contains(t, keysOfType("project.Eval"), "evaluators") assert.Contains(t, keysOfType("project.EvalConfig"), "datasets") - assert.Contains(t, keysOfType("project.DatasetDecl"), "source") + assert.Contains(t, keysOfType("project.DatasetDecl"), "file") assert.Empty(t, keysOfType("project.Unknown")) } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_name_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_name_test.go index b74ab1cdf96..f1570a5cc6c 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_name_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_name_test.go @@ -12,7 +12,7 @@ import ( "github.com/stretchr/testify/require" ) -const minimalConfig = "datasets:\n - name: golden\n source: ./datasets/golden.jsonl\n" +const minimalConfig = "datasets:\n - name: golden\n file: ./datasets/golden.jsonl\n" // The file is named for azd, the way azure.yaml is. func TestEvalConfigPath_IsTheAzdPrefixedName(t *testing.T) { @@ -44,7 +44,7 @@ func TestSaveEvalConfig_WritesBackOverALegacyFile(t *testing.T) { require.NoError(t, os.WriteFile(legacy, []byte(minimalConfig), 0o600)) require.NoError(t, SaveEvalConfig(dir, &EvalConfig{ - Datasets: []DatasetDecl{{Name: "added", Source: "./datasets/added.jsonl"}}, + Datasets: []DatasetDecl{{Name: "added", File: "./datasets/added.jsonl"}}, })) body, err := os.ReadFile(legacy) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_roundtrip_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_roundtrip_test.go index 27cbcc1fe27..f47afeb9bc8 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_roundtrip_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_roundtrip_test.go @@ -29,7 +29,7 @@ func TestEvalConfigRoundTripKeepsWhatTheAuthorWrote(t *testing.T) { - evaluator: builtin.task_adherence datasets: - name: golden - source: ./datasets/golden.jsonl + file: ./datasets/golden.jsonl ` require.NoError(t, os.WriteFile(path, []byte(authored), 0o600)) @@ -77,7 +77,7 @@ func TestEvalConfigAcceptsEveryKeyItWrites(t *testing.T) { require.NoError(t, os.WriteFile(path, []byte(`datasets: - name: golden - source: ./datasets/golden.jsonl + file: ./datasets/golden.jsonl version: "2" evaluators: - name: quality diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_store.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_store.go index aefd74e551d..258f9077de0 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_store.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_store.go @@ -16,6 +16,7 @@ import ( "azureaieval/internal/messages" + "github.com/azure/azure-dev/cli/azd/pkg/foundry" "go.yaml.in/yaml/v3" ) @@ -39,14 +40,39 @@ const EvalConfigBase = "azure.eval.yaml" // not silently grow a second configuration beside it. const LegacyEvalConfigBase = "eval.yaml" -// EvalConfigPath is the configuration file inside an eval directory. It is -// exported for error messages and for the azure.yaml $ref; readers should -// prefer OpenEvalConfig. -func EvalConfigPath(evalDir string) string { - return filepath.Join(evalDir, EvalConfigBase) +// EvalConfigPath is the configuration file at a location. It is exported for +// error messages and for the azure.yaml $ref; readers should prefer +// OpenEvalConfig. +// +// A location is normally the eval directory, and the file inside it is named by +// convention. It may also be the file itself, because azure.yaml's `$ref` names +// one by name rather than by directory: a project is free to declare +// `./config/nightly.yaml`, and looking for `azure.eval.yaml` beside it would +// report the configuration missing while `azd up` deployed it. +func EvalConfigPath(location string) string { + if namesAFile(location) { + return location + } + return filepath.Join(location, EvalConfigBase) +} + +// EvalDirOf is the directory a location's relative paths resolve against. +func EvalDirOf(location string) string { + if namesAFile(location) { + return filepath.Dir(location) + } + return location +} + +// namesAFile reports whether a location is the configuration file rather than +// the directory holding it. A path that does not exist is read as a directory, +// which is what `init` is given before it writes anything. +func namesAFile(location string) bool { + info, err := os.Stat(location) + return err == nil && !info.IsDir() } -// ResolveEvalConfigPath is the configuration this directory actually holds: +// ResolveEvalConfigPath is the configuration this location actually holds: // the current name, or the legacy one when that is the only file there. // // It refuses a directory holding both, rather than leaving that to the caller. @@ -55,21 +81,24 @@ func EvalConfigPath(evalDir string) string { // while `run`, `init` and `generate` all refused. Returning an error is what // makes the guard unavoidable: there is no longer a way to ask this question // and not be told. -func ResolveEvalConfigPath(evalDir string) (string, error) { - if err := checkOneConfig(evalDir); err != nil { +func ResolveEvalConfigPath(location string) (string, error) { + if err := checkOneConfig(location); err != nil { return "", err } - return resolvedConfigPath(evalDir), nil + return resolvedConfigPath(location), nil } // resolvedConfigPath is the naming rule on its own, for the two functions that // have already applied the guard. -func resolvedConfigPath(evalDir string) string { - current := EvalConfigPath(evalDir) +func resolvedConfigPath(location string) string { + if namesAFile(location) { + return location + } + current := EvalConfigPath(location) if _, err := os.Stat(current); err == nil { return current } - legacy := filepath.Join(evalDir, LegacyEvalConfigBase) + legacy := filepath.Join(location, LegacyEvalConfigBase) if _, err := os.Stat(legacy); err == nil { return legacy } @@ -80,10 +109,14 @@ func resolvedConfigPath(evalDir string) string { // // Preferring one silently is the dangerous answer: `azure.yaml` `$ref`s a // single file by name, so the CLI would edit one configuration while `azd up` -// deployed the other, and nothing would say so. -func checkOneConfig(evalDir string) error { - current := EvalConfigPath(evalDir) - legacy := filepath.Join(evalDir, LegacyEvalConfigBase) +// deployed the other, and nothing would say so. A location that already names +// the file has nothing to disambiguate. +func checkOneConfig(location string) error { + if namesAFile(location) { + return nil + } + current := EvalConfigPath(location) + legacy := filepath.Join(location, LegacyEvalConfigBase) if _, err := os.Stat(current); err != nil { return nil } @@ -93,36 +126,176 @@ func checkOneConfig(evalDir string) error { return messages.AmbiguousEvalConfig(current, legacy) } -// OpenEvalConfig reads the configuration under evalDir. +// OpenEvalConfig reads the configuration at a location, with `$ref` includes +// resolved. This is the reader for commands that *use* the configuration. // // A missing file returns (nil, nil): generate runs before init, so "no // configuration yet" is an ordinary state rather than a failure. -func OpenEvalConfig(evalDir string) (*EvalConfig, error) { - if err := checkOneConfig(evalDir); err != nil { +// +// Commands that write the configuration back must use OpenEvalConfigForEdit +// instead. Resolution and editing do not mix: what comes back here is the +// configuration with every include spliced in, and saving that replaces the +// author's `$ref` with its content. +func OpenEvalConfig(location string) (*EvalConfig, error) { + return openEvalConfig(location, true) +} + +// OpenEvalConfigForEdit reads the configuration exactly as written, leaving +// `$ref` directives alone. +// +// `init` and `generate` read, modify and write the same file. Handing them a +// resolved configuration and saving the result inlined the author's includes, +// orphaned the files they named, and left the paths inside those files +// resolving against the wrong directory -- a `source: ./quality.json` written +// beside `evaluators/quality.yaml` came back pointing at the project root. +// None of it was reported, because from the writer's point of view it had +// simply saved what it read. +func OpenEvalConfigForEdit(location string) (*EvalConfig, error) { + return openEvalConfig(location, false) +} + +func openEvalConfig(location string, resolve bool) (*EvalConfig, error) { + if err := checkOneConfig(location); err != nil { return nil, err } - cfg, err := LoadEvalConfig(resolvedConfigPath(evalDir)) + cfg, err := loadEvalConfig(resolvedConfigPath(location), resolve) if errors.Is(err, fs.ErrNotExist) { return nil, nil } return cfg, err } -// LoadEvalConfig reads a configuration from an explicit path. The path is used -// verbatim, relative to the process working directory — never re-rooted. +// LoadEvalConfig reads a configuration from an explicit path, with `$ref` +// includes resolved. The path is used verbatim, relative to the process working +// directory — never re-rooted. // // Decoded strictly: a key this extension does not know is a typo, and reading // it as nothing leaves a configuration that looks fine and fails later // somewhere else. `agent:` written under `target:` instead of `type:`/`name:` // used to produce an empty target and a run that complained about the target. func LoadEvalConfig(path string) (*EvalConfig, error) { + return loadEvalConfig(path, true) +} + +func loadEvalConfig(path string, resolve bool) (*EvalConfig, error) { data, err := ReadFileNoBOM(path) if err != nil { return nil, messages.ReadingEvalConfig(path, err) } + if resolve { + data, err = resolveConfigRefs(data, filepath.Dir(path), path) + if err != nil { + return nil, err + } + } return DecodeEvalConfig(data, path) } +// resolveConfigRefs expands `$ref` includes before the strict decode. +// +// Core owns the resolver but does not run it for us: it hands each extension +// the entry with `$ref` still in it. The service target has always called it, +// and this path did not, so `azd up` accepted an include that every CLI command +// then refused as an unknown key — the same file meaning two different things +// depending on which command opened it. +// +// A configuration with no `$ref` is returned untouched rather than round-tripped +// through a map, so the overwhelmingly common case keeps the decoder's own line +// numbers in its diagnostics. +func resolveConfigRefs(data []byte, baseDir, name string) ([]byte, error) { + if !bytes.Contains(data, []byte("$ref")) { + return data, nil + } + + var raw map[string]any + if err := yaml.Unmarshal(data, &raw); err != nil { + return nil, messages.ParsingEvalConfig(name, err) + } + if raw == nil { + return data, nil + } + + resolved, err := resolveEvalRefs(raw, baseDir) + if err != nil { + return nil, err + } + + out, err := yaml.Marshal(resolved) + if err != nil { + return nil, messages.ParsingEvalConfig(name, err) + } + return out, nil +} + +// resolveEvalRefs is the one place `$ref` is resolved, so the CLI and `azd up` +// cannot disagree about what an include means. +// +// They read the configuration by different routes -- off disk, and out of the +// service entry -- and each used to resolve for itself. Every rule then had to +// be added twice, and twice it was not: an include `azd up` accepted and every +// CLI command refused, and later the reverse. Callers differ in how they obtain +// the map and what they do with it; everything between is here. +func resolveEvalRefs(values map[string]any, baseDir string) (map[string]any, error) { + resolved, err := foundry.ResolveFileRefs(values, baseDir) + if err != nil { + return nil, messages.ResolvingServiceRefs(err) + } + // `$ref` is a directive rather than configuration, and the strict decoder + // would report the leftover as a mistyped key. + delete(resolved, "$ref") + nestSplicedRubrics(resolved) + return resolved, nil +} + +// evaluatorDeclKeys are the keys an evaluator entry declares in its own right. +// Anything else at entry level was spliced in by a `$ref`. +var evaluatorDeclKeys = map[string]bool{ + "$ref": true, "name": true, "source": true, "version": true, "definition": true, +} + +// nestSplicedRubrics moves a rubric that a `$ref` spliced in at entry level +// down under `definition`. +// +// `$ref` splices the referenced file's top-level keys into the entry, and a +// rubric file is a bare `{type, dimensions}` -- the shape `generate` downloads +// from the service -- so its keys land beside `name` and the strict decoder +// rejects them. Moving them is what lets a `$ref` name a rubric. +// +// `dimensions` is what marks the leftovers as a rubric rather than a typo, and +// it is the same key normalizeRubricBody insists on before it will treat a +// document as a definition. Without that gate this would be a catch-all by +// another name, filing a misspelled `name` as rubric content and publishing it +// to the service instead of reporting it. +// +// Structural rather than positional on purpose: an earlier version marked +// entries by index before resolution, which cannot see the evaluators inside a +// config that is itself behind a `$ref` -- the layout the README documents. +func nestSplicedRubrics(resolved map[string]any) { + entries, _ := resolved["evaluators"].([]any) + for _, entry := range entries { + m, ok := entry.(map[string]any) + if !ok { + continue + } + // A file already shaped `{name, definition}` needs no rescue, and + // merging into it would guess at which one the author meant. + if _, has := m["definition"]; has { + continue + } + if _, isRubric := m["dimensions"]; !isRubric { + continue + } + rubric := map[string]any{} + for key, value := range m { + if !evaluatorDeclKeys[key] { + rubric[key] = value + delete(m, key) + } + } + m["definition"] = rubric + } +} + // DecodeEvalConfig is the one strict decoder, so every route into a // configuration reports a mistyped key the same way. // diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_test.go index ba81f5fcdf3..4a49cd660db 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_test.go @@ -16,7 +16,7 @@ import ( const sampleEvalConfig = ` datasets: - name: support-golden - source: ./datasets/support-golden.jsonl + file: ./datasets/support-golden.jsonl version: "1" - name: prod-registered @@ -64,7 +64,7 @@ func TestLoadEvalConfig_ParsesAllSections(t *testing.T) { require.Len(t, cfg.Datasets, 2) require.Equal(t, "support-golden", cfg.Datasets[0].Name) - require.Equal(t, "./datasets/support-golden.jsonl", cfg.Datasets[0].Source) + require.Equal(t, "./datasets/support-golden.jsonl", cfg.Datasets[0].File) require.Equal(t, "1", cfg.Datasets[0].Version) require.Len(t, cfg.Evaluators, 1) @@ -159,7 +159,7 @@ func TestDeclarationLookups(t *testing.T) { ds, ok := cfg.DatasetDeclaration("support-golden") require.True(t, ok) - require.Equal(t, "./datasets/support-golden.jsonl", ds.Source) + require.Equal(t, "./datasets/support-golden.jsonl", ds.File) _, ok = cfg.DatasetDeclaration("missing") require.False(t, ok) @@ -193,7 +193,7 @@ func TestOpenEvalConfig_MissingIsNotAnError(t *testing.T) { func TestSaveEvalConfig_CreatesTheDirectory(t *testing.T) { dir := filepath.Join(t.TempDir(), "evals") require.NoError(t, SaveEvalConfig(dir, &EvalConfig{ - Datasets: []DatasetDecl{{Name: "generated", Source: "./datasets/generated.jsonl"}}, + Datasets: []DatasetDecl{{Name: "generated", File: "./datasets/generated.jsonl"}}, })) cfg, err := OpenEvalConfig(dir) @@ -311,7 +311,7 @@ func TestValidate_Rejects(t *testing.T) { }, { name: "dataset without a name", - body: "datasets:\n - source: ./d.jsonl\n" + oneEval, + body: "datasets:\n - file: ./d.jsonl\n" + oneEval, wantErr: "'name' is required", }, { @@ -329,6 +329,18 @@ func TestValidate_Rejects(t *testing.T) { body: "evaluators:\n - name: q\n source: ./q.json\n version: \"3\"\n" + oneEval, wantErr: "cannot be set with `source`", }, + { + name: "rubric both named and written out", + body: "evaluators:\n - name: q\n source: ./q.json\n" + + " definition:\n dimensions: []\n" + oneEval, + wantErr: "both give the rubric", + }, + { + name: "version pinned alongside a definition", + body: "evaluators:\n - name: q\n version: \"3\"\n" + + " definition:\n dimensions: []\n" + oneEval, + wantErr: "cannot be set with `definition`", + }, { name: "no evals", body: "datasets:\n - name: d\n", diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/max_samples_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/max_samples_test.go index 218f5c159bb..5e65bd60a06 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/max_samples_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/max_samples_test.go @@ -17,7 +17,7 @@ import ( // the opposite of what a cap asks for, and with nothing said about it. func TestNegativeMaxSamplesIsRefused(t *testing.T) { cfg := &EvalConfig{ - Datasets: []DatasetDecl{{Name: "golden", Source: "./datasets/golden.jsonl"}}, + Datasets: []DatasetDecl{{Name: "golden", File: "./datasets/golden.jsonl"}}, Evals: []Eval{{ Name: "support-quality", Dataset: "golden", @@ -38,7 +38,7 @@ func TestNegativeMaxSamplesIsRefused(t *testing.T) { // Zero is how a config says "send every row", and has to keep working. func TestUnsetMaxSamplesIsStillAllowed(t *testing.T) { cfg := &EvalConfig{ - Datasets: []DatasetDecl{{Name: "golden", Source: "./datasets/golden.jsonl"}}, + Datasets: []DatasetDecl{{Name: "golden", File: "./datasets/golden.jsonl"}}, Evals: []Eval{{ Name: "support-quality", Dataset: "golden", diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/nested_ref_rubric_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/nested_ref_rubric_test.go new file mode 100644 index 00000000000..58fbaefcd88 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/nested_ref_rubric_test.go @@ -0,0 +1,74 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package project + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// The documented layout puts the whole eval config behind a `$ref` on the +// service, and the evaluators inside it carry `$ref`s of their own. +func TestARefdRubricInsideARefdConfig(t *testing.T) { + dir := t.TempDir() + evals := filepath.Join(dir, "evals") + require.NoError(t, os.MkdirAll(filepath.Join(evals, "evaluators"), 0o755)) + + require.NoError(t, os.WriteFile( + filepath.Join(evals, "evaluators", "quality.json"), + []byte(`{"type":"rubric","dimensions":[{"id":"tone","weight":3}]}`), + 0o600)) + + require.NoError(t, os.WriteFile( + filepath.Join(evals, EvalConfigBase), []byte(` +evaluators: + - $ref: ./evaluators/quality.json + name: quality + +evals: + - name: nightly + dataset: golden +`), 0o600)) + + svc := serviceWith(t, map[string]any{"$ref": "./evals/" + EvalConfigBase}) + + cfg, err := EvalConfigFromService(svc, dir) + require.NoError(t, err, "the layout the README and spec document has to deploy") + require.Len(t, cfg.Evaluators, 1) + assert.Equal(t, "quality", cfg.Evaluators[0].Name) + assert.Equal(t, "rubric", cfg.Evaluators[0].Definition["type"]) +} + +// `dimensions` is what tells a spliced rubric from a mistake, so a `$ref` to a +// file that is not a rubric is still reported rather than filed away. +// +// This is the whole difference between the rescue and a catch-all, so it is +// worth a test of its own: widening the gate would make every misspelling in an +// evaluator entry publishable content. +func TestARefToSomethingThatIsNotARubricIsStillRejected(t *testing.T) { + dir := t.TempDir() + + require.NoError(t, os.MkdirAll(filepath.Join(dir, "evaluators"), 0o755)) + require.NoError(t, os.WriteFile( + filepath.Join(dir, "evaluators", "quality.json"), + []byte(`{"nmae":"quality","weight":3}`), 0o600)) + + path := filepath.Join(dir, EvalConfigBase) + require.NoError(t, os.WriteFile(path, []byte(` +evaluators: + - $ref: ./evaluators/quality.json + +evals: + - name: nightly + dataset: golden +`), 0o600)) + + _, err := LoadEvalConfig(path) + require.Error(t, err) + assert.Contains(t, err.Error(), "nmae", "the error has to name the key that is wrong") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/one_resolver_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/one_resolver_test.go new file mode 100644 index 00000000000..d2c30404750 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/one_resolver_test.go @@ -0,0 +1,54 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package project + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// `$ref` is resolved in exactly one place. +// +// The CLI and `azd up` reach a configuration by different routes and each used +// to resolve for itself, so every rule had to be added twice -- and twice it +// was not, each time producing an include one route accepted and the other +// refused. A second caller of ResolveFileRefs is how that comes back, and it +// comes back silently, so it is worth failing the build over. +func TestRefsAreResolvedInOnePlace(t *testing.T) { + const resolver = "resolveEvalRefs" + + callers := map[string][]int{} + require.NoError(t, filepath.WalkDir(".", func(path string, d os.DirEntry, err error) error { + if err != nil || d.IsDir() || !strings.HasSuffix(path, ".go") { + return err + } + if strings.HasSuffix(path, "_test.go") { + return nil // including this file, which names the call it is counting + } + body, err := os.ReadFile(path) + if err != nil { + return err + } + for i, line := range strings.Split(string(body), "\n") { + if strings.Contains(line, "foundry.ResolveFileRefs(") { + callers[path] = append(callers[path], i+1) + } + } + return nil + })) + + total := 0 + for _, lines := range callers { + total += len(lines) + } + assert.Equal(t, 1, total, + "ResolveFileRefs has more than one caller (%v); route it through %s instead, "+ + "or the next `$ref` rule will land on one path and not the other", + callers, resolver) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/readfile_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/readfile_test.go index 07ba4497226..b205a89b32b 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/readfile_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/readfile_test.go @@ -59,7 +59,7 @@ func TestLoadEvalConfig_AcceptsAByteOrderMark(t *testing.T) { dir := t.TempDir() path := filepath.Join(dir, EvalConfigBase) body := append([]byte{0xEF, 0xBB, 0xBF}, - []byte("datasets:\n - name: d\n source: ./d.jsonl\n")...) + []byte("datasets:\n - name: d\n file: ./d.jsonl\n")...) require.NoError(t, os.WriteFile(path, body, 0o600)) cfg, err := LoadEvalConfig(path) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/readme_example_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/readme_example_test.go new file mode 100644 index 00000000000..71c8e81b573 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/readme_example_test.go @@ -0,0 +1,53 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package project + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +// The README's configuration example has to be one the CLI can load. +// +// It documented `options.eval_model`, which is not a key this decoder has ever +// had: following the README produced `unknown key "options"`. Nothing caught it +// because the example was prose, so correcting the keys alone would only have +// reset the clock on the same drift. +func TestTheREADMEExampleLoads(t *testing.T) { + readme, err := os.ReadFile(filepath.Join("..", "..", "README.md")) + require.NoError(t, err) + + body := evalConfigExample(t, string(readme)) + + path := filepath.Join(t.TempDir(), EvalConfigBase) + require.NoError(t, os.WriteFile(path, []byte(body), 0o600)) + + _, err = LoadEvalConfig(path) + require.NoError(t, err, "the README example has to survive the decoder it documents") +} + +// evalConfigExample returns the fenced yaml block the README labels as the eval +// configuration, identified by the file name comment on its first line. +func evalConfigExample(t *testing.T, readme string) string { + t.Helper() + + const marker = "```yaml\n# evals/" + EvalConfigBase + "\n" + + readme = strings.ReplaceAll(readme, "\r\n", "\n") + + start := strings.Index(readme, marker) + require.NotEqual(t, -1, start, + "the README no longer opens the example with `# evals/%s`; retarget this test "+ + "rather than deleting it", EvalConfigBase) + + rest := readme[start+len(marker):] + end := strings.Index(rest, "```") + require.NotEqual(t, -1, end, "the example's fence is unterminated") + + return rest[:end] +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/ref_directive_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/ref_directive_test.go new file mode 100644 index 00000000000..88729042ac6 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/ref_directive_test.go @@ -0,0 +1,52 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package project + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// `$ref` decodes rather than being refused, and survives being written back. +// +// It was a directive the strict decoder rejected, on the reasoning that one +// reaching the decoder meant resolution had been skipped. That reasoning only +// held while every reader resolved. The commands that read, modify and save the +// file deliberately do not, because saving a resolved configuration inlines the +// author's includes -- so the decoder has to carry the directive through +// untouched instead of naming it a typo. +// +// TestRefResolvesOnTheCLIPathToo covers the resolved route; +// TestEditingReadsLeaveIncludesAlone covers the round trip. +func TestARefDirectiveDecodesAndSurvives(t *testing.T) { + withRef := []byte(` +evaluators: + - $ref: ./evaluators/quality.yaml +evals: + - name: nightly +`) + + cfg, err := DecodeEvalConfig(withRef, "azure.eval.yaml") + + require.NoError(t, err, "the editing readers hand this straight to the decoder") + require.Len(t, cfg.Evaluators, 1) + assert.Equal(t, "./evaluators/quality.yaml", cfg.Evaluators[0].Ref) + assert.Empty(t, cfg.Evaluators[0].Name, + "an entry that is only a $ref has no name until the file it names supplies one") + + // The spelling that needs no resolution at all. + cfg, err = DecodeEvalConfig([]byte(` +evaluators: + - name: quality + source: ./evaluators/quality.json +evals: + - name: nightly +`), "azure.eval.yaml") + + require.NoError(t, err) + assert.Equal(t, "./evaluators/quality.json", cfg.Evaluators[0].Source) + assert.Empty(t, cfg.Evaluators[0].Ref) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/ref_resolution_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/ref_resolution_test.go new file mode 100644 index 00000000000..71e0d2978c1 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/ref_resolution_test.go @@ -0,0 +1,232 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package project + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// `$ref` means the same thing to `azd up` and to the CLI commands. +// +// Core owns the resolver but does not run it for us -- it hands each extension +// the entry with `$ref` still in it. The service target called it and this path +// did not, so an include deployed fine and then failed every `azd ai eval` +// command with `unknown key "$ref"`: one file, two meanings, decided by which +// command opened it. +func TestRefResolvesOnTheCLIPathToo(t *testing.T) { + dir := t.TempDir() + + require.NoError(t, os.MkdirAll(filepath.Join(dir, "evaluators"), 0o755)) + require.NoError(t, os.WriteFile( + filepath.Join(dir, "evaluators", "quality.yaml"), + []byte("name: support-agent-quality\nsource: ./quality.json\n"), + 0o600)) + + path := filepath.Join(dir, "azure.eval.yaml") + require.NoError(t, os.WriteFile(path, []byte(` +datasets: + - name: golden + file: ./datasets/golden.jsonl + +evaluators: + - $ref: ./evaluators/quality.yaml + +evals: + - name: nightly + dataset: golden +`), 0o600)) + + cfg, err := LoadEvalConfig(path) + require.NoError(t, err, "the include the service target resolves has to resolve here too") + require.Len(t, cfg.Evaluators, 1) + assert.Equal(t, "support-agent-quality", cfg.Evaluators[0].Name, + "the referenced file's content replaces the directive") + assert.Equal(t, "./quality.json", cfg.Evaluators[0].Source) +} + +// A `$ref` can name the rubric itself, not only a pointer to one. +// +// This is the shape the spec documents, and it works because resolution splices +// the referenced file's keys into the entry: they have to land on fields of the +// declaration or strict decoding rejects them. `definition` is that field. +func TestRefCanNameTheRubricItself(t *testing.T) { + dir := t.TempDir() + + require.NoError(t, os.MkdirAll(filepath.Join(dir, "evaluators"), 0o755)) + require.NoError(t, os.WriteFile( + filepath.Join(dir, "evaluators", "quality.json"), + []byte(`{"name":"support-agent-quality",`+ + `"definition":{"type":"rubric","dimensions":[{"name":"tone","weight":1}]}}`), + 0o600)) + + path := filepath.Join(dir, "azure.eval.yaml") + require.NoError(t, os.WriteFile(path, []byte(` +evaluators: + - $ref: ./evaluators/quality.json + +evals: + - name: nightly + dataset: golden +`), 0o600)) + + cfg, err := LoadEvalConfig(path) + require.NoError(t, err, "a rubric named by $ref has to decode") + require.Len(t, cfg.Evaluators, 1) + assert.Equal(t, "support-agent-quality", cfg.Evaluators[0].Name) + assert.Equal(t, "rubric", cfg.Evaluators[0].Definition["type"], + "the rubric travels with the declaration, so there is no second file to find") + assert.Empty(t, cfg.Evaluators[0].Source, + "a definition in hand is not a path to resolve against anything") +} + +// Both routes into a configuration have to agree about a `$ref`'d rubric. +// +// The deploy path resolves includes itself rather than going through +// LoadEvalConfig, so a rescue added on one side only would recreate the exact +// asymmetry that started this work -- an include `azd up` accepted and every +// CLI command refused, in mirror image. +func TestBothRoutesReadARefdRubricTheSameWay(t *testing.T) { + dir := t.TempDir() + + require.NoError(t, os.MkdirAll(filepath.Join(dir, "evaluators"), 0o755)) + require.NoError(t, os.WriteFile( + filepath.Join(dir, "evaluators", "quality.json"), + []byte(`{"type":"rubric","dimensions":[{"id":"tone","weight":3}]}`), + 0o600)) + + path := filepath.Join(dir, "azure.eval.yaml") + require.NoError(t, os.WriteFile(path, []byte(` +evaluators: + - $ref: ./evaluators/quality.json + name: quality + +evals: + - name: nightly + dataset: golden +`), 0o600)) + + fromDisk, err := LoadEvalConfig(path) + require.NoError(t, err) + + svc := serviceWith(t, map[string]any{ + "evaluators": []any{map[string]any{ + "$ref": "./evaluators/quality.json", + "name": "quality", + }}, + "evals": []any{map[string]any{"name": "nightly", "dataset": "golden"}}, + }) + fromService, err := EvalConfigFromService(svc, dir) + require.NoError(t, err, "`azd up` has to read what the CLI reads") + + assert.Equal(t, fromDisk.Evaluators, fromService.Evaluators, + "one file, one meaning, whichever command opened it") +} + +// A `$ref` can name a bare rubric file, which is the shape the spec documents +// and the shape `generate` downloads from the service. +// +// `$ref` splices the file's top-level keys into the entry, so `dimensions` and +// friends land beside `name` and used to be rejected outright. They are moved +// under `definition` instead. Wrapping the file would have been the smaller +// change and the wrong one: the tool writes that file, so the config has to +// read what the tool writes. +func TestRefCanNameABareRubricFile(t *testing.T) { + dir := t.TempDir() + + require.NoError(t, os.MkdirAll(filepath.Join(dir, "evaluators"), 0o755)) + require.NoError(t, os.WriteFile( + filepath.Join(dir, "evaluators", "support-agent-quality.json"), + []byte(`{"type":"rubric","pass_threshold":0.7,`+ + `"dimensions":[{"id":"resolves_issue","weight":9,"description":"Resolves it."}]}`), + 0o600)) + + path := filepath.Join(dir, "azure.eval.yaml") + require.NoError(t, os.WriteFile(path, []byte(` +evaluators: + - $ref: ./evaluators/support-agent-quality.json + name: support-agent-quality + +evals: + - name: nightly + dataset: golden +`), 0o600)) + + cfg, err := LoadEvalConfig(path) + require.NoError(t, err, "the spec's own example has to load") + require.Len(t, cfg.Evaluators, 1) + assert.Equal(t, "support-agent-quality", cfg.Evaluators[0].Name, + "the sibling name stays the author's, not a key from the rubric") + assert.Equal(t, "rubric", cfg.Evaluators[0].Definition["type"]) + assert.Equal(t, 0.7, cfg.Evaluators[0].Definition["pass_threshold"], + "every rubric key travels, not just the ones this decoder happens to know") + assert.Len(t, cfg.Evaluators[0].Definition["dimensions"], 1) +} + +// The rescue above is scoped to entries written as a `$ref`, so a misspelling +// in a hand-written entry is still an error rather than rubric content. +func TestAMisspelledEvaluatorKeyIsStillRejected(t *testing.T) { + dir := t.TempDir() + + path := filepath.Join(dir, "azure.eval.yaml") + require.NoError(t, os.WriteFile(path, []byte(` +evaluators: + - nmae: support-agent-quality + definition: + type: rubric +`), 0o600)) + + _, err := LoadEvalConfig(path) + require.Error(t, err) + assert.Contains(t, err.Error(), "nmae", "the error has to name the key that is wrong") +} + +// Sibling keys overlay the loaded file, which is what lets a name live in the +// configuration while the definition it names lives beside the code it grades. +func TestRefSiblingKeysOverlayTheLoadedFile(t *testing.T) { + dir := t.TempDir() + + require.NoError(t, os.MkdirAll(filepath.Join(dir, "evaluators"), 0o755)) + require.NoError(t, os.WriteFile( + filepath.Join(dir, "evaluators", "quality.yaml"), + []byte("name: from-the-file\nsource: ./quality.json\n"), + 0o600)) + + path := filepath.Join(dir, "azure.eval.yaml") + require.NoError(t, os.WriteFile(path, []byte(` +evaluators: + - $ref: ./evaluators/quality.yaml + name: from-the-configuration + +evals: + - name: nightly +`), 0o600)) + + cfg, err := LoadEvalConfig(path) + require.NoError(t, err) + require.Len(t, cfg.Evaluators, 1) + assert.Equal(t, "from-the-configuration", cfg.Evaluators[0].Name) +} + +// A configuration with no include is handed to the decoder untouched, so its +// diagnostics keep the line numbers of the file the author actually wrote. +func TestConfigWithoutRefIsNotRoundTripped(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "azure.eval.yaml") + require.NoError(t, os.WriteFile(path, []byte(` +datasets: + - name: golden + fiel: ./datasets/golden.jsonl +`), 0o600)) + + _, err := LoadEvalConfig(path) + + require.Error(t, err) + assert.Contains(t, err.Error(), "line 4", + "a typo is reported where it was written, not where a re-marshal put it") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/service_config_strict_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/service_config_strict_test.go index 73f59c69ff7..8bb3dad0a04 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/service_config_strict_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/service_config_strict_test.go @@ -45,7 +45,7 @@ func TestEvalConfigFromServiceRejectsAMistypedKey(t *testing.T) { // the door on the authoring style it is meant to serve. func TestEvalConfigFromServiceAcceptsADeclaredConfig(t *testing.T) { svc := serviceWith(t, map[string]any{ - "datasets": []any{map[string]any{"name": "golden", "source": "./datasets/golden.jsonl"}}, + "datasets": []any{map[string]any{"name": "golden", "file": "./datasets/golden.jsonl"}}, "evals": []any{map[string]any{ "name": "support-agent-eval", "dataset": "golden", diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go index 92951143f2d..539717d07cc 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go @@ -15,7 +15,6 @@ import ( "azureaieval/internal/messages" "github.com/azure/azure-dev/cli/azd/pkg/azdext" - "github.com/azure/azure-dev/cli/azd/pkg/foundry" "go.yaml.in/yaml/v3" "google.golang.org/protobuf/types/known/structpb" ) @@ -154,14 +153,14 @@ func (p *EvalServiceTargetProvider) Deploy( // 1. Datasets the configuration owns. Paths are kept so an eval that names // one can derive its columns without reading the blob back. // - // A declaration with no `source:` is included rather than skipped: it names + // A declaration with no `file:` is included rather than skipped: it names // a dataset that is already registered, and reconciling it is what confirms // it is really there and settles which version a `version:` pin selected. // Skipping it would leave a misspelled name to surface as a failed run. datasetPaths := map[string]string{} for _, decl := range cfg.Datasets { report(progress, messages.ReconcilingDataset(decl.Name)) - localPath := ResolveSource(baseDir, decl.Source) + localPath := ResolveSource(baseDir, decl.File) datasetPaths[decl.Name] = localPath version, changed, err := reconciler.EnsureDataset(ctx, decl, localPath) if err != nil { @@ -291,9 +290,9 @@ func EvalConfigFromService(svc *azdext.ServiceConfig, projectRoot string) (*Eval values := props.AsMap() if projectRoot != "" { - resolved, err := foundry.ResolveFileRefs(values, projectRoot) + resolved, err := resolveEvalRefs(values, projectRoot) if err != nil { - return nil, messages.ResolvingServiceRefs(err) + return nil, err } values = resolved } @@ -382,6 +381,13 @@ func Fingerprint(path string) (string, error) { return hex.EncodeToString(sum[:]), nil } +// FingerprintBytes hashes content that was never a file, which is how a rubric +// written in the configuration gets the change detection a rubric file has. +func FingerprintBytes(data []byte) string { + sum := sha256.Sum256(data) + return hex.EncodeToString(sum[:]) +} + // FingerprintGroup hashes an eval's own declaration. // // Change detection on upstream artifacts is not sufficient: editing a group's diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval_test.go index e20ecd49009..90442c47014 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval_test.go @@ -71,7 +71,7 @@ func TestEvalConfigFromServiceReadsInlineConfig(t *testing.T) { Name: "support-agent-evals", AdditionalProperties: propsFrom(t, map[string]any{ "datasets": []any{ - map[string]any{"name": "golden", "source": "./datasets/golden.jsonl"}, + map[string]any{"name": "golden", "file": "./datasets/golden.jsonl"}, }, "evals": []any{ map[string]any{ diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/unknown_keys_depth_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/unknown_keys_depth_test.go index e8c9dc47cf4..1ec60ac56b1 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/unknown_keys_depth_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/unknown_keys_depth_test.go @@ -41,9 +41,9 @@ func TestUnknownKeysAreNamedAtEveryDepth(t *testing.T) { }, { where: "inside a dataset declaration", - body: "datasets:\n - name: golden\n sourse: ./rows.jsonl\n", - key: "sourse", - nearer: "source", + body: "datasets:\n - name: golden\n fil: ./rows.jsonl\n", + key: "fil", + nearer: "file", line: "line 3", }, } diff --git a/cli/azd/extensions/azure.ai.evaluations/schemas/azure.ai.eval.json b/cli/azd/extensions/azure.ai.evaluations/schemas/azure.ai.eval.json new file mode 100644 index 00000000000..711fb58d30a --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/schemas/azure.ai.eval.json @@ -0,0 +1,249 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://raw.githubusercontent.com/Azure/azure-dev/main/cli/azd/extensions/azure.ai.evaluations/schemas/azure.ai.eval.json", + "title": "Azure AI Foundry evaluation service", + "description": "Service-level configuration for a host: azure.ai.eval entry. The service key is the evaluation configuration name. The body is normally supplied with $ref to an evals/azure.eval.yaml holding the dataset and evaluator catalogs and every eval defined over them; relative paths inside that file resolve against the file's own directory.", + "type": "object", + "additionalProperties": true, + "properties": { + "datasets": { + "type": "array", + "description": "Catalog of named datasets. An entry with a file is uploaded on deploy; without one the name must already resolve to a registered dataset.", + "items": { "$ref": "#/definitions/DatasetDecl" } + }, + "evaluators": { + "type": "array", + "description": "Catalog of named custom evaluators. An entry is a declaration, or a $ref to a file holding one. Built-in evaluators are referenced straight from an eval and are never declared here.", + "items": { + "oneOf": [ + { "$ref": "#/definitions/EvaluatorDecl" }, + { "$ref": "#/definitions/FileRef" } + ] + } + }, + "evals": { + "type": "array", + "description": "The evaluations defined over the catalogs. A list, because one target is normally gated by more than one evaluation.", + "items": { "$ref": "#/definitions/Eval" } + } + }, + "definitions": { + "FileRef": { + "type": "object", + "required": ["$ref"], + "additionalProperties": true, + "description": "Replace an inline definition with a reference to an external YAML or JSON file. Sibling properties on the same object act as overlay overrides on top of the loaded file. Cloned rather than referenced across extensions, following azure.ai.projects.", + "properties": { + "$ref": { + "type": "string", + "description": "Path to a YAML or JSON file containing the definition. Relative paths resolve from the file containing this $ref. Absolute paths are also accepted; remote URLs are not supported." + } + } + }, + "DatasetDecl": { + "type": "object", + "required": ["name"], + "additionalProperties": false, + "properties": { + "name": { + "type": "string", + "description": "Name the dataset is registered under. Letters, digits, dashes and underscores, up to 255 characters." + }, + "file": { + "type": "string", + "description": "Path to a local .jsonl whose rows are uploaded on deploy, relative to this file. Deliberately not a $ref: the rows are a data artifact to publish, not a definition to splice in, and a .jsonl is neither YAML nor JSON. Omit to use a dataset already registered under this name." + }, + "version": { + "type": "string", + "description": "Pin to an already-registered version. Omit to publish and track the newest." + } + } + }, + "EvaluatorDecl": { + "type": "object", + "required": ["name"], + "additionalProperties": false, + "properties": { + "name": { + "type": "string", + "description": "Name the evaluator is registered under. Letters, digits, dashes and underscores, up to 255 characters." + }, + "source": { + "type": "string", + "description": "Path to a local .json rubric -- a list of weighted scoring dimensions -- relative to this file. Published on deploy and fingerprinted locally so a later deploy can tell an edit here from a version published elsewhere." + }, + "definition": { + "type": "object", + "description": "The rubric written out here instead of named as a file. This is also what a $ref to a rubric file becomes once it is resolved. Its keys belong to the evaluator service, so they are not constrained here.", + "required": ["dimensions"], + "properties": { + "type": { + "type": "string", + "description": "Defaults to rubric when omitted." + }, + "dimensions": { + "type": "array", + "description": "The weighted scoring dimensions the evaluator grades on." + } + } + }, + "version": { + "type": "string", + "description": "Pin to an already-registered version. Omit to publish and track the newest." + } + }, + "allOf": [ + { + "$comment": "A rubric is named or written out, never both.", + "if": { "required": ["definition"] }, + "then": { "properties": { "source": false } } + } + ] + }, + "Eval": { + "type": "object", + "required": ["name"], + "additionalProperties": false, + "description": "One evaluation. dataset and source are alternatives for where rows come from; target is a separate axis and means what gets invoked, so an eval can read traces and invoke nothing.", + "properties": { + "name": { + "type": "string", + "description": "Name of the eval. The azd environment records the created eval id under it, which is what --eval resolves." + }, + "id": { + "type": "string", + "description": "An existing eval id, for an eval created outside this configuration." + }, + "description": { "type": "string" }, + "dataset": { + "type": "string", + "description": "Name of an entry in datasets[]. Mutually exclusive with source." + }, + "source": { + "$ref": "#/definitions/SourceDecl", + "description": "Where rows come from when they are not a catalog dataset. Mutually exclusive with dataset." + }, + "evaluation_level": { + "type": "string", + "enum": ["turn", "conversation"], + "description": "Whether each row is graded as a single turn or a whole conversation. The service default is turn." + }, + "max_samples": { + "type": "integer", + "minimum": 0, + "description": "Cap on rows graded per run." + }, + "evaluators": { + "type": "array", + "description": "The evaluators this eval grades with. Entries are either builtin. or a name declared in evaluators[].", + "items": { "$ref": "#/definitions/EvaluatorRef" } + }, + "target": { + "$ref": "#/definitions/Target", + "description": "What the run invokes to produce the responses being graded. Omit to grade the responses already present in the rows." + } + }, + "allOf": [ + { + "$comment": "Rows come from a catalog dataset or from a source, never both.", + "if": { "required": ["dataset"] }, + "then": { "properties": { "source": false } } + } + ] + }, + "SourceDecl": { + "type": "object", + "required": ["type"], + "additionalProperties": false, + "properties": { + "type": { + "type": "string", + "enum": ["traces", "responses"], + "description": "traces reads recorded spans from the project's Application Insights; responses reads named response ids." + }, + "lookback_hours": { + "type": "integer", + "minimum": 0, + "maximum": 87600, + "description": "How far back the trace window reaches from its end, capped at ten years (project.MaxLookbackHours). Zero means unset. Only for type: traces." + }, + "max_traces": { + "type": "integer", + "minimum": 0, + "description": "Cap on traces read. Only for type: traces. Removing it restores the service default." + }, + "agent_name": { + "type": "string", + "description": "Name of the agent whose traces are read. Only for type: traces." + }, + "agent_version": { + "type": "string", + "description": "Pin the traces to one agent version." + }, + "response_ids": { + "type": "array", + "items": { "type": "string" }, + "description": "Explicit response ids to grade. Only for type: responses." + }, + "max_turns": { + "type": "integer", + "minimum": 0, + "description": "Cap on turns taken from each conversation." + }, + "start_time": { + "type": "string", + "description": "Start of the window, RFC 3339. With end_time this pins an absolute window instead of a lookback." + }, + "end_time": { + "type": "string", + "description": "End of the window, RFC 3339." + } + } + }, + "EvaluatorRef": { + "type": "object", + "required": ["evaluator"], + "additionalProperties": false, + "properties": { + "evaluator": { + "type": "string", + "description": "builtin. for a built-in, or the name of an entry in evaluators[]." + }, + "name": { + "type": "string", + "description": "Label for this evaluator within the eval, when the same evaluator is used more than once." + }, + "version": { + "type": "string", + "description": "Pin this eval to one evaluator version, so a later publish does not change what it grades." + }, + "initialization_parameters": { + "type": "object", + "additionalProperties": true, + "description": "Passed to the evaluator when the run starts; model names the judge deployment." + }, + "data_mapping": { + "type": "object", + "additionalProperties": { "type": "string" }, + "description": "Binds the evaluator's required inputs to dataset columns, for inputs beyond the agent shape such as ground_truth or context." + } + } + }, + "Target": { + "type": "object", + "required": ["type", "name"], + "additionalProperties": false, + "properties": { + "type": { + "type": "string", + "enum": ["agent", "model"], + "description": "What kind of thing is invoked." + }, + "name": { + "type": "string", + "description": "Name of the agent service or the model deployment to invoke." + } + } + } + } +} diff --git a/cli/azd/extensions/azure.ai.evaluations/schemas/examples/inline.azure.yaml b/cli/azd/extensions/azure.ai.evaluations/schemas/examples/inline.azure.yaml new file mode 100644 index 00000000000..8f191be32a5 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/schemas/examples/inline.azure.yaml @@ -0,0 +1,58 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/Azure/azure-dev/main/schemas/v1.0/azure.yaml.json +# The same configuration written inline, which is what the $ref form resolves to. +name: support-agent-app + +services: + support-agent-evals: + host: azure.ai.eval + uses: [ai-project, support-agent] + + datasets: + - name: support-agent-regression + file: ./datasets/support-agent-regression.jsonl + - name: prod-golden + file: ./datasets/prod-golden.jsonl + version: "2" + + evaluators: + - name: support-agent-quality + source: ./evaluators/support-agent-quality.json + # Or pulled in from its own file, the way agents and projects do it. + - $ref: ./evaluators/tone.yaml + name: tone + # Or written out here, which is also what a $ref to a rubric resolves to. + - name: brevity + definition: + type: rubric + dimensions: + - name: length + weight: 1 + description: Answers the question without restating it. + + evals: + - name: support-agent-regression-eval + dataset: support-agent-regression + evaluation_level: turn + max_samples: 50 + evaluators: + - evaluator: builtin.task_adherence + initialization_parameters: + model: gpt-5.6-luna + - evaluator: support-agent-quality + version: "2" + data_mapping: + ground_truth: "{{item.expected}}" + target: + type: agent + name: support-agent + + - name: support-agent-trace-eval + source: + type: traces + agent_name: support-agent + lookback_hours: 24 + max_traces: 20 + evaluators: + - evaluator: builtin.task_adherence + initialization_parameters: + model: gpt-5.6-luna diff --git a/cli/azd/extensions/azure.ai.evaluations/schemas/examples/ref.azure.yaml b/cli/azd/extensions/azure.ai.evaluations/schemas/examples/ref.azure.yaml new file mode 100644 index 00000000000..695b8dae4a0 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/schemas/examples/ref.azure.yaml @@ -0,0 +1,10 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/Azure/azure-dev/main/schemas/v1.0/azure.yaml.json +# The shape `azd ai eval init` writes: the eval body lives in its own file, and +# relative paths inside that file resolve against the file, not the project root. +name: support-agent-app + +services: + support-agent-evals: + host: azure.ai.eval + uses: [ai-project, support-agent] + $ref: ./evals/azure.eval.yaml diff --git a/cli/azd/extensions/azure.ai.evaluations/tests/hero/init_test.go b/cli/azd/extensions/azure.ai.evaluations/tests/hero/init_test.go index 1ab00592ce1..2f839f02319 100644 --- a/cli/azd/extensions/azure.ai.evaluations/tests/hero/init_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/tests/hero/init_test.go @@ -383,6 +383,6 @@ func TestHeroInitSuppliedDatasetIsNotGenerated(t *testing.T) { body, err := os.ReadFile(filepath.Join(dir, "evals", "azure.eval.yaml")) require.NoError(t, err) require.Contains(t, string(body), "dataset: prod-golden") - require.NotContains(t, string(body), "source:", + require.NotContains(t, string(body), "file:", "a registered dataset has nothing to upload") } diff --git a/schemas/alpha/azure.yaml.json b/schemas/alpha/azure.yaml.json index d103b88fb2a..17190ce1973 100644 --- a/schemas/alpha/azure.yaml.json +++ b/schemas/alpha/azure.yaml.json @@ -253,7 +253,8 @@ "azure.ai.connection", "azure.ai.toolbox", "azure.ai.skill", - "azure.ai.routine" + "azure.ai.routine", + "azure.ai.eval" ] }, "language": { @@ -557,6 +558,26 @@ } } }, + { + "comment": "Azure AI Foundry evaluation host - code-less resource service; the service key is the evaluation configuration name and it uses: the project and whatever it grades", + "if": { + "properties": { + "host": { "const": "azure.ai.eval" } + } + }, + "then": { + "allOf": [ + { "$ref": "https://raw.githubusercontent.com/Azure/azure-dev/main/cli/azd/extensions/azure.ai.evaluations/schemas/azure.ai.eval.json" } + ], + "properties": { + "project": false, + "runtime": false, + "docker": false, + "image": false, + "config": false + } + } + }, { "comment": "Legacy Microsoft Foundry host - compatibility for old non-network files; new provisioning uses azure.ai.project", "if": { diff --git a/schemas/v1.0/azure.yaml.json b/schemas/v1.0/azure.yaml.json index f7a7c9b5895..76a057e3e8e 100644 --- a/schemas/v1.0/azure.yaml.json +++ b/schemas/v1.0/azure.yaml.json @@ -214,7 +214,8 @@ "azure.ai.connection", "azure.ai.toolbox", "azure.ai.skill", - "azure.ai.routine" + "azure.ai.routine", + "azure.ai.eval" ] }, "language": { @@ -517,6 +518,26 @@ } } }, + { + "comment": "Azure AI Foundry evaluation host - code-less resource service; the service key is the evaluation configuration name and it uses: the project and whatever it grades", + "if": { + "properties": { + "host": { "const": "azure.ai.eval" } + } + }, + "then": { + "allOf": [ + { "$ref": "https://raw.githubusercontent.com/Azure/azure-dev/main/cli/azd/extensions/azure.ai.evaluations/schemas/azure.ai.eval.json" } + ], + "properties": { + "project": false, + "runtime": false, + "docker": false, + "image": false, + "config": false + } + } + }, { "comment": "Legacy Microsoft Foundry host - compatibility for old non-network files; new provisioning uses azure.ai.project", "if": { From 2c0509e068c901e07fe0bfc7728e6372a15fa3ed Mon Sep 17 00:00:00 2001 From: Mohamed Hessien Date: Thu, 20 Aug 2026 20:24:35 +0300 Subject: [PATCH 312/320] Publish an evaluator that carries its own rubric (#6) Both publish loops selected on 'source' alone, and validation guarantees a rubric written under 'definition' comes with no source. So a ref to a rubric decoded, validated, reported nothing and published nothing, and the eval was then created against an evaluator the service had never been told about. The whole feature was inert. Every test for it stopped at decoding, which is why none of them noticed. A test now asserts a carried rubric reaches the publish set. Also: the rescue is gated on the document actually using a ref, structurally, on both routes. The CLI's no-ref fast path skipped nesting while deploy still applied it, so a hand-written entry carrying rubric keys deployed and was then refused by every command that read it -- the same asymmetry a third time. The gate is structural rather than a byte scan, so a ref mentioned in a comment no longer changes an unrelated entry's meaning. The one-resolver guard now scans the whole extension; it walked only its own package, which is not where a second caller would appear. --- .../internal/cmd/eval_group.go | 8 +- .../internal/project/eval_config.go | 7 +- .../internal/project/eval_config_store.go | 40 ++++++- .../internal/project/one_resolver_test.go | 5 +- .../internal/project/owned_evaluators_test.go | 104 ++++++++++++++++++ .../internal/project/ref_resolution_test.go | 8 +- .../internal/project/service_target_eval.go | 6 +- 7 files changed, 168 insertions(+), 10 deletions(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/project/owned_evaluators_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/eval_group.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/eval_group.go index 8298b0801ff..e255a4dfa25 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/eval_group.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/eval_group.go @@ -118,10 +118,14 @@ func newEvalCreateCommand() *cobra.Command { for _, ref := range eval.Evaluators { decl, ok := cfg.EvaluatorDeclaration(ref.Evaluator) // A built-in, or one already registered, has nothing local to publish. - if !ok || decl.Source == "" { + if !ok || (decl.Source == "" && decl.Definition == nil) { continue } - local := project.ResolveSource(baseDir, decl.Source) + // A rubric written out in the configuration has no file to read. + local := "" + if decl.Source != "" { + local = project.ResolveSource(baseDir, decl.Source) + } version, changed, err := reconciler.EnsureEvaluator(ctx, *decl, local) if err != nil { return messages.EvaluatorProblem(decl.Name, err) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go index 4e46395be50..efc12749a92 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go @@ -207,12 +207,13 @@ func (c *EvalConfig) EvaluatorDeclaration(name string) (*EvaluatorDecl, bool) { return nil, false } -// CustomEvaluators are the catalog entries this configuration owns — the ones -// carrying a local source, published before the evals that name them. +// CustomEvaluators are the catalog entries this configuration owns -- the ones +// carrying a rubric, either as a local source or written out under +// `definition`, published before the evals that name them. func (c *EvalConfig) CustomEvaluators() []EvaluatorDecl { var owned []EvaluatorDecl for _, decl := range c.Evaluators { - if decl.Source == "" { + if decl.Source == "" && decl.Definition == nil { continue } owned = append(owned, decl) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_store.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_store.go index 258f9077de0..89f658caa4b 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_store.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_store.go @@ -236,6 +236,13 @@ func resolveConfigRefs(data []byte, baseDir, name string) ([]byte, error) { // CLI command refused, and later the reverse. Callers differ in how they obtain // the map and what they do with it; everything between is here. func resolveEvalRefs(values map[string]any, baseDir string) (map[string]any, error) { + // Read before resolution, which removes the directive. Both routes gate the + // rescue on it so they cannot disagree: without it, the CLI's no-`$ref` fast + // path would skip nesting while the deploy path still applied it, and a + // hand-written entry carrying rubric keys would deploy and then be refused + // by every command that reads it. + spliced := containsRefDirective(values) + resolved, err := foundry.ResolveFileRefs(values, baseDir) if err != nil { return nil, messages.ResolvingServiceRefs(err) @@ -243,10 +250,41 @@ func resolveEvalRefs(values map[string]any, baseDir string) (map[string]any, err // `$ref` is a directive rather than configuration, and the strict decoder // would report the leftover as a mistyped key. delete(resolved, "$ref") - nestSplicedRubrics(resolved) + if spliced { + nestSplicedRubrics(resolved) + } return resolved, nil } +// containsRefDirective reports whether the document uses `$ref` anywhere. +// +// Structural rather than a text scan: the byte "$ref" also appears in comments +// and in prose values, and letting those decide whether an unrelated entry is +// rescued would make one entry's meaning depend on another's wording. +func containsRefDirective(value any) bool { + switch typed := value.(type) { + case map[string]any: + if _, ok := typed[refDirective]; ok { + return true + } + for _, child := range typed { + if containsRefDirective(child) { + return true + } + } + case []any: + for _, child := range typed { + if containsRefDirective(child) { + return true + } + } + } + return false +} + +// refDirective is the include key azd core owns. +const refDirective = "$ref" + // evaluatorDeclKeys are the keys an evaluator entry declares in its own right. // Anything else at entry level was spliced in by a `$ref`. var evaluatorDeclKeys = map[string]bool{ diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/one_resolver_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/one_resolver_test.go index d2c30404750..a9286cd9a39 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/one_resolver_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/one_resolver_test.go @@ -24,7 +24,10 @@ func TestRefsAreResolvedInOnePlace(t *testing.T) { const resolver = "resolveEvalRefs" callers := map[string][]int{} - require.NoError(t, filepath.WalkDir(".", func(path string, d os.DirEntry, err error) error { + // The whole extension, not this package: both current routes already live + // here, so the plausible place for a second caller is internal/cmd, where a + // command wanting resolution would reach for the helper directly. + require.NoError(t, filepath.WalkDir("../..", func(path string, d os.DirEntry, err error) error { if err != nil || d.IsDir() || !strings.HasSuffix(path, ".go") { return err } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/owned_evaluators_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/owned_evaluators_test.go new file mode 100644 index 00000000000..28e6494ffbd --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/owned_evaluators_test.go @@ -0,0 +1,104 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package project + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// An evaluator carrying its rubric is one this configuration owns, so it has to +// reach the publish loops. +// +// Both loops selected on `source` alone, and validation guarantees a rubric +// written under `definition` comes with no source. So a `$ref` to a rubric +// decoded, validated, reported nothing, and published nothing: the eval was then +// created against an evaluator the service had never been told about. Every test +// for that feature stopped at decoding, which is why none of them noticed. +func TestAnEvaluatorCarryingItsRubricIsPublished(t *testing.T) { + dir := t.TempDir() + + require.NoError(t, os.MkdirAll(filepath.Join(dir, "evaluators"), 0o755)) + require.NoError(t, os.WriteFile( + filepath.Join(dir, "evaluators", "quality.json"), + []byte(`{"type":"rubric","dimensions":[{"id":"tone","weight":3}]}`), + 0o600)) + + path := filepath.Join(dir, EvalConfigBase) + require.NoError(t, os.WriteFile(path, []byte(` +evaluators: + - $ref: ./evaluators/quality.json + name: quality + - name: from-a-file + source: ./evaluators/quality.json + - name: builtin.relevance-is-not-ours + version: "3" + +evals: + - name: nightly + dataset: golden +`), 0o600)) + + cfg, err := LoadEvalConfig(path) + require.NoError(t, err) + + var owned []string + for _, decl := range cfg.CustomEvaluators() { + owned = append(owned, decl.Name) + } + + assert.Contains(t, owned, "quality", + "a rubric this configuration carries is one it owns, so it must be published") + assert.Contains(t, owned, "from-a-file") + assert.NotContains(t, owned, "builtin.relevance-is-not-ours", + "an entry that only pins a registered version has nothing to publish") +} + +// The rescue is gated on the document actually using `$ref`, and both routes +// have to gate on it the same way. +// +// The CLI returns a configuration with no `$ref` untouched so the decoder keeps +// its own line numbers. That fast path skipped the rescue while the deploy route +// still applied it, so a hand-written entry carrying rubric keys deployed and was +// then refused by every command that read it -- the same asymmetry twice over. +func TestRubricKeysWithoutARefAreRefusedOnBothRoutes(t *testing.T) { + dir := t.TempDir() + + body := ` +evaluators: + - name: quality + type: rubric + dimensions: + - id: tone + weight: 3 + +evals: + - name: nightly + dataset: golden +` + path := filepath.Join(dir, EvalConfigBase) + require.NoError(t, os.WriteFile(path, []byte(body), 0o600)) + + _, fromDisk := LoadEvalConfig(path) + require.Error(t, fromDisk, "rubric keys nobody spliced are a mistake, not a rubric") + assert.Contains(t, fromDisk.Error(), "dimensions") + + svc := serviceWith(t, map[string]any{ + "evaluators": []any{map[string]any{ + "name": "quality", + "type": "rubric", + "dimensions": []any{ + map[string]any{"id": "tone", "weight": 3}, + }, + }}, + "evals": []any{map[string]any{"name": "nightly", "dataset": "golden"}}, + }) + _, fromService := EvalConfigFromService(svc, dir) + require.Error(t, fromService, "`azd up` has to refuse what every command refuses") + assert.Contains(t, fromService.Error(), "dimensions") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/ref_resolution_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/ref_resolution_test.go index 71e0d2978c1..71c372f124c 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/ref_resolution_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/ref_resolution_test.go @@ -168,8 +168,12 @@ evals: assert.Len(t, cfg.Evaluators[0].Definition["dimensions"], 1) } -// The rescue above is scoped to entries written as a `$ref`, so a misspelling -// in a hand-written entry is still an error rather than rubric content. +// A configuration that uses no `$ref` is never rescued, so a misspelling in a +// hand-written entry is reported rather than filed away. +// +// The `dimensions` gate is what separates the rescue from a catch-all, and it is +// exercised in nested_ref_rubric_test.go; this pins the other half, that a +// document nobody spliced into is left strictly alone. func TestAMisspelledEvaluatorKeyIsStillRejected(t *testing.T) { dir := t.TempDir() diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go index 539717d07cc..3e5a8a70ffa 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go @@ -173,7 +173,11 @@ func (p *EvalServiceTargetProvider) Deploy( // ones need no publish. for _, decl := range cfg.CustomEvaluators() { report(progress, messages.ReconcilingEvaluator(decl.Name)) - localPath := ResolveSource(baseDir, decl.Source) + // A rubric written out in the configuration has no file to read. + localPath := "" + if decl.Source != "" { + localPath = ResolveSource(baseDir, decl.Source) + } version, changed, err := reconciler.EnsureEvaluator(ctx, decl, localPath) if err != nil { return nil, messages.EvaluatorProblem(decl.Name, err) From 6cf1e9b6a8db9fdf810393468003d9550ba2ca87 Mon Sep 17 00:00:00 2001 From: mohessie Date: Thu, 20 Aug 2026 20:45:38 +0300 Subject: [PATCH 313/320] Cut the thirteenth bug bash build --- cli/azd/extensions/azure.ai.evaluations/CHANGELOG.md | 2 +- cli/azd/extensions/azure.ai.evaluations/extension.yaml | 2 +- cli/azd/extensions/azure.ai.evaluations/version.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cli/azd/extensions/azure.ai.evaluations/CHANGELOG.md b/cli/azd/extensions/azure.ai.evaluations/CHANGELOG.md index 532ebf558da..6e9341806f5 100644 --- a/cli/azd/extensions/azure.ai.evaluations/CHANGELOG.md +++ b/cli/azd/extensions/azure.ai.evaluations/CHANGELOG.md @@ -1,6 +1,6 @@ # Release History -## 1.0.16-beta (Unreleased) +## 1.0.17-beta (Unreleased) First release of the Foundry evaluations extension. diff --git a/cli/azd/extensions/azure.ai.evaluations/extension.yaml b/cli/azd/extensions/azure.ai.evaluations/extension.yaml index 8032f9bfb43..2699d62da5f 100644 --- a/cli/azd/extensions/azure.ai.evaluations/extension.yaml +++ b/cli/azd/extensions/azure.ai.evaluations/extension.yaml @@ -5,7 +5,7 @@ displayName: Foundry evaluations (Beta) description: Define and run Foundry evaluations from your terminal. (Beta) usage: azd ai eval [options] # NOTE: Make sure version.txt is in sync with this version. -version: 1.0.16-beta +version: 1.0.17-beta requiredAzdVersion: ">=1.27.1" language: go capabilities: diff --git a/cli/azd/extensions/azure.ai.evaluations/version.txt b/cli/azd/extensions/azure.ai.evaluations/version.txt index 7ae7d4397e7..0b7583d8244 100644 --- a/cli/azd/extensions/azure.ai.evaluations/version.txt +++ b/cli/azd/extensions/azure.ai.evaluations/version.txt @@ -1 +1 @@ -1.0.16-beta +1.0.17-beta From 974d0027a66765ae5ea70830286b7a186a3fe35e Mon Sep 17 00:00:00 2001 From: Mohamed Hessien Date: Fri, 21 Aug 2026 00:06:21 +0300 Subject: [PATCH 314/320] Model the include on every entry core can splice (#7) * Model the include on every entry core can splice Core resolves a ref on every node, but only the evaluator entry modelled it. A configuration with a ref on a dataset or an eval deployed and ran fine, and could not be opened by init, generate or the catalog writers at all: they read the file exactly as written, and the strict decoder refused the very key that pointed at the content. The schema said the same, so an editor underlined it too. Also: an include reached without a project root is refused rather than discarded. Resolution was skipped when there was nowhere to resolve against and the directive was then deleted, so a service mixing an include with inline content deployed only the inline half and failed later as a missing eval. And a relative source inside a refd file is named where it bites. Core rebases only the two path keys it owns, so such a path arrives verbatim and resolves against the configuration rather than the referenced file. The failure said the definition had not been generated; it now says why the path is wrong and what to do instead. * Refuse a catalog name an include already declares The editing read sees the directive, not the entry behind it, so the duplicate scan had nothing to match on and generate appended a second entry with the same name. The collision then surfaced on the next resolving read, naming a duplicate the author never wrote and could not see in the file in front of them. A configuration that will not resolve is left to the commands that resolve it: failing a generate over an unrelated broken include would be its own surprise. Also corrects the README, which said relative paths inside a refd file resolve against that file. True of the configuration as a whole, false for a ref on a single entry, which is exactly the trap. It now says so and steers to definition. * Refuse an include that carries its own name, too The check only caught a pure ref, where no name is visible. An entry written as a ref with an overlay name -- the shape the README recommends for a rubric -- matched by name and was updated in place, writing source beside the directive. Resolution then produced both a spliced rubric and a source, and the next read rejected the catalog for declaring the rubric twice. Neither shape is editable here. Datasets get the same path diagnostic evaluators got: a relative file inside a refd entry resolves against the configuration, not against that file. And the new include test no longer blesses that broken shape. It wrote the dataset path beside the referenced file and asserted only the name, which is the same silent endorsement this round criticised elsewhere. --- .../extensions/azure.ai.evaluations/README.md | 14 ++- .../internal/cmd/catalog.go | 49 ++++++++++ .../internal/cmd/catalog_include_test.go | 93 +++++++++++++++++++ .../internal/messages/messages.go | 31 ++++++- .../internal/project/config_keys_test.go | 8 +- .../internal/project/eval_config.go | 18 +++- .../project/ref_on_every_entry_test.go | 60 ++++++++++++ .../internal/project/ref_resolution_test.go | 8 +- .../project/service_config_strict_test.go | 20 ++-- .../internal/project/service_target_eval.go | 7 ++ .../schemas/azure.ai.eval.json | 18 +++- 11 files changed, 302 insertions(+), 24 deletions(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/catalog_include_test.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/project/ref_on_every_entry_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/README.md b/cli/azd/extensions/azure.ai.evaluations/README.md index dcdde896504..ee4faf9de69 100644 --- a/cli/azd/extensions/azure.ai.evaluations/README.md +++ b/cli/azd/extensions/azure.ai.evaluations/README.md @@ -53,9 +53,21 @@ evals: `azd up` reconciles **datasets → evaluators → eval groups**, in that order, because a group references the versions the first two resolve to. -Relative paths inside a `$ref`'d file resolve against **that file's** +Relative paths inside the `$ref`'d configuration resolve against **that file's** directory, so `./datasets/x.jsonl` above means `evals/datasets/x.jsonl`. +That holds for the configuration as a whole. It does **not** hold for a `$ref` +on a single catalog entry: azd rebases only the path keys it owns, so a relative +`source:` written inside `evals/evaluators/quality.yaml` still resolves against +`azure.eval.yaml` and will not be found. An entry pulled in from its own file +should carry the rubric under `definition:` rather than point at a second file: + +```yaml +evaluators: + - $ref: ./evaluators/quality.json # the rubric itself, not a pointer to one + name: quality +``` + ### Repeated deploys do not create redundant versions Datasets are fingerprinted locally, because the dataset API exposes no content diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/catalog.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/catalog.go index 0b8acd61887..60b1db39cbf 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/catalog.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/catalog.go @@ -66,6 +66,52 @@ func addEvaluatorToCatalog(cmd *cobra.Command, evalDir string, ref *project.Arti }) } +// checkNameNotBehindAnInclude refuses a name whose entry lives in another file. +// +// Two shapes reach this. A pure `$ref` has no name here at all, so the duplicate +// scan had nothing to match on and appended a second entry with the same name -- +// a collision that surfaced only on the next resolving read. A `$ref` carrying +// an overlay `name` does match, and updating it in place writes `source:` beside +// the directive, so resolution then produces a rubric and a source and the +// configuration is rejected for declaring it twice. Neither is editable here. +// +// A configuration that will not resolve is left to the commands that resolve it: +// failing a generate over an unrelated broken include would be its own surprise. +func checkNameNotBehindAnInclude(evalDir string, asWritten *project.EvalConfig, kind, name string) error { + if ref, ok := catalogEntryRef(asWritten, kind, name); ok { + if ref != "" { + return messages.CatalogNameBehindAnInclude(kind, name) + } + return nil + } + resolved, err := project.OpenEvalConfig(evalDir) + if err != nil || resolved == nil { + return nil + } + if _, ok := catalogEntryRef(resolved, kind, name); ok { + return messages.CatalogNameBehindAnInclude(kind, name) + } + return nil +} + +// catalogEntryRef returns the include this entry was written as, and whether the +// configuration names it at all. +func catalogEntryRef(cfg *project.EvalConfig, kind, name string) (string, bool) { + if cfg == nil { + return "", false + } + if kind == "dataset" { + if decl, ok := cfg.DatasetDeclaration(name); ok { + return decl.Ref, true + } + return "", false + } + if decl, ok := cfg.EvaluatorDeclaration(name); ok { + return decl.Ref, true + } + return "", false +} + // updateCatalog applies a change to the configuration and writes it back. // // A missing configuration is created holding only the catalog. `generate` runs @@ -96,6 +142,9 @@ func updateCatalog( if created { cfg = &project.EvalConfig{} } + if err := checkNameNotBehindAnInclude(evalDir, cfg, kind, ref.Name); err != nil { + return err + } if !apply(cfg) { return nil } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/catalog_include_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/catalog_include_test.go new file mode 100644 index 00000000000..f6a1a68cedd --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/catalog_include_test.go @@ -0,0 +1,93 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "os" + "path/filepath" + "testing" + + "azureaieval/internal/project" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// A name declared through a `$ref` is refused, not appended alongside. +// +// The editing read sees the directive rather than the entry behind it, so the +// duplicate scan had nothing to match on and appended a second entry with the +// same name. The collision then surfaced on the next resolving read, naming a +// duplicate the author never wrote and could not see in the file in front of +// them. +func TestGenerateRefusesANameAnIncludeAlreadyDeclares(t *testing.T) { + dir := t.TempDir() + require.NoError(t, os.MkdirAll(filepath.Join(dir, "parts"), 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(dir, "parts", "quality.yaml"), + []byte("name: quality\nsource: ./quality.json\n"), 0o600)) + + require.NoError(t, os.WriteFile(filepath.Join(dir, project.EvalConfigBase), []byte(` +evaluators: + - $ref: ./parts/quality.yaml +`), 0o600)) + + err := checkNameNotBehindAnInclude( + dir, mustOpenForEdit(t, dir), "evaluator", "quality") + + require.Error(t, err, "the name is taken, even though this file does not show it") + assert.Contains(t, err.Error(), "quality") + assert.Contains(t, err.Error(), "$ref", "the reader has to be told where the name lives") +} + +// A name nothing declares is still free, so generation is not blocked by the +// mere presence of an include elsewhere. +func TestGenerateStillAddsANameNobodyDeclares(t *testing.T) { + dir := t.TempDir() + require.NoError(t, os.MkdirAll(filepath.Join(dir, "parts"), 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(dir, "parts", "quality.yaml"), + []byte("name: quality\nsource: ./quality.json\n"), 0o600)) + + require.NoError(t, os.WriteFile(filepath.Join(dir, project.EvalConfigBase), []byte(` +evaluators: + - $ref: ./parts/quality.yaml +`), 0o600)) + + require.NoError(t, checkNameNotBehindAnInclude( + dir, mustOpenForEdit(t, dir), "evaluator", "tone")) +} + +// An include carrying an overlay `name` is refused too, even though the name is +// right there in the file. +// +// This is the shape the README recommends for a rubric. Updating it in place +// writes `source:` beside the directive, and resolution then yields both a +// spliced rubric and a source -- a catalog the next read rejects for declaring +// the rubric twice. The name being visible is what made this the easier one to +// miss. +func TestGenerateRefusesAnIncludeThatCarriesItsName(t *testing.T) { + dir := t.TempDir() + require.NoError(t, os.MkdirAll(filepath.Join(dir, "evaluators"), 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(dir, "evaluators", "quality.json"), + []byte(`{"type":"rubric","dimensions":[{"id":"tone","weight":3}]}`), 0o600)) + + require.NoError(t, os.WriteFile(filepath.Join(dir, project.EvalConfigBase), []byte(` +evaluators: + - $ref: ./evaluators/quality.json + name: quality +`), 0o600)) + + err := checkNameNotBehindAnInclude( + dir, mustOpenForEdit(t, dir), "evaluator", "quality") + + require.Error(t, err, "the entry is an include, so it cannot be updated in place") + assert.Contains(t, err.Error(), "quality") +} + +func mustOpenForEdit(t *testing.T, dir string) *project.EvalConfig { + t.Helper() + cfg, err := project.OpenEvalConfigForEdit(dir) + require.NoError(t, err) + require.NotNil(t, cfg) + return cfg +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go b/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go index 26f5d2e0506..2c66e9469a0 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go @@ -1057,7 +1057,10 @@ func DatasetNotGeneratedYet(dataset, path string) error { return fmt.Errorf( "its rows %s have not been generated yet. "+ "Run `azd ai eval generate --dataset --dataset-name %s` to write them, "+ - "or point the declaration at a .jsonl you already have", + "or point the declaration at a .jsonl you already have. "+ + "If this entry came from a `$ref`, note that a relative `file:` inside "+ + "the referenced file resolves against azure.eval.yaml rather than against "+ + "that file -- write the path relative to the configuration instead", filepath.ToSlash(path), shellArg(dataset)) } @@ -1222,7 +1225,10 @@ func EvaluatorNotGeneratedYet(evaluator, path string) error { return fmt.Errorf( "its definition %s has not been generated yet. "+ "Run `azd ai eval generate --evaluator --evaluator-name %s` to write it, "+ - "or drop the evaluator from azure.eval.yaml", + "or drop the evaluator from azure.eval.yaml. "+ + "If this entry came from a `$ref`, note that a relative `source:` inside "+ + "the referenced file resolves against azure.eval.yaml rather than against "+ + "that file -- carry the rubric under `definition:` instead", filepath.ToSlash(path), shellArg(evaluator)) } @@ -1414,6 +1420,27 @@ func ResolvingServiceRefs(err error) error { return fmt.Errorf("resolving $ref in the eval service configuration: %w", err) } +// RefNeedsAProjectRoot reports an include reached without a directory to +// resolve it against. +func RefNeedsAProjectRoot(service string) error { + return fmt.Errorf( + "service %q uses `$ref`, but the project directory could not be determined, "+ + "so the referenced file cannot be found. Run from inside the azd project, "+ + "or inline the configuration this service points at", + service) +} + +// CatalogNameBehindAnInclude reports a name declared through a `$ref`, which +// this command cannot edit in place. +func CatalogNameBehindAnInclude(kind, name string) error { + return fmt.Errorf( + "%s %q is already declared through a `$ref`, so this command cannot update "+ + "it here: adding a second entry would collide with the first only after "+ + "the include is resolved. Edit the referenced file, or generate under a "+ + "different name", + kind, name) +} + // ReadingServiceConfig reports the service entry failing to serialize. func ReadingServiceConfig(err error) error { return fmt.Errorf("reading the eval service configuration: %w", err) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/config_keys_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/config_keys_test.go index 22fb17482de..f109334026a 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/config_keys_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/config_keys_test.go @@ -51,11 +51,12 @@ func TestEvalConfigKeys(t *testing.T) { "the top-level shape is the spec's configuration model") } -// An eval names what it evaluates, what it reads, and how to grade it. +// An eval names what it evaluates, what it reads, and how to grade it, or is +// pulled in whole with `$ref`. func TestEvalKeys(t *testing.T) { assert.ElementsMatch(t, []string{ - "name", "id", "description", "dataset", "source", + "$ref", "name", "id", "description", "dataset", "source", "evaluation_level", "max_samples", "evaluators", "target", }, yamlKeys(t, Eval{})) @@ -87,7 +88,8 @@ func TestSourceDeclKeys(t *testing.T) { // and saves the file writes the author's include back out instead of inlining // it. func TestCatalogKeys(t *testing.T) { - assert.ElementsMatch(t, []string{"name", "file", "version"}, yamlKeys(t, DatasetDecl{})) + assert.ElementsMatch(t, + []string{"$ref", "name", "file", "version"}, yamlKeys(t, DatasetDecl{})) assert.ElementsMatch(t, []string{"$ref", "name", "source", "version", "definition"}, yamlKeys(t, EvaluatorDecl{})) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go index efc12749a92..90e6a9d856c 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go @@ -29,11 +29,18 @@ type EvalConfig struct { // DatasetDecl is a catalog entry. A local File is uploaded on deploy; without // one the name must already resolve to a registered dataset. // -// Deliberately not a `$ref`: that directive replaces a definition with one -// loaded from a YAML or JSON file, and these rows are an artifact to publish. -// A `.jsonl` is neither, so there would be nothing to splice. +// Deliberately no `$ref` in place of `file`: that directive replaces a +// definition with one loaded from a YAML or JSON file, and these rows are an +// artifact to publish. A `.jsonl` is neither, so there would be nothing to +// splice. +// +// Ref is the directive an author may still write to load this whole entry from +// its own file. Core resolves it on every node, so the commands that use the +// configuration never see it -- but the ones that read, modify and save do, and +// modelling it is what keeps them from refusing a file `azd up` accepts. type DatasetDecl struct { - Name string `yaml:"name" json:"name"` + Ref string `yaml:"$ref,omitempty" json:"$ref,omitempty"` + Name string `yaml:"name,omitempty" json:"name,omitempty"` File string `yaml:"file,omitempty" json:"file,omitempty"` Version string `yaml:"version,omitempty" json:"version,omitempty"` } @@ -70,7 +77,8 @@ type EvaluatorDecl struct { // from a source such as production traces. Target is what gets invoked, and is // a separate axis — an eval can read traces and invoke nothing. type Eval struct { - Name string `yaml:"name" json:"name"` + Ref string `yaml:"$ref,omitempty" json:"$ref,omitempty"` + Name string `yaml:"name,omitempty" json:"name,omitempty"` ID string `yaml:"id,omitempty" json:"id,omitempty"` Description string `yaml:"description,omitempty" json:"description,omitempty"` Dataset string `yaml:"dataset,omitempty" json:"dataset,omitempty"` diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/ref_on_every_entry_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/ref_on_every_entry_test.go new file mode 100644 index 00000000000..648378fba83 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/ref_on_every_entry_test.go @@ -0,0 +1,60 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package project + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// Core resolves `$ref` on every node, not just evaluator entries, so a dataset +// or an eval can be pulled in from its own file too. +// +// Only the evaluator entry modelled the directive, so a configuration that +// deployed and ran fine could not be opened by `init`, `generate` or the catalog +// writers at all: they read the file exactly as written, and the strict decoder +// refused the very key that pointed at the content. +func TestEveryEntryCoreCanSpliceCanAlsoBeEdited(t *testing.T) { + dir := t.TempDir() + require.NoError(t, os.MkdirAll(filepath.Join(dir, "parts"), 0o755)) + + require.NoError(t, os.WriteFile(filepath.Join(dir, "parts", "golden.yaml"), + // Relative to the configuration, not to this file: core rebases only the + // path keys it owns, so a path written beside this file would be resolved + // beside azure.eval.yaml and not found. + []byte("name: golden\nfile: ./datasets/golden.jsonl\n"), 0o600)) + require.NoError(t, os.WriteFile(filepath.Join(dir, "parts", "nightly.yaml"), + []byte("name: nightly\ndataset: golden\n"), 0o600)) + + path := filepath.Join(dir, EvalConfigBase) + require.NoError(t, os.WriteFile(path, []byte(` +datasets: + - $ref: ./parts/golden.yaml + +evals: + - $ref: ./parts/nightly.yaml +`), 0o600)) + + resolved, err := LoadEvalConfig(path) + require.NoError(t, err, "the resolving route has always accepted this") + require.Len(t, resolved.Datasets, 1) + assert.Equal(t, "golden", resolved.Datasets[0].Name) + assert.Equal(t, "./datasets/golden.jsonl", resolved.Datasets[0].File, + "the spliced path is resolved against the configuration, so it is written that way") + require.Len(t, resolved.Evals, 1) + assert.Equal(t, "nightly", resolved.Evals[0].Name) + + asWritten, err := OpenEvalConfigForEdit(dir) + require.NoError(t, err, + "a command that edits the file has to be able to open what azd up deploys") + require.Len(t, asWritten.Datasets, 1) + assert.Equal(t, "./parts/golden.yaml", asWritten.Datasets[0].Ref, + "the include survives an editing read, so saving writes it back") + require.Len(t, asWritten.Evals, 1) + assert.Equal(t, "./parts/nightly.yaml", asWritten.Evals[0].Ref) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/ref_resolution_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/ref_resolution_test.go index 71c372f124c..7e6a2b9a9f9 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/ref_resolution_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/ref_resolution_test.go @@ -47,7 +47,13 @@ evals: require.Len(t, cfg.Evaluators, 1) assert.Equal(t, "support-agent-quality", cfg.Evaluators[0].Name, "the referenced file's content replaces the directive") - assert.Equal(t, "./quality.json", cfg.Evaluators[0].Source) + // Verbatim, deliberately: core rebases only the two path keys it owns, so a + // relative `source:` written beside the referenced file arrives unchanged and + // is then resolved against azure.eval.yaml. That is a known limitation, not + // the behaviour this asserts -- carrying the rubric under `definition:` + // avoids it, and EvaluatorNotGeneratedYet names it when the path misses. + assert.Equal(t, "./quality.json", cfg.Evaluators[0].Source, + "a spliced path is not rebased; see the note above before changing this") } // A `$ref` can name the rubric itself, not only a pointer to one. diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/service_config_strict_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/service_config_strict_test.go index 8bb3dad0a04..ad54453ef9a 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/service_config_strict_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/service_config_strict_test.go @@ -62,11 +62,15 @@ func TestEvalConfigFromServiceAcceptsADeclaredConfig(t *testing.T) { assert.Equal(t, "golden", cfg.Datasets[0].Name) } -// `$ref` is a directive rather than configuration. ResolveFileRefs replaces it -// with the file's content, but it survives when resolution was skipped, and a -// strict decoder would then refuse a config for carrying the very thing that -// pointed at it. -func TestEvalConfigFromServiceIgnoresTheRefDirective(t *testing.T) { +// An include reached without a project directory is refused rather than +// discarded. +// +// Resolution was skipped when there was nowhere to resolve against, and the +// directive was then deleted so the strict decoder would not trip on it. This +// fixture shows the cost: a service mixing an include with inline content +// deployed only the inline half, and the failure surfaced as a missing eval +// rather than as the include nobody could read. +func TestARefWithoutAProjectRootIsRefused(t *testing.T) { svc := serviceWith(t, map[string]any{ "$ref": "./evals/azure.eval.yaml", "evals": []any{map[string]any{ @@ -75,8 +79,8 @@ func TestEvalConfigFromServiceIgnoresTheRefDirective(t *testing.T) { }}, }) - cfg, err := EvalConfigFromService(svc, "") + _, err := EvalConfigFromService(svc, "") - require.NoError(t, err) - require.Len(t, cfg.Evals, 1) + require.Error(t, err, "half a configuration is not a configuration") + assert.Contains(t, err.Error(), "$ref") } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go index 3e5a8a70ffa..dc6252471ee 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go @@ -299,6 +299,13 @@ func EvalConfigFromService(svc *azdext.ServiceConfig, projectRoot string) (*Eval return nil, err } values = resolved + } else if containsRefDirective(values) { + // Without a project root there is nothing to resolve paths against, and + // deleting the directive below would deploy a silently truncated + // configuration: an entry that mixes inline content with an include + // loses only the included half, and the failure then names a missing + // eval rather than the include nobody could read. + return nil, messages.RefNeedsAProjectRoot(svc.GetName()) } // `$ref` is a directive, not configuration: ResolveFileRefs has already diff --git a/cli/azd/extensions/azure.ai.evaluations/schemas/azure.ai.eval.json b/cli/azd/extensions/azure.ai.evaluations/schemas/azure.ai.eval.json index 711fb58d30a..0bcee881584 100644 --- a/cli/azd/extensions/azure.ai.evaluations/schemas/azure.ai.eval.json +++ b/cli/azd/extensions/azure.ai.evaluations/schemas/azure.ai.eval.json @@ -8,8 +8,13 @@ "properties": { "datasets": { "type": "array", - "description": "Catalog of named datasets. An entry with a file is uploaded on deploy; without one the name must already resolve to a registered dataset.", - "items": { "$ref": "#/definitions/DatasetDecl" } + "description": "Catalog of named datasets. An entry with a file is uploaded on deploy; without one the name must already resolve to a registered dataset. An entry may instead be a $ref to a file holding the declaration.", + "items": { + "oneOf": [ + { "$ref": "#/definitions/DatasetDecl" }, + { "$ref": "#/definitions/FileRef" } + ] + } }, "evaluators": { "type": "array", @@ -23,8 +28,13 @@ }, "evals": { "type": "array", - "description": "The evaluations defined over the catalogs. A list, because one target is normally gated by more than one evaluation.", - "items": { "$ref": "#/definitions/Eval" } + "description": "The evaluations defined over the catalogs. A list, because one target is normally gated by more than one evaluation. An entry may instead be a $ref to a file holding the eval.", + "items": { + "oneOf": [ + { "$ref": "#/definitions/Eval" }, + { "$ref": "#/definitions/FileRef" } + ] + } } }, "definitions": { From 7888d313470429ba1b6fd02b6c4b535769c43414 Mon Sep 17 00:00:00 2001 From: mohessie Date: Fri, 21 Aug 2026 01:25:57 +0300 Subject: [PATCH 315/320] Answer the review on the include work Refuse a catalog entry that already carries its rubric, share the ownership predicate the two publish loops had each re-derived, fetch the project root once, and correct the schema paths that told authors a spliced path is rebased. --- .../extensions/azure.ai.evaluations/README.md | 9 +- .../internal/cmd/catalog.go | 56 +++++++---- .../internal/cmd/catalog_include_test.go | 97 ++++++++++++++++++- .../internal/cmd/eval_group.go | 2 +- .../internal/messages/messages.go | 25 ++++- .../internal/project/eval_config.go | 16 ++- .../internal/project/one_ownership_test.go | 74 ++++++++++++++ .../internal/project/service_target_eval.go | 21 ++-- .../schemas/azure.ai.eval.json | 9 +- 9 files changed, 267 insertions(+), 42 deletions(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/project/one_ownership_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/README.md b/cli/azd/extensions/azure.ai.evaluations/README.md index ee4faf9de69..a27c5be2706 100644 --- a/cli/azd/extensions/azure.ai.evaluations/README.md +++ b/cli/azd/extensions/azure.ai.evaluations/README.md @@ -60,7 +60,9 @@ That holds for the configuration as a whole. It does **not** hold for a `$ref` on a single catalog entry: azd rebases only the path keys it owns, so a relative `source:` written inside `evals/evaluators/quality.yaml` still resolves against `azure.eval.yaml` and will not be found. An entry pulled in from its own file -should carry the rubric under `definition:` rather than point at a second file: +should carry the rubric rather than point at a second file — either written out +under `definition:`, or as a `$ref` straight at the rubric, whose keys are +spliced in and become that `definition:`: ```yaml evaluators: @@ -68,6 +70,11 @@ evaluators: name: quality ``` +An entry declared this way is read and deployed normally, but it lives in the +referenced file, so `azd ai eval generate` will not update it in place and says +so rather than writing a second declaration of the same rubric beside the +directive. Edit the referenced file, or generate under a different name. + ### Repeated deploys do not create redundant versions Datasets are fingerprinted locally, because the dataset API exposes no content diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/catalog.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/catalog.go index 60b1db39cbf..552fa2b4af1 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/catalog.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/catalog.go @@ -66,21 +66,29 @@ func addEvaluatorToCatalog(cmd *cobra.Command, evalDir string, ref *project.Arti }) } -// checkNameNotBehindAnInclude refuses a name whose entry lives in another file. +// checkCatalogEntryIsEditable refuses a name this command cannot rewrite in +// place without corrupting the entry. // -// Two shapes reach this. A pure `$ref` has no name here at all, so the duplicate -// scan had nothing to match on and appended a second entry with the same name -- -// a collision that surfaced only on the next resolving read. A `$ref` carrying -// an overlay `name` does match, and updating it in place writes `source:` beside -// the directive, so resolution then produces a rubric and a source and the -// configuration is rejected for declaring it twice. Neither is editable here. +// Three shapes reach this. A pure `$ref` has no name here at all, so the +// duplicate scan had nothing to match on and appended a second entry with the +// same name -- a collision that surfaced only on the next resolving read. A +// `$ref` carrying an overlay `name` does match, and updating it in place writes +// `source:` beside the directive, so resolution then produces a rubric and a +// source and the configuration is rejected for declaring it twice. An entry +// already carrying its rubric under `definition:` fails that same way with no +// include involved, because recording the generated file leaves both in one +// entry -- and it fails after the generation job has been billed and the file +// written, which is why it is refused here rather than left to the next read. // // A configuration that will not resolve is left to the commands that resolve it: // failing a generate over an unrelated broken include would be its own surprise. -func checkNameNotBehindAnInclude(evalDir string, asWritten *project.EvalConfig, kind, name string) error { - if ref, ok := catalogEntryRef(asWritten, kind, name); ok { - if ref != "" { +func checkCatalogEntryIsEditable(evalDir string, asWritten *project.EvalConfig, kind, name string) error { + if entry, ok := catalogEntryShapeOf(asWritten, kind, name); ok { + switch { + case entry.ref != "": return messages.CatalogNameBehindAnInclude(kind, name) + case entry.inlineRubric: + return messages.EvaluatorRubricWrittenInPlace(name) } return nil } @@ -88,28 +96,38 @@ func checkNameNotBehindAnInclude(evalDir string, asWritten *project.EvalConfig, if err != nil || resolved == nil { return nil } - if _, ok := catalogEntryRef(resolved, kind, name); ok { + if _, ok := catalogEntryShapeOf(resolved, kind, name); ok { return messages.CatalogNameBehindAnInclude(kind, name) } return nil } -// catalogEntryRef returns the include this entry was written as, and whether the +// catalogEntryShape is how an entry was written, for deciding whether this +// command may rewrite it. +type catalogEntryShape struct { + // ref is the `$ref` directive the entry carries, empty when it is written + // out here. + ref string + // inlineRubric is an evaluator holding its rubric under `definition:`. + inlineRubric bool +} + +// catalogEntryShapeOf returns how the entry was written, and whether the // configuration names it at all. -func catalogEntryRef(cfg *project.EvalConfig, kind, name string) (string, bool) { +func catalogEntryShapeOf(cfg *project.EvalConfig, kind, name string) (catalogEntryShape, bool) { if cfg == nil { - return "", false + return catalogEntryShape{}, false } if kind == "dataset" { if decl, ok := cfg.DatasetDeclaration(name); ok { - return decl.Ref, true + return catalogEntryShape{ref: decl.Ref}, true } - return "", false + return catalogEntryShape{}, false } if decl, ok := cfg.EvaluatorDeclaration(name); ok { - return decl.Ref, true + return catalogEntryShape{ref: decl.Ref, inlineRubric: decl.Definition != nil}, true } - return "", false + return catalogEntryShape{}, false } // updateCatalog applies a change to the configuration and writes it back. @@ -142,7 +160,7 @@ func updateCatalog( if created { cfg = &project.EvalConfig{} } - if err := checkNameNotBehindAnInclude(evalDir, cfg, kind, ref.Name); err != nil { + if err := checkCatalogEntryIsEditable(evalDir, cfg, kind, ref.Name); err != nil { return err } if !apply(cfg) { diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/catalog_include_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/catalog_include_test.go index f6a1a68cedd..a664ffe39a6 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/catalog_include_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/catalog_include_test.go @@ -32,7 +32,7 @@ evaluators: - $ref: ./parts/quality.yaml `), 0o600)) - err := checkNameNotBehindAnInclude( + err := checkCatalogEntryIsEditable( dir, mustOpenForEdit(t, dir), "evaluator", "quality") require.Error(t, err, "the name is taken, even though this file does not show it") @@ -53,7 +53,7 @@ evaluators: - $ref: ./parts/quality.yaml `), 0o600)) - require.NoError(t, checkNameNotBehindAnInclude( + require.NoError(t, checkCatalogEntryIsEditable( dir, mustOpenForEdit(t, dir), "evaluator", "tone")) } @@ -77,13 +77,104 @@ evaluators: name: quality `), 0o600)) - err := checkNameNotBehindAnInclude( + err := checkCatalogEntryIsEditable( dir, mustOpenForEdit(t, dir), "evaluator", "quality") require.Error(t, err, "the entry is an include, so it cannot be updated in place") assert.Contains(t, err.Error(), "quality") } +// The dataset branch of the guard is its own lookup, so it gets its own tests. +// +// Every case above is an evaluator, and `catalogEntryShapeOf` dispatches on kind +// before it looks anything up. A regression in the dataset branch would +// reintroduce the duplicate entries the guard exists to prevent while the +// evaluator tests stayed green. +func TestGenerateRefusesADatasetNameAnIncludeAlreadyDeclares(t *testing.T) { + dir := t.TempDir() + require.NoError(t, os.MkdirAll(filepath.Join(dir, "parts"), 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(dir, "parts", "golden.yaml"), + []byte("name: golden\nfile: ./datasets/golden.jsonl\n"), 0o600)) + + require.NoError(t, os.WriteFile(filepath.Join(dir, project.EvalConfigBase), []byte(` +datasets: + - $ref: ./parts/golden.yaml +`), 0o600)) + + err := checkCatalogEntryIsEditable( + dir, mustOpenForEdit(t, dir), "dataset", "golden") + + require.Error(t, err, "the dataset name is taken by the included file") + assert.Contains(t, err.Error(), "golden") + assert.Contains(t, err.Error(), "dataset", "the message names the kind it refused") +} + +// A dataset include carrying an overlay `name`, the shape the evaluator test +// above covers, refused through the dataset branch. +func TestGenerateRefusesADatasetIncludeThatCarriesItsName(t *testing.T) { + dir := t.TempDir() + require.NoError(t, os.MkdirAll(filepath.Join(dir, "parts"), 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(dir, "parts", "golden.yaml"), + []byte("file: ./datasets/golden.jsonl\n"), 0o600)) + + require.NoError(t, os.WriteFile(filepath.Join(dir, project.EvalConfigBase), []byte(` +datasets: + - $ref: ./parts/golden.yaml + name: golden +`), 0o600)) + + err := checkCatalogEntryIsEditable( + dir, mustOpenForEdit(t, dir), "dataset", "golden") + + require.Error(t, err, "the entry is an include, so it cannot be updated in place") + assert.Contains(t, err.Error(), "golden") +} + +// An evaluator already carrying its rubric under `definition:` is refused. +// +// No include is involved. Recording a generated file against it writes +// `source:` into an entry that already holds a `definition:`, and the next read +// rejects the whole configuration for declaring the rubric twice -- after the +// generation job has been billed and the file written. Refusing here is what +// keeps the failure ahead of the cost. +func TestGenerateRefusesAnEvaluatorThatAlreadyCarriesItsRubric(t *testing.T) { + dir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(dir, project.EvalConfigBase), []byte(` +evaluators: + - name: quality + definition: + type: rubric + dimensions: + - id: tone + weight: 3 +`), 0o600)) + + err := checkCatalogEntryIsEditable( + dir, mustOpenForEdit(t, dir), "evaluator", "quality") + + require.Error(t, err, "there is nowhere to record a file without declaring the rubric twice") + assert.Contains(t, err.Error(), "quality") + assert.Contains(t, err.Error(), "definition", "the reader has to be told which half is already there") +} + +// An entry written out here, with no include and no inline rubric, stays +// editable -- the case the guard must not catch. +func TestGenerateStillUpdatesAnEntryWrittenInThisFile(t *testing.T) { + dir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(dir, project.EvalConfigBase), []byte(` +datasets: + - name: golden + file: ./datasets/golden.jsonl +evaluators: + - name: quality + source: ./evaluators/quality.json +`), 0o600)) + + cfg := mustOpenForEdit(t, dir) + require.NoError(t, checkCatalogEntryIsEditable(dir, cfg, "evaluator", "quality")) + require.NoError(t, checkCatalogEntryIsEditable(dir, cfg, "dataset", "golden")) +} + func mustOpenForEdit(t *testing.T, dir string) *project.EvalConfig { t.Helper() cfg, err := project.OpenEvalConfigForEdit(dir) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/eval_group.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/eval_group.go index e255a4dfa25..56486220048 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/eval_group.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/eval_group.go @@ -118,7 +118,7 @@ func newEvalCreateCommand() *cobra.Command { for _, ref := range eval.Evaluators { decl, ok := cfg.EvaluatorDeclaration(ref.Evaluator) // A built-in, or one already registered, has nothing local to publish. - if !ok || (decl.Source == "" && decl.Definition == nil) { + if !ok || !decl.CarriesItsRubric() { continue } // A rubric written out in the configuration has no file to read. diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go b/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go index 2c66e9469a0..0704856c12d 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go @@ -1432,15 +1432,32 @@ func RefNeedsAProjectRoot(service string) error { // CatalogNameBehindAnInclude reports a name declared through a `$ref`, which // this command cannot edit in place. +// +// One message for two shapes, so the cause is stated as what they share: the +// entry lives in the referenced file. Naming only the duplicate-on-resolve case +// would misdescribe an overlay `name`, which collides with nothing and instead +// ends up declaring the rubric twice. func CatalogNameBehindAnInclude(kind, name string) error { return fmt.Errorf( - "%s %q is already declared through a `$ref`, so this command cannot update "+ - "it here: adding a second entry would collide with the first only after "+ - "the include is resolved. Edit the referenced file, or generate under a "+ - "different name", + "%s %q is declared in a file pulled in with `$ref`, so this command cannot "+ + "update it here: an entry written beside the directive takes effect only "+ + "once the include is resolved, and not as it reads. Edit the referenced "+ + "file, or generate under a different name", kind, name) } +// EvaluatorRubricWrittenInPlace reports an evaluator whose rubric is already +// written out under `definition:`, so there is nowhere to record a generated +// file without declaring the rubric twice. +func EvaluatorRubricWrittenInPlace(name string) error { + return fmt.Errorf( + "evaluator %q already carries its rubric under `definition:`, so this "+ + "command cannot record a generated file against it: an entry holding "+ + "both a `definition:` and a `source:` is refused on the next read. Edit "+ + "the rubric in place, or generate under a different name", + name) +} + // ReadingServiceConfig reports the service entry failing to serialize. func ReadingServiceConfig(err error) error { return fmt.Errorf("reading the eval service configuration: %w", err) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go index 90e6a9d856c..ffb38bd20ca 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go @@ -215,13 +215,27 @@ func (c *EvalConfig) EvaluatorDeclaration(name string) (*EvaluatorDecl, bool) { return nil, false } +// CarriesItsRubric reports whether this configuration owns the evaluator and +// has to publish it, rather than referring to a built-in or to one already +// registered under this name. +// +// Both fields have to be tested, and this is the only place that should test +// them. Validation forbids declaring the rubric twice, so `definition` implies +// an empty `source`: a selector written as `source == ""` reads as "nothing +// local to publish" but silently drops every evaluator carrying its rubric +// inline. That shipped once already -- the eval was created bound to an +// evaluator the service had never been told about. +func (d EvaluatorDecl) CarriesItsRubric() bool { + return d.Source != "" || d.Definition != nil +} + // CustomEvaluators are the catalog entries this configuration owns -- the ones // carrying a rubric, either as a local source or written out under // `definition`, published before the evals that name them. func (c *EvalConfig) CustomEvaluators() []EvaluatorDecl { var owned []EvaluatorDecl for _, decl := range c.Evaluators { - if decl.Source == "" && decl.Definition == nil { + if !decl.CarriesItsRubric() { continue } owned = append(owned, decl) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/one_ownership_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/one_ownership_test.go new file mode 100644 index 00000000000..7657a5f34b0 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/one_ownership_test.go @@ -0,0 +1,74 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package project + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// Whether this configuration owns an evaluator is decided in exactly one place. +// +// Two publish loops ask it -- `azd up` through CustomEvaluators, and `eval +// create` over the evaluators one eval names. Each carried its own copy of the +// test, and the first version of both read `source == ""`. That is wrong in a +// way nothing surfaces: validation forbids declaring the rubric twice, so an +// evaluator carrying its rubric under `definition:` has an empty `source`, was +// read as "nothing local to publish", and was skipped. The eval was then created +// bound to an evaluator the service had never been told about. +// +// A second copy of the predicate is how that returns, and it returns quietly -- +// the config still decodes, the deploy still reports success. So the shape is +// worth failing the build over rather than trusting a reviewer to spot it. +func TestEvaluatorOwnershipIsDecidedInOnePlace(t *testing.T) { + const predicate = "CarriesItsRubric" + + // Either half of the pair, written inline. `Definition == nil` on its own is + // legitimate -- the reconciler branches on it to choose what to publish -- + // so it is the pairing with a `Source` test that means someone has + // re-derived ownership. + sightings := map[string][]int{} + require.NoError(t, filepath.WalkDir("../..", func(path string, d os.DirEntry, err error) error { + if err != nil || d.IsDir() || !strings.HasSuffix(path, ".go") { + return err + } + if strings.HasSuffix(path, "_test.go") { + return nil // including this file, which spells out the shape it is looking for + } + body, err := os.ReadFile(path) + if err != nil { + return err + } + insidePredicate := false + for i, line := range strings.Split(string(body), "\n") { + // The predicate is the one place allowed to say this, so skip its body. + if strings.Contains(line, predicate+"() bool {") { + insidePredicate = true + continue + } + if insidePredicate { + if strings.HasPrefix(line, "}") { + insidePredicate = false + } + continue + } + source := strings.Contains(line, `Source == ""`) || strings.Contains(line, `Source != ""`) + definition := strings.Contains(line, "Definition == nil") || strings.Contains(line, "Definition != nil") + if source && definition { + sightings[path] = append(sightings[path], i+1) + } + } + return nil + })) + + assert.Empty(t, sightings, + "ownership is re-derived at %v; call %s instead, or the next rule about "+ + "what this configuration publishes will land on one loop and not the other", + sightings, predicate) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go index dc6252471ee..f4812538cdd 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go @@ -135,7 +135,13 @@ func (p *EvalServiceTargetProvider) Deploy( targetResource *azdext.TargetResource, progress azdext.ProgressReporter, ) (*azdext.ServiceDeployResult, error) { - cfg, err := EvalConfigFromService(serviceConfig, p.projectRoot(ctx)) + // Asked once and reused: a second call could fail where the first + // succeeded, and the empty root that comes back is indistinguishable from a + // project that has none. The include guard below would have passed while + // artifact paths quietly resolved against this process's directory instead. + projectRoot := p.projectRoot(ctx) + + cfg, err := EvalConfigFromService(serviceConfig, projectRoot) if err != nil { return nil, err } @@ -148,7 +154,7 @@ func (p *EvalServiceTargetProvider) Deploy( return nil, err } - baseDir := p.evalBaseDir(ctx, serviceConfig) + baseDir := baseDirUnder(projectRoot, serviceConfig) // 1. Datasets the configuration owns. Paths are kept so an eval that names // one can derive its columns without reading the blob back. @@ -218,7 +224,8 @@ func (p *EvalServiceTargetProvider) projectRoot(ctx context.Context) string { return resp.GetProject().GetPath() } -// evalBaseDir is the directory a declaration's `source:` resolves against. +// baseDirUnder places a service's directory under the project -- the directory +// a declaration's `source:` resolves against. // // serviceRelativeDir answers relative to the project, because that is what the // service's `$ref` and relativePath are written relative to. Left there it was @@ -230,14 +237,6 @@ func (p *EvalServiceTargetProvider) projectRoot(ctx context.Context) string { // generation job to rewrite a file already on disk. // // The same join is what agent_instructions.go does with the same helper. -func (p *EvalServiceTargetProvider) evalBaseDir( - ctx context.Context, - serviceConfig *azdext.ServiceConfig, -) string { - return baseDirUnder(p.projectRoot(ctx), serviceConfig) -} - -// baseDirUnder places a service's directory under the project. // // azd does not re-root an absolute `$ref` or an absolute `project:`, so neither // does this: joining one under the project produced /C:/shared/evals, diff --git a/cli/azd/extensions/azure.ai.evaluations/schemas/azure.ai.eval.json b/cli/azd/extensions/azure.ai.evaluations/schemas/azure.ai.eval.json index 0bcee881584..f980d734e7d 100644 --- a/cli/azd/extensions/azure.ai.evaluations/schemas/azure.ai.eval.json +++ b/cli/azd/extensions/azure.ai.evaluations/schemas/azure.ai.eval.json @@ -61,7 +61,7 @@ }, "file": { "type": "string", - "description": "Path to a local .jsonl whose rows are uploaded on deploy, relative to this file. Deliberately not a $ref: the rows are a data artifact to publish, not a definition to splice in, and a .jsonl is neither YAML nor JSON. Omit to use a dataset already registered under this name." + "description": "Path to a local .jsonl whose rows are uploaded on deploy, resolved against the evaluation configuration -- also when this entry arrived through a $ref, because a spliced path is not rebased to the file it was written in. Deliberately not a $ref: the rows are a data artifact to publish, not a definition to splice in, and a .jsonl is neither YAML nor JSON. Omit to use a dataset already registered under this name." }, "version": { "type": "string", @@ -80,7 +80,7 @@ }, "source": { "type": "string", - "description": "Path to a local .json rubric -- a list of weighted scoring dimensions -- relative to this file. Published on deploy and fingerprinted locally so a later deploy can tell an edit here from a version published elsewhere." + "description": "Path to a local .json rubric -- a list of weighted scoring dimensions -- resolved against the evaluation configuration, also when this entry arrived through a $ref, because a spliced path is not rebased; an entry pulled in from its own file should carry the rubric under definition instead. Published on deploy and fingerprinted locally so a later deploy can tell an edit here from a version published elsewhere." }, "definition": { "type": "object", @@ -107,6 +107,11 @@ "$comment": "A rubric is named or written out, never both.", "if": { "required": ["definition"] }, "then": { "properties": { "source": false } } + }, + { + "$comment": "A version pins one already registered, so there is nothing local to publish alongside it. Both shapes are refused when the configuration is read; stating them here is what makes the editor agree.", + "if": { "required": ["version"] }, + "then": { "properties": { "source": false, "definition": false } } } ] }, From 27a2f53c272ea63263052bacd0b28c7aad413644 Mon Sep 17 00:00:00 2001 From: mohessie Date: Fri, 21 Aug 2026 01:32:02 +0300 Subject: [PATCH 316/320] Take the modernization the CI gate asks for go fix -diff flags the slice walk in containsRefDirective, and lint-ext-azure-ai-evaluations.yml runs that gate against this module. --- .../internal/project/eval_config_store.go | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_store.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_store.go index 89f658caa4b..823729e2b96 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_store.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_store.go @@ -11,6 +11,7 @@ import ( "os" "path/filepath" "runtime" + "slices" "syscall" "time" @@ -273,11 +274,7 @@ func containsRefDirective(value any) bool { } } case []any: - for _, child := range typed { - if containsRefDirective(child) { - return true - } - } + return slices.ContainsFunc(typed, containsRefDirective) } return false } From 480aa9dcf444ead63fc0f2929db17bfb90e82404 Mon Sep 17 00:00:00 2001 From: mohessie Date: Fri, 21 Aug 2026 02:00:06 +0300 Subject: [PATCH 317/320] Gate the rubric rescue on the entry's own include Asking whether the document used a directive anywhere let a ref on an unrelated dataset rewrite a hand-written dimensions key into a rubric, so the same evaluator was refused or accepted according to a neighbour. --- .../internal/project/eval_config_store.go | 71 ++++++++++++++----- .../internal/project/neighbour_ref_test.go | 66 +++++++++++++++++ 2 files changed, 118 insertions(+), 19 deletions(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/project/neighbour_ref_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_store.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_store.go index 823729e2b96..cdd60f1a47c 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_store.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_store.go @@ -237,12 +237,12 @@ func resolveConfigRefs(data []byte, baseDir, name string) ([]byte, error) { // CLI command refused, and later the reverse. Callers differ in how they obtain // the map and what they do with it; everything between is here. func resolveEvalRefs(values map[string]any, baseDir string) (map[string]any, error) { - // Read before resolution, which removes the directive. Both routes gate the - // rescue on it so they cannot disagree: without it, the CLI's no-`$ref` fast - // path would skip nesting while the deploy path still applied it, and a - // hand-written entry carrying rubric keys would deploy and then be refused - // by every command that reads it. - spliced := containsRefDirective(values) + // Read before resolution, which consumes the directive. Both routes gate the + // rescue on the same answer so they cannot disagree: without it, the CLI's + // no-`$ref` fast path would skip nesting while the deploy path still applied + // it, and a hand-written entry carrying rubric keys would deploy and then be + // refused by every command that reads it. + spliced, visible := splicedEvaluators(values) resolved, err := foundry.ResolveFileRefs(values, baseDir) if err != nil { @@ -251,12 +251,41 @@ func resolveEvalRefs(values map[string]any, baseDir string) (map[string]any, err // `$ref` is a directive rather than configuration, and the strict decoder // would report the leftover as a mistyped key. delete(resolved, "$ref") - if spliced { - nestSplicedRubrics(resolved) - } + nestSplicedRubrics(resolved, spliced, visible) return resolved, nil } +// splicedEvaluators reports which evaluator entries carry an include of their +// own, and whether the list could be read at all. +// +// Entry level rather than document level. Asking only whether the document used +// `$ref` anywhere made one entry's meaning depend on another's: a directive on +// an unrelated dataset switched the rescue on for the whole file, so a +// hand-written `dimensions:` -- a mistake the strict decoder exists to report -- +// was filed as rubric content and published instead. The same evaluator was +// then refused or accepted according to a neighbour. +func splicedEvaluators(values map[string]any) (map[int]bool, bool) { + entries, ok := values["evaluators"].([]any) + if !ok { + // The configuration is itself behind a `$ref`, so its entries do not + // exist yet and nothing here was hand-written to protect. + return nil, false + } + spliced := map[int]bool{} + for i, entry := range entries { + m, ok := entry.(map[string]any) + if !ok { + continue + } + if _, has := m[refDirective]; has { + spliced[i] = true + } + } + // Position survives resolution: entries are replaced in place, never added + // or dropped. + return spliced, true +} + // containsRefDirective reports whether the document uses `$ref` anywhere. // // Structural rather than a text scan: the byte "$ref" also appears in comments @@ -296,18 +325,22 @@ var evaluatorDeclKeys = map[string]bool{ // from the service -- so its keys land beside `name` and the strict decoder // rejects them. Moving them is what lets a `$ref` name a rubric. // -// `dimensions` is what marks the leftovers as a rubric rather than a typo, and -// it is the same key normalizeRubricBody insists on before it will treat a -// document as a definition. Without that gate this would be a catch-all by -// another name, filing a misspelled `name` as rubric content and publishing it -// to the service instead of reporting it. +// Only the entries that carried a directive are touched. `dimensions` then +// marks the leftovers as a rubric rather than a typo, and it is the same key +// normalizeRubricBody insists on before it will treat a document as a +// definition. Without both gates this is a catch-all by another name, filing a +// misspelled `name` as rubric content and publishing it to the service instead +// of reporting it. // -// Structural rather than positional on purpose: an earlier version marked -// entries by index before resolution, which cannot see the evaluators inside a -// config that is itself behind a `$ref` -- the layout the README documents. -func nestSplicedRubrics(resolved map[string]any) { +// visible is false when the configuration is itself behind a `$ref`, where the +// entries only exist after resolution and there is nothing written here to tell +// them apart from. That is the layout the README documents. +func nestSplicedRubrics(resolved map[string]any, spliced map[int]bool, visible bool) { entries, _ := resolved["evaluators"].([]any) - for _, entry := range entries { + for i, entry := range entries { + if visible && !spliced[i] { + continue + } m, ok := entry.(map[string]any) if !ok { continue diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/neighbour_ref_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/neighbour_ref_test.go new file mode 100644 index 00000000000..35f87e742a0 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/neighbour_ref_test.go @@ -0,0 +1,66 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package project + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// A `$ref` on one entry does not change what a different entry means. +// +// The rescue that moves spliced rubric keys under `definition` used to be gated +// on "this document uses `$ref` somewhere". A directive on an unrelated dataset +// therefore switched it on for every evaluator in the file, and a hand-written +// `dimensions:` -- a mistake the strict decoder exists to report -- was silently +// filed as rubric content and published to the service instead. +// +// The same evaluator, refused in one file and accepted in another because of a +// neighbour, is the shape this whole mechanism is supposed to rule out. +func TestARefOnOneEntryDoesNotRescueAnother(t *testing.T) { + withoutRef := ` +datasets: + - name: golden + file: ./datasets/golden.jsonl +evaluators: + - name: quality + dimensions: + - id: tone + weight: 3 +` + withUnrelatedRef := ` +datasets: + - $ref: ./parts/golden.yaml +evaluators: + - name: quality + dimensions: + - id: tone + weight: 3 +` + + refused := func(t *testing.T, body string) error { + t.Helper() + dir := t.TempDir() + require.NoError(t, os.MkdirAll(filepath.Join(dir, "parts"), 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(dir, "parts", "golden.yaml"), + []byte("name: golden\nfile: ./datasets/golden.jsonl\n"), 0o600)) + require.NoError(t, os.WriteFile(filepath.Join(dir, EvalConfigBase), []byte(body), 0o600)) + _, err := OpenEvalConfig(dir) + return err + } + + baseline := refused(t, withoutRef) + require.Error(t, baseline, + "a rubric key written at entry level is a mistake, and the strict decoder reports it") + assert.Contains(t, baseline.Error(), "dimensions") + + neighbour := refused(t, withUnrelatedRef) + require.Error(t, neighbour, + "the dataset's `$ref` says nothing about this evaluator, so the same entry is still a mistake") + assert.Contains(t, neighbour.Error(), "dimensions") +} From a17f817b4e88385187e5df16f96eec5f5eb6ae8e Mon Sep 17 00:00:00 2001 From: mohessie Date: Fri, 21 Aug 2026 03:05:43 +0300 Subject: [PATCH 318/320] Refuse a pinned evaluator too A version pin says the rubric is already registered, so recording a generated file leaves the entry claiming both, which the next read rejects. --- .../internal/cmd/catalog.go | 22 ++++++++--- .../internal/cmd/catalog_include_test.go | 37 +++++++++++++++++++ .../internal/messages/messages.go | 11 ++++++ 3 files changed, 64 insertions(+), 6 deletions(-) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/catalog.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/catalog.go index 552fa2b4af1..9800fe1ff61 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/catalog.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/catalog.go @@ -69,16 +69,17 @@ func addEvaluatorToCatalog(cmd *cobra.Command, evalDir string, ref *project.Arti // checkCatalogEntryIsEditable refuses a name this command cannot rewrite in // place without corrupting the entry. // -// Three shapes reach this. A pure `$ref` has no name here at all, so the +// Four shapes reach this. A pure `$ref` has no name here at all, so the // duplicate scan had nothing to match on and appended a second entry with the // same name -- a collision that surfaced only on the next resolving read. A // `$ref` carrying an overlay `name` does match, and updating it in place writes // `source:` beside the directive, so resolution then produces a rubric and a // source and the configuration is rejected for declaring it twice. An entry -// already carrying its rubric under `definition:` fails that same way with no -// include involved, because recording the generated file leaves both in one -// entry -- and it fails after the generation job has been billed and the file -// written, which is why it is refused here rather than left to the next read. +// already carrying its rubric under `definition:`, and an entry pinned to a +// registered `version:`, both fail the same way with no include involved: +// recording the generated file leaves two rubrics, or a pin and a file, in one +// entry. Each of those is refused on the next read -- after the generation job +// has been billed and the file written, which is why they are refused here. // // A configuration that will not resolve is left to the commands that resolve it: // failing a generate over an unrelated broken include would be its own surprise. @@ -89,6 +90,8 @@ func checkCatalogEntryIsEditable(evalDir string, asWritten *project.EvalConfig, return messages.CatalogNameBehindAnInclude(kind, name) case entry.inlineRubric: return messages.EvaluatorRubricWrittenInPlace(name) + case entry.pinned: + return messages.EvaluatorPinnedToAVersion(name) } return nil } @@ -110,6 +113,9 @@ type catalogEntryShape struct { ref string // inlineRubric is an evaluator holding its rubric under `definition:`. inlineRubric bool + // pinned is an evaluator naming a registered `version:`. A dataset may hold + // a file and a version together; an evaluator may not. + pinned bool } // catalogEntryShapeOf returns how the entry was written, and whether the @@ -125,7 +131,11 @@ func catalogEntryShapeOf(cfg *project.EvalConfig, kind, name string) (catalogEnt return catalogEntryShape{}, false } if decl, ok := cfg.EvaluatorDeclaration(name); ok { - return catalogEntryShape{ref: decl.Ref, inlineRubric: decl.Definition != nil}, true + return catalogEntryShape{ + ref: decl.Ref, + inlineRubric: decl.Definition != nil, + pinned: decl.Version != "", + }, true } return catalogEntryShape{}, false } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/catalog_include_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/catalog_include_test.go index a664ffe39a6..27e569f828c 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/catalog_include_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/catalog_include_test.go @@ -175,6 +175,43 @@ evaluators: require.NoError(t, checkCatalogEntryIsEditable(dir, cfg, "dataset", "golden")) } +// An evaluator pinned to a registered version is refused, for the same reason +// an inline rubric is: there is nowhere to record the generated file. +// +// A pin says the rubric already lives in the service. Writing `source:` beside +// it leaves the entry claiming both, which the next read rejects -- again after +// the job has been billed and the file written. +func TestGenerateRefusesAnEvaluatorPinnedToAVersion(t *testing.T) { + dir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(dir, project.EvalConfigBase), []byte(` +evaluators: + - name: quality + version: "3" +`), 0o600)) + + err := checkCatalogEntryIsEditable( + dir, mustOpenForEdit(t, dir), "evaluator", "quality") + + require.Error(t, err, "a pin and a file in one entry is refused on the next read") + assert.Contains(t, err.Error(), "quality") + assert.Contains(t, err.Error(), "version", "the reader has to be told what is already there") +} + +// A dataset may carry a file and a version together -- the version says which +// one to publish -- so the pin must not make it uneditable. +func TestGenerateStillUpdatesAVersionedDataset(t *testing.T) { + dir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(dir, project.EvalConfigBase), []byte(` +datasets: + - name: golden + file: ./datasets/golden.jsonl + version: "4" +`), 0o600)) + + require.NoError(t, checkCatalogEntryIsEditable( + dir, mustOpenForEdit(t, dir), "dataset", "golden")) +} + func mustOpenForEdit(t *testing.T, dir string) *project.EvalConfig { t.Helper() cfg, err := project.OpenEvalConfigForEdit(dir) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go b/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go index 0704856c12d..affd440b0ec 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go @@ -1458,6 +1458,17 @@ func EvaluatorRubricWrittenInPlace(name string) error { name) } +// EvaluatorPinnedToAVersion reports an evaluator pinned to a registered +// version, which leaves nowhere to record a generated file. +func EvaluatorPinnedToAVersion(name string) error { + return fmt.Errorf( + "evaluator %q is pinned to a registered `version:`, so this command cannot "+ + "record a generated file against it: an entry holding both a `version:` "+ + "and a `source:` is refused on the next read. Remove the pin to publish "+ + "from a file, or generate under a different name", + name) +} + // ReadingServiceConfig reports the service entry failing to serialize. func ReadingServiceConfig(err error) error { return fmt.Errorf("reading the eval service configuration: %w", err) From 39acb24843e4e360e4448f78cbc0aba7e264a260 Mon Sep 17 00:00:00 2001 From: mohessie Date: Fri, 21 Aug 2026 03:10:58 +0300 Subject: [PATCH 319/320] Bump the evaluations extension to 1.0.18-beta --- .../azure.ai.evaluations/CHANGELOG.md | 34 ++++++++++++++++++- .../azure.ai.evaluations/extension.yaml | 2 +- .../azure.ai.evaluations/version.txt | 2 +- 3 files changed, 35 insertions(+), 3 deletions(-) diff --git a/cli/azd/extensions/azure.ai.evaluations/CHANGELOG.md b/cli/azd/extensions/azure.ai.evaluations/CHANGELOG.md index 6e9341806f5..3639ce7a484 100644 --- a/cli/azd/extensions/azure.ai.evaluations/CHANGELOG.md +++ b/cli/azd/extensions/azure.ai.evaluations/CHANGELOG.md @@ -1,6 +1,38 @@ # Release History -## 1.0.17-beta (Unreleased) +## 1.0.18-beta (2026-08-21) + +### Bugs Fixed + +- An evaluator carrying its rubric under `definition:`, rather than naming a + file, is now published. Both publish loops selected on `source:` alone, and + because a written-out rubric leaves `source:` empty they skipped it silently: + the eval was created bound to an evaluator the service had never been told + about. The two loops now share one test for what this configuration owns. +- `generate` no longer corrupts a catalog entry it cannot rewrite in place. An + entry reached through `$ref`, one already carrying its rubric under + `definition:`, and one pinned to a registered `version:` are each refused with + an explanation, instead of being written back holding two declarations of the + same rubric and failing on the next read. +- A `$ref` on one entry no longer changes what another entry means. The rescue + that moves a spliced rubric under `definition:` was switched on for the whole + file, so a directive on an unrelated dataset turned a mistyped `dimensions:` + into rubric content and published it. It is now decided per entry. +- The project directory is read once during a deploy. A second read could fail + where the first succeeded, leaving artifact paths resolved against the + extension's own working directory. +- An include reached without a project directory is refused rather than + silently discarded. + +### Other Changes + +- `$ref` is modelled on datasets and evals as well as evaluators, so a + configuration that deploys can also be opened by the commands that edit it. +- Schema descriptions for `file:` and `source:` now say that a path is resolved + against the evaluation configuration even when the entry arrived through a + `$ref`, and point at `definition:` for an entry kept in its own file. + +## 1.0.17-beta (2026-08-20) First release of the Foundry evaluations extension. diff --git a/cli/azd/extensions/azure.ai.evaluations/extension.yaml b/cli/azd/extensions/azure.ai.evaluations/extension.yaml index 2699d62da5f..77221506512 100644 --- a/cli/azd/extensions/azure.ai.evaluations/extension.yaml +++ b/cli/azd/extensions/azure.ai.evaluations/extension.yaml @@ -5,7 +5,7 @@ displayName: Foundry evaluations (Beta) description: Define and run Foundry evaluations from your terminal. (Beta) usage: azd ai eval [options] # NOTE: Make sure version.txt is in sync with this version. -version: 1.0.17-beta +version: 1.0.18-beta requiredAzdVersion: ">=1.27.1" language: go capabilities: diff --git a/cli/azd/extensions/azure.ai.evaluations/version.txt b/cli/azd/extensions/azure.ai.evaluations/version.txt index 0b7583d8244..f0c9b46eefb 100644 --- a/cli/azd/extensions/azure.ai.evaluations/version.txt +++ b/cli/azd/extensions/azure.ai.evaluations/version.txt @@ -1 +1 @@ -1.0.17-beta +1.0.18-beta From e24e4779d0ad43a2aa70d2ee51439150f67f0ebb Mon Sep 17 00:00:00 2001 From: mohessie Date: Fri, 21 Aug 2026 03:27:18 +0300 Subject: [PATCH 320/320] Write down the known gaps Each entry says what is wrong, why it was not fixed where it was found, and what closing it involves. --- .../azure.ai.evaluations/docs/KNOWN-GAPS.md | 124 ++++++++++++++++++ 1 file changed, 124 insertions(+) create mode 100644 cli/azd/extensions/azure.ai.evaluations/docs/KNOWN-GAPS.md diff --git a/cli/azd/extensions/azure.ai.evaluations/docs/KNOWN-GAPS.md b/cli/azd/extensions/azure.ai.evaluations/docs/KNOWN-GAPS.md new file mode 100644 index 00000000000..340cc5cbfef --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/docs/KNOWN-GAPS.md @@ -0,0 +1,124 @@ +# Known gaps + +Open work on `azd ai eval`, written down so it is decided rather than +rediscovered. Each entry says what is wrong, why it was not fixed where it was +found, and what closing it involves. + +## 1. The editing path should not go through typed structs + +**The one that matters.** Everything in section 2 is a symptom of this. + +`generate`, `init` and the catalog commands read `evals/azure.eval.yaml` into +Go structs and write it back. Two consequences: + +- **The file is rewritten, not edited.** Every comment is deleted and the + indentation is normalised. An author who annotates their configuration loses + those notes the first time they run `generate`. +- **`$ref` has to be modelled shape by shape.** azd core resolves the directive + on *any* object at any depth, but a typed round-trip only survives it where a + `Ref` field exists. It is currently on `DatasetDecl`, `EvaluatorDecl` and + `Eval`, and each addition needed a specification change. Anywhere it is + missing, the file deploys and the editing commands refuse it. + +azd core already ships the mechanism: `foundry.YAMLDocument` in +`cli/azd/pkg/foundry/includes_edit.go` is comment-preserving and `$ref`-aware +(`EntryRef`, `EditRefFile`, `SetServiceField`), and `cli/azd/pkg/yamlnode` +provides `Find` / `Set` / `Append` over a node tree. `YAMLDocument` has no +callers today; its own doc comment says it was written for the composition +command write path. + +Moving the editing commands onto it would preserve the author's file, make the +directive work wherever core supports it, and let the three `Ref` fields and the +`splicedEvaluators` / `nestSplicedRubrics` machinery be deleted. `YAMLDocument` +is oriented around service entries in `azure.yaml`, so this needs either a +generalisation there or direct use of `yamlnode`. + +## 2. Gaps that item 1 would close + +### 2a. `$ref` on nested shapes deploys but cannot be edited + +`$ref` under an eval's `source:`, under its `target:`, or on an item of its +`evaluators:` list resolves and deploys correctly, then fails the editing read +with `unknown key "$ref"`. + +Not fixed in place because modelling it means adding a fourth, fifth and sixth +`Ref` field, which `internal/project/config_keys_test.go` deliberately fails: +those tests pin each shape to the specification, and a new key there is a +promise the spec does not make. It also needs validation changes, since a +reference carrying only a directive currently fails `evaluator entry is missing +'evaluator'`. + +Two ways to close it without item 1: extend the spec to allow the include on +those shapes, or refuse it on the resolving path so both reads fail together. +The second would forbid something item 1 intends to support, so it is not +recommended. + +### 2b. A rubric rescue is document-wide when the configuration is itself included + +`splicedEvaluators` decides per entry whether a spliced rubric should be moved +under `definition:`. When the service entry is itself a `$ref` to an evaluation +configuration, those entries do not exist until after resolution and core does +not report which node came from which include, so the whole evaluator list is +rescued. + +The effect is that the same file is stricter opened directly than reached +through the service entry: an entry-level `dimensions:` typo is rejected on one +route and filed as rubric content on the other. + +Recovering provenance means either re-implementing resolution — which breaks the +single-resolver invariant that `one_resolver_test.go` enforces, and that +invariant exists because the two read routes disagreeing has caused three +separate bugs — or removing the need for the rescue entirely, which item 1 does. + +## 3. Smaller, independent + +### 3a. `init` can append a duplicate eval behind a `$ref` + +`init` checks only the unresolved configuration, so a pure-`$ref` eval decodes +with an empty name and `HasEval` misses it. `init --name nightly` then appends a +second entry, reports success, and the next resolving read fails naming a +duplicate the author cannot see. + +The catalog commands already refuse this for datasets and evaluators via +`checkCatalogEntryIsEditable`; `init` has no equivalent and `catalogEntryShapeOf` +has no `eval` branch. Related: `init --force` calls `RemoveEval`, which on an +overlay-name include deletes the directive and orphans the referenced file +without saying so. + +### 3b. A bare rubric's own `name` becomes the catalog name + +A `$ref` pointed straight at a rubric file splices that file's `name` into the +entry, so the rubric names the evaluator. Intended for the documented layout, +surprising if the rubric was written for something else. + +### 3c. `errors.As` and `sort.Strings` predate the modernization guidance + +`cli/azd/AGENTS.md` asks for `errors.AsType[T]` and the `slices` equivalents. +Neither is rewritten by `go fix -diff`, which is what +`lint-ext-azure-ai-evaluations.yml` actually gates on, so this is convention +rather than a build failure. Pre-existing across this extension and +`cli/azd/pkg` (40 and 5 occurrences). Worth one sweep, not a change per PR. + +### 3d. The schema cannot express every version conflict + +`EvaluatorDecl` forbids `version:` alongside `source:` or `definition:`, but an +entry written as a `$ref` matches the `FileRef` branch, whose +`additionalProperties` is `true`, so the conditional never applies. + +Left as is deliberately. Whether an overlay `version` conflicts depends on what +the referenced file holds, and JSON Schema cannot see the resolved document. +Forbidding `version` on the evaluator reference shape would reject a legally +pinned evaluator whose referenced file carries only a name. The runtime rejects +the real conflicts with a message naming the entry. + +## 4. Known and not a defect + +A fresh clone republishes every dataset and evaluator on its first deploy. +Version identity lives in the azd environment, which is not in the repository, +so a new clone cannot know what is already registered. + +## 5. Open product question + +How much should `init` and `generate` do on the author's behalf, and what should +they leave behind when they fail partway? Six tracked bugs reduce to that one +question and are blocked on it, not on implementation.