diff --git a/cli/azd/extensions/azure.ai.agents/README.md b/cli/azd/extensions/azure.ai.agents/README.md index ab7e09f0495..2653b77cbcb 100644 --- a/cli/azd/extensions/azure.ai.agents/README.md +++ b/cli/azd/extensions/azure.ai.agents/README.md @@ -33,6 +33,20 @@ effect: activity-protocol agents open the Microsoft 365 Agents Playground rather than the Agent Inspector, and `--port 8087` on its own collides with the inspector's own default UI port. +### Local client route telemetry + +When installed from the official registry, the extension reports the +`local_client.route.selected` usage event after `azd ai agent run` resolves the +service and protocol profile. Its `ext.route` attribute is exactly one of: + +- `inspector` for a non-activity agent; +- `playground` for an activity-protocol agent; or +- `suppressed` when `--no-client` or the deprecated `--no-inspector` is set. + +The event is emitted before checking client availability, starting the local +agent, or launching a client. It records route selection, not successful client +launch. + ## Migrating Legacy Agent Configuration New Foundry agent projects keep the agent definition directly on the diff --git a/cli/azd/extensions/azure.ai.agents/go.mod b/cli/azd/extensions/azure.ai.agents/go.mod index 571a6db8721..a62d2a7b235 100644 --- a/cli/azd/extensions/azure.ai.agents/go.mod +++ b/cli/azd/extensions/azure.ai.agents/go.mod @@ -12,7 +12,7 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerregistry/armcontainerregistry v1.3.0-beta.3 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.2.0 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armsubscriptions v1.3.0 - github.com/azure/azure-dev/cli/azd v1.28.0 + github.com/azure/azure-dev/cli/azd v1.31.0 github.com/braydonk/yaml v0.9.0 github.com/drone/envsubst v1.0.3 // indirect github.com/fatih/color v1.18.0 diff --git a/cli/azd/extensions/azure.ai.agents/go.sum b/cli/azd/extensions/azure.ai.agents/go.sum index 6b3b1b91d9f..6b902c14a89 100644 --- a/cli/azd/extensions/azure.ai.agents/go.sum +++ b/cli/azd/extensions/azure.ai.agents/go.sum @@ -61,8 +61,8 @@ github.com/aymanbagabas/go-udiff v0.2.0 h1:TK0fH4MteXUDspT88n8CKzvK0X9O2xu9yQjWp github.com/aymanbagabas/go-udiff v0.2.0/go.mod h1:RE4Ex0qsGkTAJoQdQQCA0uG+nAzJO/pI/QwceO5fgrA= github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4= -github.com/azure/azure-dev/cli/azd v1.28.0 h1:mqqyV85m7A1XfWJFjV/Ut0QoIEImFeF++1Ruq/cRp0s= -github.com/azure/azure-dev/cli/azd v1.28.0/go.mod h1:Ge7QaU9PoJM7i6J0xArDoQCf2tUn6O7OIKkoItxFTA8= +github.com/azure/azure-dev/cli/azd v1.31.0 h1:p0U4F6w2bPrdzmzavksqfJCnlXoQu9GTQogy+6KXMmM= +github.com/azure/azure-dev/cli/azd v1.31.0/go.mod h1:HFBGeWRWhNsOoYaUcyToqaowibqcbSCfkfJfnIfI4nU= github.com/bahlo/generic-list-go v0.2.0 h1:5sz/EEAK+ls5wF+NeqDpk5+iNdMDXrh3z3nPnH1Wvgk= github.com/bahlo/generic-list-go v0.2.0/go.mod h1:2KvAjgMlE5NNynlg/5iLrrCCZ2+5xWbdbCW3pNTGyYg= github.com/benbjohnson/clock v1.3.5 h1:VvXlSJBzZpA/zum6Sj74hxwYI2DIxRWuNIoXAzHZz5o= diff --git a/cli/azd/extensions/azure.ai.agents/internal/cmd/run.go b/cli/azd/extensions/azure.ai.agents/internal/cmd/run.go index 272e618a278..497930bf9d1 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/cmd/run.go +++ b/cli/azd/extensions/azure.ai.agents/internal/cmd/run.go @@ -38,6 +38,11 @@ import ( const ( agentInspectorExtensionID = "azure.ai.inspector" agentInspectorReadyPollPeriod = 250 * time.Millisecond + localClientRouteSelectedEvent = "local_client.route.selected" + localClientRouteAttribute = "route" + localClientRouteInspector = "inspector" + localClientRoutePlayground = "playground" + localClientRouteSuppressed = "suppressed" // defaultInspectorUIPort mirrors the default UI port of the // azure.ai.inspector extension. The inspector extension remains the source // of truth for the actual default: when --inspector-port is unset we do not @@ -152,6 +157,8 @@ func runRun(ctx context.Context, flags *runFlags, noPrompt bool) error { // validation can fail without starting a process, and such a failure must // not clear a session belonging to an already-running agent. activityProfile := resolveActivityRunProfile(runCtx.Definition) + suppressClient := flags.noInspector || flags.noClient + reportLocalClientRouteSelected(ctx, azdClient.Telemetry(), activityProfile, suppressClient) if err := validateInspectorPortForProfile(flags, activityProfile.IsActivity); err != nil { return err } @@ -171,7 +178,6 @@ func runRun(ctx context.Context, flags *runFlags, noPrompt bool) error { // Resolve local-client availability before the agent starts so advisory // port warnings can account for whether an inspector will actually launch. // Reuse the result after proc.Start rather than issuing a second RPC. - suppressClient := flags.noInspector || flags.noClient inspectorInstalled := false var inspectorInstallErr error if !activityProfile.IsActivity && !suppressClient { @@ -379,6 +385,29 @@ func runRun(ctx context.Context, flags *runFlags, noPrompt bool) error { return nil } +func reportLocalClientRouteSelected( + ctx context.Context, + telemetry azdext.TelemetryServiceClient, + activityProfile activityRunProfile, + suppressClient bool, +) { + route := localClientRouteInspector + if suppressClient { + route = localClientRouteSuppressed + } else if activityProfile.IsActivity { + route = localClientRoutePlayground + } + + if _, err := telemetry.ReportUsage(ctx, &azdext.ReportUsageRequest{ + EventName: localClientRouteSelectedEvent, + Attributes: map[string]string{ + localClientRouteAttribute: route, + }, + }); err != nil { + log.Printf("run: failed to report local client route selection: %v", err) + } +} + func handleInspectorAutoLaunch( ctx context.Context, workflow azdext.WorkflowServiceClient, diff --git a/cli/azd/extensions/azure.ai.agents/internal/cmd/run_test.go b/cli/azd/extensions/azure.ai.agents/internal/cmd/run_test.go index f94f6504498..eb05db9aa5e 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/cmd/run_test.go +++ b/cli/azd/extensions/azure.ai.agents/internal/cmd/run_test.go @@ -9,6 +9,7 @@ import ( "errors" "fmt" "io" + "maps" "net" "net/http" "net/http/httptest" @@ -238,6 +239,73 @@ func TestWaitForLocalPort(t *testing.T) { }) } +func TestReportLocalClientRouteSelected(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + activityProfile activityRunProfile + suppressClient bool + reportErr error + wantRoute string + }{ + { + name: "selects Inspector for non-activity agent", + wantRoute: localClientRouteInspector, + }, + { + name: "selects Playground for activity agent", + activityProfile: activityRunProfile{IsActivity: true}, + wantRoute: localClientRoutePlayground, + }, + { + name: "selects suppressed for non-activity agent", + suppressClient: true, + wantRoute: localClientRouteSuppressed, + }, + { + name: "suppression overrides activity route", + activityProfile: activityRunProfile{IsActivity: true}, + suppressClient: true, + wantRoute: localClientRouteSuppressed, + }, + { + name: "reporting failure is best effort", + reportErr: errors.New("telemetry unavailable"), + wantRoute: localClientRouteInspector, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + telemetry := &recordingTelemetryClient{err: tt.reportErr} + reportLocalClientRouteSelected( + t.Context(), + telemetry, + tt.activityProfile, + tt.suppressClient, + ) + + if telemetry.request == nil { + t.Fatal("expected telemetry request") + } + if telemetry.request.EventName != localClientRouteSelectedEvent { + t.Fatalf( + "event name = %q, want %q", + telemetry.request.EventName, + localClientRouteSelectedEvent, + ) + } + wantAttributes := map[string]string{localClientRouteAttribute: tt.wantRoute} + if !maps.Equal(telemetry.request.Attributes, wantAttributes) { + t.Fatalf("attributes = %v, want %v", telemetry.request.Attributes, wantAttributes) + } + }) + } +} + func TestLaunchInspectorUsesWorkflowCommand(t *testing.T) { t.Parallel() @@ -743,6 +811,11 @@ type recordingWorkflowClient struct { called chan struct{} } +type recordingTelemetryClient struct { + request *azdext.ReportUsageRequest + err error +} + type lockedBuffer struct { mu sync.Mutex bytes.Buffer @@ -772,6 +845,18 @@ func (c *recordingWorkflowClient) Run( return &azdext.EmptyResponse{}, c.err } +func (c *recordingTelemetryClient) ReportUsage( + _ context.Context, + request *azdext.ReportUsageRequest, + _ ...grpc.CallOption, +) (*azdext.ReportUsageResponse, error) { + c.request = request + if c.err != nil { + return nil, c.err + } + return &azdext.ReportUsageResponse{Accepted: true}, nil +} + // createVenv sets up a minimal .venv directory structure for testing. // Returns the path to the .venv directory. func createVenv(t *testing.T, projectDir string) string { diff --git a/cli/azd/extensions/azure.ai.inspector/README.md b/cli/azd/extensions/azure.ai.inspector/README.md index f273fd65d53..e1303d7e083 100644 --- a/cli/azd/extensions/azure.ai.inspector/README.md +++ b/cli/azd/extensions/azure.ai.inspector/README.md @@ -40,6 +40,21 @@ azd ai inspector launch --session-id --conversation-id | `--session-id` | _(SPA mints UUID)_ | Optional explicit session ID for the SPA. | | `--conversation-id` | _(SPA mints UUID)_ | Optional explicit conversation ID for the SPA. | +## Telemetry + +When the SPA sends `setViewReady` after mounting, the extension reports this +best-effort usage event through azd: + +```text +extension.event = inspector.funnel.stage +ext.stage = ui_ready +ext.outcome = succeeded +``` + +The event means the Inspector UI loaded. It does not mean that the UI connected +to the agent or sent a request. No ports, URLs, IDs, prompts, or responses are +included. + ## Local Development ### Prerequisites diff --git a/cli/azd/extensions/azure.ai.inspector/extension.yaml b/cli/azd/extensions/azure.ai.inspector/extension.yaml index f2bfbfa3e69..a8e6a66ce95 100644 --- a/cli/azd/extensions/azure.ai.inspector/extension.yaml +++ b/cli/azd/extensions/azure.ai.inspector/extension.yaml @@ -6,7 +6,7 @@ description: Browser-based inspector UI for locally running Foundry agents. (Bet usage: azd ai inspector [options] # NOTE: Make sure version.txt is in sync with this version. version: 1.0.0-beta.3 -requiredAzdVersion: ">=1.27.0" +requiredAzdVersion: ">=1.31.0" language: go capabilities: - custom-commands diff --git a/cli/azd/extensions/azure.ai.inspector/go.mod b/cli/azd/extensions/azure.ai.inspector/go.mod index 3b8f62095f4..f756df96a3a 100644 --- a/cli/azd/extensions/azure.ai.inspector/go.mod +++ b/cli/azd/extensions/azure.ai.inspector/go.mod @@ -3,14 +3,16 @@ module azureaiinspector go 1.26.4 require ( - github.com/azure/azure-dev/cli/azd v1.24.3 + github.com/azure/azure-dev/cli/azd v1.31.0 github.com/cli/browser v1.3.0 github.com/gorilla/websocket v1.5.3 github.com/spf13/cobra v1.10.2 github.com/spf13/pflag v1.0.10 + github.com/stretchr/testify v1.11.1 ) require ( + dario.cat/mergo v1.0.2 // indirect github.com/AlecAivazis/survey/v2 v2.3.7 // indirect github.com/Azure/azure-sdk-for-go/sdk/azcore v1.20.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1 // indirect @@ -75,7 +77,6 @@ require ( github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 // indirect github.com/sethvargo/go-retry v0.3.0 // indirect github.com/spf13/cast v1.10.0 // indirect - github.com/stretchr/testify v1.11.1 // indirect github.com/theckman/yacspin v0.13.12 // indirect github.com/wk8/go-ordered-map/v2 v2.1.8 // indirect github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect @@ -91,7 +92,8 @@ require ( go.uber.org/multierr v1.11.0 // indirect golang.org/x/crypto v0.53.0 // indirect golang.org/x/exp v0.0.0-20250911091902-df9299821621 // indirect - golang.org/x/net v0.55.0 // indirect + golang.org/x/net v0.56.0 // indirect + golang.org/x/sync v0.21.0 // indirect golang.org/x/sys v0.46.0 // indirect golang.org/x/term v0.44.0 // indirect golang.org/x/text v0.38.0 // indirect diff --git a/cli/azd/extensions/azure.ai.inspector/go.sum b/cli/azd/extensions/azure.ai.inspector/go.sum index 30f15d039ba..a8724ad76bc 100644 --- a/cli/azd/extensions/azure.ai.inspector/go.sum +++ b/cli/azd/extensions/azure.ai.inspector/go.sum @@ -1,4 +1,6 @@ code.cloudfoundry.org/clock v0.0.0-20180518195852-02e53af36e6c/go.mod h1:QD9Lzhd/ux6eNQVUDVRJX/RKTigpewimNYBi7ivZKY8= +dario.cat/mergo v1.0.2 h1:85+piFYR1tMbRrLcDwR18y4UKJ3aH1Tbzi24VRW1TK8= +dario.cat/mergo v1.0.2/go.mod h1:E/hbnu0NxMFBjpMIE34DRGLWqDy0g5FuKDhCb31ngxA= github.com/AlecAivazis/survey/v2 v2.3.7 h1:6I/u8FvytdGsgonrYsVn2t8t4QiRnh6QSTqkkhIiSjQ= github.com/AlecAivazis/survey/v2 v2.3.7/go.mod h1:xUTIdE4KCOIjsBAE1JYsUPoCqYdZ1reCfTwbto0Fduo= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.20.0 h1:JXg2dwJUmPB9JmtVmdEB16APJ7jurfbY5jnfXpJoRMc= @@ -45,8 +47,8 @@ github.com/aymanbagabas/go-udiff v0.2.0 h1:TK0fH4MteXUDspT88n8CKzvK0X9O2xu9yQjWp github.com/aymanbagabas/go-udiff v0.2.0/go.mod h1:RE4Ex0qsGkTAJoQdQQCA0uG+nAzJO/pI/QwceO5fgrA= github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4= -github.com/azure/azure-dev/cli/azd v1.24.3 h1:r2kEr2YYLu4ImKo6nR/WjhHg/1SliN1uwmAVqnM8t3o= -github.com/azure/azure-dev/cli/azd v1.24.3/go.mod h1:YANepMw36aWA8/mQyXau6JCAG84oK0ZgfvLF8rN5asU= +github.com/azure/azure-dev/cli/azd v1.31.0 h1:p0U4F6w2bPrdzmzavksqfJCnlXoQu9GTQogy+6KXMmM= +github.com/azure/azure-dev/cli/azd v1.31.0/go.mod h1:HFBGeWRWhNsOoYaUcyToqaowibqcbSCfkfJfnIfI4nU= github.com/bahlo/generic-list-go v0.2.0 h1:5sz/EEAK+ls5wF+NeqDpk5+iNdMDXrh3z3nPnH1Wvgk= github.com/bahlo/generic-list-go v0.2.0/go.mod h1:2KvAjgMlE5NNynlg/5iLrrCCZ2+5xWbdbCW3pNTGyYg= github.com/benbjohnson/clock v1.3.5 h1:VvXlSJBzZpA/zum6Sj74hxwYI2DIxRWuNIoXAzHZz5o= @@ -259,11 +261,13 @@ golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.55.0 h1:bcvxaJn3e1U6InsFWt1JUq1aSjnRxLzT2rtD2KfkDF8= -golang.org/x/net v0.55.0/go.mod h1:L5U2KuzuOe1lY7Z+aWVIKK6qEeJXnXV9yzGA+WCHJww= +golang.org/x/net v0.56.0 h1:Rw8j/hFzGvJUZwNBXnAtf5sVDVt+65SK2C7IxCxZt5o= +golang.org/x/net v0.56.0/go.mod h1:D3Ku6r+V6JROoZK144D2XfMHFcMq/0zSfLelVTCFKec= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM= +golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= diff --git a/cli/azd/extensions/azure.ai.inspector/internal/cmd/inspector.go b/cli/azd/extensions/azure.ai.inspector/internal/cmd/inspector.go index 179cd0b6abd..b243c5a7674 100644 --- a/cli/azd/extensions/azure.ai.inspector/internal/cmd/inspector.go +++ b/cli/azd/extensions/azure.ai.inspector/internal/cmd/inspector.go @@ -15,6 +15,7 @@ import ( "azureaiinspector/internal/inspector" + "github.com/azure/azure-dev/cli/azd/pkg/azdext" "github.com/cli/browser" "github.com/spf13/cobra" ) @@ -98,6 +99,9 @@ func runInspector(ctx context.Context, flags *inspectorFlags) error { } } + reportUsage, closeTelemetry := newUsageReporter(ctx) + defer closeTelemetry() + srv := inspector.New(inspector.Config{ Port: flags.inspectorPort, AgentPort: flags.port, @@ -106,6 +110,7 @@ func runInspector(ctx context.Context, flags *inspectorFlags) error { ConversationID: flags.conversationID, SSESink: sseSink, Silent: flags.silent, + ReportUsage: reportUsage, }) url := srv.URL() @@ -137,6 +142,31 @@ func runInspector(ctx context.Context, flags *inspectorFlags) error { return srv.Start(ctx, ready) } +func newUsageReporter(ctx context.Context) (inspector.ReportUsageFunc, func()) { + azdClient, err := azdext.NewAzdClient() + if err != nil { + log.Printf("inspector: failed to create telemetry client: %v", err) + return nil, func() {} + } + + return usageReporter(ctx, azdClient.Telemetry()), azdClient.Close +} + +func usageReporter(ctx context.Context, telemetry azdext.TelemetryServiceClient) inspector.ReportUsageFunc { + // Capture the command context because WebSocket request contexts do not carry + // the azd access token or parent trace metadata needed by ReportUsage. + reportUsage := func(eventName string, attributes map[string]string) { + if _, err := telemetry.ReportUsage(ctx, &azdext.ReportUsageRequest{ + EventName: eventName, + Attributes: attributes, + }); err != nil { + log.Printf("inspector: failed to report %s: %v", eventName, err) + } + } + + return reportUsage +} + // injectSSEEvents wraps the local agentserver SSE stream so it matches the // Foundry SSE shape that readSSEStream expects. agentserver discriminates // chunks via a JSON `type` field on each `data:` line and omits the diff --git a/cli/azd/extensions/azure.ai.inspector/internal/cmd/inspector_test.go b/cli/azd/extensions/azure.ai.inspector/internal/cmd/inspector_test.go index 95a0d0a1234..56908fb93aa 100644 --- a/cli/azd/extensions/azure.ai.inspector/internal/cmd/inspector_test.go +++ b/cli/azd/extensions/azure.ai.inspector/internal/cmd/inspector_test.go @@ -4,14 +4,47 @@ package cmd import ( + "context" "errors" "fmt" "io" "strings" "testing" "time" + + "github.com/azure/azure-dev/cli/azd/pkg/azdext" + "github.com/stretchr/testify/require" + "google.golang.org/grpc" + "google.golang.org/grpc/metadata" ) +func TestUsageReporterForwardsEventWithCommandContext(t *testing.T) { + ctx := metadata.NewOutgoingContext(t.Context(), metadata.Pairs( + "authorization", "token", + "traceparent", "00-11111111111111111111111111111111-2222222222222222-01", + )) + telemetry := &recordingTelemetryClient{} + + usageReporter(ctx, telemetry)("inspector.funnel.stage", map[string]string{ + "stage": "ui_ready", + "outcome": "succeeded", + }) + + require.Equal(t, &azdext.ReportUsageRequest{ + EventName: "inspector.funnel.stage", + Attributes: map[string]string{ + "stage": "ui_ready", + "outcome": "succeeded", + }, + }, telemetry.request) + md, ok := metadata.FromOutgoingContext(telemetry.ctx) + require.True(t, ok) + require.Equal(t, []string{"token"}, md.Get("authorization")) + require.Equal(t, []string{ + "00-11111111111111111111111111111111-2222222222222222-01", + }, md.Get("traceparent")) +} + func TestInjectSSEEventsSynthesizesEventLines(t *testing.T) { input := "data: {\"type\":\"response.output_text.delta\"}\n\n" @@ -92,3 +125,18 @@ type errorReader struct { func (r errorReader) Read([]byte) (int, error) { return 0, r.err } + +type recordingTelemetryClient struct { + ctx context.Context + request *azdext.ReportUsageRequest +} + +func (c *recordingTelemetryClient) ReportUsage( + ctx context.Context, + request *azdext.ReportUsageRequest, + _ ...grpc.CallOption, +) (*azdext.ReportUsageResponse, error) { + c.ctx = ctx + c.request = request + return &azdext.ReportUsageResponse{Accepted: true}, nil +} diff --git a/cli/azd/extensions/azure.ai.inspector/internal/inspector/rpc.go b/cli/azd/extensions/azure.ai.inspector/internal/inspector/rpc.go index 244a30e5147..b469f04e94c 100644 --- a/cli/azd/extensions/azure.ai.inspector/internal/inspector/rpc.go +++ b/cli/azd/extensions/azure.ai.inspector/internal/inspector/rpc.go @@ -47,9 +47,10 @@ const ( // rpcSession owns one WebSocket. writeMu enforces gorilla/websocket's // single-writer requirement. type rpcSession struct { - cfg Config - conn *websocket.Conn - logger *log.Logger + cfg Config + conn *websocket.Conn + logger *log.Logger + reportUIReady func() writeMu sync.Mutex @@ -86,6 +87,7 @@ func (s *Server) handleWS(w http.ResponseWriter, r *http.Request) { nextRequestID: 1, rootCtx: rootCtx, rootCancel: rootCancel, + reportUIReady: s.reportUIReady, } defer sess.cleanup() go sess.pingLoop(wsPingPeriod) @@ -192,6 +194,12 @@ func (s *rpcSession) route(method string, params json.RawMessage) (any, error) { func (s *rpcSession) handleNotification(method string, params json.RawMessage) { switch method { case "setViewReady": + // Report after the navigation attempt so telemetry cannot delay UI initialization. + defer func() { + if s.reportUIReady != nil { + s.reportUIReady() + } + }() // SPA has mounted; tell it which agent port to target. payload := map[string]any{ "port": s.cfg.AgentPort, diff --git a/cli/azd/extensions/azure.ai.inspector/internal/inspector/rpc_test.go b/cli/azd/extensions/azure.ai.inspector/internal/inspector/rpc_test.go index 0230ed658ce..aa92e3928d5 100644 --- a/cli/azd/extensions/azure.ai.inspector/internal/inspector/rpc_test.go +++ b/cli/azd/extensions/azure.ai.inspector/internal/inspector/rpc_test.go @@ -34,15 +34,23 @@ func TestRegisterStreamAfterCleanupCancelsWithoutPanic(t *testing.T) { } func TestHandleMessageSafelyRecoversPanic(t *testing.T) { + uiReadyReported := false sess := &rpcSession{ cfg: Config{AgentPort: 8088}, logger: log.New(io.Discard, "", 0), streams: make(map[string]context.CancelFunc), rootCtx: t.Context(), rootCancel: func() {}, + reportUIReady: func() { + uiReadyReported = true + }, } // setViewReady writes to the websocket. A nil conn would panic without the // recover wrapper around per-message goroutines. sess.handleMessageSafely(rpcMessage{Method: "setViewReady"}) + + if !uiReadyReported { + t.Fatal("setViewReady should report the UI-ready funnel stage") + } } diff --git a/cli/azd/extensions/azure.ai.inspector/internal/inspector/server.go b/cli/azd/extensions/azure.ai.inspector/internal/inspector/server.go index f901db2d03e..1a38ed989f5 100644 --- a/cli/azd/extensions/azure.ai.inspector/internal/inspector/server.go +++ b/cli/azd/extensions/azure.ai.inspector/internal/inspector/server.go @@ -42,13 +42,17 @@ type Config struct { // Silent suppresses terminal output that is useful for standalone // inspector runs but noisy when azd ai agent run auto-launches it. Silent bool + + // ReportUsage records extension-owned usage events. If nil, telemetry is disabled. + ReportUsage ReportUsageFunc } type Server struct { - cfg Config - httpSrv *http.Server - upgrader websocket.Upgrader - logger *log.Logger + cfg Config + httpSrv *http.Server + upgrader websocket.Upgrader + logger *log.Logger + reportUIReady func() } func New(cfg Config) *Server { @@ -57,8 +61,9 @@ func New(cfg Config) *Server { logger = log.New(log.Writer(), "[inspector] ", log.LstdFlags) } return &Server{ - cfg: cfg, - logger: logger, + cfg: cfg, + logger: logger, + reportUIReady: newUIReadyReporter(cfg.ReportUsage), upgrader: websocket.Upgrader{ CheckOrigin: func(r *http.Request) bool { origin := r.Header.Get("Origin") diff --git a/cli/azd/extensions/azure.ai.inspector/internal/inspector/telemetry.go b/cli/azd/extensions/azure.ai.inspector/internal/inspector/telemetry.go new file mode 100644 index 00000000000..3dbad797c5c --- /dev/null +++ b/cli/azd/extensions/azure.ai.inspector/internal/inspector/telemetry.go @@ -0,0 +1,31 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package inspector + +import "sync" + +const ( + inspectorFunnelStageEvent = "inspector.funnel.stage" + inspectorFunnelStageAttribute = "stage" + inspectorFunnelOutcomeAttribute = "outcome" + + inspectorFunnelStageUIReady = "ui_ready" + inspectorFunnelSucceeded = "succeeded" +) + +// ReportUsageFunc records one extension-owned usage event. +type ReportUsageFunc func(eventName string, attributes map[string]string) + +func newUIReadyReporter(reportUsage ReportUsageFunc) func() { + return sync.OnceFunc(func() { + if reportUsage == nil { + return + } + + reportUsage(inspectorFunnelStageEvent, map[string]string{ + inspectorFunnelStageAttribute: inspectorFunnelStageUIReady, + inspectorFunnelOutcomeAttribute: inspectorFunnelSucceeded, + }) + }) +} diff --git a/cli/azd/extensions/azure.ai.inspector/internal/inspector/telemetry_test.go b/cli/azd/extensions/azure.ai.inspector/internal/inspector/telemetry_test.go new file mode 100644 index 00000000000..d68a96bf0b1 --- /dev/null +++ b/cli/azd/extensions/azure.ai.inspector/internal/inspector/telemetry_test.go @@ -0,0 +1,40 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package inspector + +import ( + "sync" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestUIReadyReporterReportsFunnelStageOnce(t *testing.T) { + type usageEvent struct { + name string + attributes map[string]string + } + + var events []usageEvent + reportUIReady := newUIReadyReporter(func(eventName string, attributes map[string]string) { + events = append(events, usageEvent{ + name: eventName, + attributes: attributes, + }) + }) + + var wg sync.WaitGroup + for range 10 { + wg.Go(reportUIReady) + } + wg.Wait() + + require.Equal(t, []usageEvent{{ + name: inspectorFunnelStageEvent, + attributes: map[string]string{ + inspectorFunnelStageAttribute: inspectorFunnelStageUIReady, + inspectorFunnelOutcomeAttribute: inspectorFunnelSucceeded, + }, + }}, events) +} diff --git a/cli/azd/extensions/azure.ai.projects/go.mod b/cli/azd/extensions/azure.ai.projects/go.mod index c12354f5e20..2be5f44976d 100644 --- a/cli/azd/extensions/azure.ai.projects/go.mod +++ b/cli/azd/extensions/azure.ai.projects/go.mod @@ -7,7 +7,7 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.14.0-beta.3 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/cognitiveservices/armcognitiveservices/v2 v2.0.0 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.2.0 - github.com/azure/azure-dev/cli/azd v1.28.0 + github.com/azure/azure-dev/cli/azd v1.31.0 github.com/drone/envsubst v1.0.3 github.com/fatih/color v1.18.0 github.com/spf13/cobra v1.10.1 diff --git a/cli/azd/extensions/azure.ai.projects/go.sum b/cli/azd/extensions/azure.ai.projects/go.sum index 19447301a86..856787edaa3 100644 --- a/cli/azd/extensions/azure.ai.projects/go.sum +++ b/cli/azd/extensions/azure.ai.projects/go.sum @@ -53,8 +53,8 @@ github.com/aymanbagabas/go-udiff v0.2.0 h1:TK0fH4MteXUDspT88n8CKzvK0X9O2xu9yQjWp github.com/aymanbagabas/go-udiff v0.2.0/go.mod h1:RE4Ex0qsGkTAJoQdQQCA0uG+nAzJO/pI/QwceO5fgrA= github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4= -github.com/azure/azure-dev/cli/azd v1.28.0 h1:mqqyV85m7A1XfWJFjV/Ut0QoIEImFeF++1Ruq/cRp0s= -github.com/azure/azure-dev/cli/azd v1.28.0/go.mod h1:Ge7QaU9PoJM7i6J0xArDoQCf2tUn6O7OIKkoItxFTA8= +github.com/azure/azure-dev/cli/azd v1.31.0 h1:p0U4F6w2bPrdzmzavksqfJCnlXoQu9GTQogy+6KXMmM= +github.com/azure/azure-dev/cli/azd v1.31.0/go.mod h1:HFBGeWRWhNsOoYaUcyToqaowibqcbSCfkfJfnIfI4nU= github.com/bahlo/generic-list-go v0.2.0 h1:5sz/EEAK+ls5wF+NeqDpk5+iNdMDXrh3z3nPnH1Wvgk= github.com/bahlo/generic-list-go v0.2.0/go.mod h1:2KvAjgMlE5NNynlg/5iLrrCCZ2+5xWbdbCW3pNTGyYg= github.com/benbjohnson/clock v1.3.5 h1:VvXlSJBzZpA/zum6Sj74hxwYI2DIxRWuNIoXAzHZz5o= diff --git a/docs/reference/telemetry-data.md b/docs/reference/telemetry-data.md index 63fbbf1fb33..2bd30f0d1e2 100644 --- a/docs/reference/telemetry-data.md +++ b/docs/reference/telemetry-data.md @@ -460,6 +460,9 @@ Emitted at provision start by the `microsoft.foundry` provisioning provider (the | `extension.version` | string | Extension version | | `extension.event` | string | Extension-chosen event name on an `ext.usage` span | | `ext.` | string | One extension-supplied attribute on an `ext.usage` span. The key after the `ext.` prefix and the value are chosen by the extension | +| `ext.route` | string | Local-client route selected by `azure.ai.agents`: `inspector`, `playground`, or `suppressed` (`local_client.route.selected`) | +| `ext.stage` | string | Agent Inspector funnel stage: currently `ui_ready` (`inspector.funnel.stage`) | +| `ext.outcome` | string | Agent Inspector funnel-stage outcome: currently `succeeded` (`inspector.funnel.stage`) | | `extension.installed` | string[] | List of installed extensions (`id@version`) | | `extension.installed.source.category` | string[] | Installed extension source categories (`id@category`) | | `extension.version.from` | string | Version before an update or promotion (`ext.update`, `ext.promote`) | @@ -489,6 +492,13 @@ source succeeds but records nothing, as does any report past the limit of 100 spans per `azd` invocation. This is a configuration-based admission check, not a cryptographic provenance guarantee. +Reviewed first-party extension usage events currently include: + +| Extension | `extension.event` | Trigger | Dynamic attributes | +|-----------|-------------------|---------|--------------------| +| `azure.ai.agents` | `local_client.route.selected` | `azd ai agent run` resolves the service and protocol profile; emitted before client availability, agent startup, and client launch | `ext.route`: `inspector`, `playground`, or `suppressed`; suppression takes precedence | +| `azure.ai.inspector` | `inspector.funnel.stage` | The Inspector SPA sends `setViewReady` after mounting | `ext.stage=ui_ready`; `ext.outcome=succeeded`; this does not indicate agent connection | + Source-category fields are classified from the configured source type and location, not the user-defined source name. Raw source names, URLs, paths, and hosts are not emitted in those fields. diff --git a/docs/specs/metrics-audit/feature-telemetry-matrix.md b/docs/specs/metrics-audit/feature-telemetry-matrix.md index 99f59d393d6..af86eee403c 100644 --- a/docs/specs/metrics-audit/feature-telemetry-matrix.md +++ b/docs/specs/metrics-audit/feature-telemetry-matrix.md @@ -168,4 +168,6 @@ reserved field contracts. | **Up-graph performance** | `up` (graph execution) | (none — enriches the `up` command span) | `perf.provision_duration_ms`, `perf.deploy_duration_ms`, `perf.total_duration_ms` | Emitted from `internal/cmd/up_graph.go` after the graph completes; provision/deploy durations set only when those phases run | | **VS RPC** | `vs-server` long-running session | `vsrpc.*` (event prefix) | Per-RPC attributes documented in `telemetry-schema.md` | Long-running RPC server for VS integration | | **Extension telemetry service** | Extension calls `ReportUsage` over the extension gRPC API | `ext.usage` | `extension.id`, `extension.version`, `extension.source`, `extension.event`, plus one `ext.` attribute per entry in the caller's attribute map | Telemetry requires no separate capability or declaration. Only extensions whose configured source matches the verified official `azd` registry name, type, and normalized URL are recorded — a call from any other source succeeds but is dropped, as is any call past 100 recorded events per `azd` invocation. Identity fields are derived from host-signed claims and the installed record, never from the request. Every caller key is prefixed with `ext.` so it cannot overwrite a host field, and the host bounds count and length only — it does not enumerate or pattern-check values | +| **Azure AI Agents local-client routing** | `azd ai agent run` resolves the service and protocol profile | `ext.usage` with `extension.event=local_client.route.selected` | `ext.route` (`inspector`, `playground`, or `suppressed`) | Records one mutually exclusive route before client availability, agent startup, and client launch; suppression takes precedence and the event does not indicate launch success | +| **Agent Inspector UI readiness** | Inspector SPA sends `setViewReady` after mounting | `ext.usage` with `extension.event=inspector.funnel.stage` | `ext.stage=ui_ready`, `ext.outcome=succeeded` | Emitted at most once per Inspector process; proves the SPA loaded, not that it connected to an agent | | **App detection** | `init`, `up` (fresh projects without `azure.yaml`, via `appdetect.Detect`) | `aspire.apphost.unsupported` | `aspire.apphost.language` (fixed enum — `typescript` / `python` / `go` / `java` / `rust`; not hashed) | Emitted from `internal/appdetect/dotnet_apphost.go` when an Aspire polyglot (non-C#) AppHost is detected; azd surfaces an actionable error referencing [#7138](https://github.com/Azure/azure-dev/issues/7138) instead of falling through to a generic source build | diff --git a/docs/specs/metrics-audit/telemetry-schema.md b/docs/specs/metrics-audit/telemetry-schema.md index f050477953c..0aa05c55565 100644 --- a/docs/specs/metrics-audit/telemetry-schema.md +++ b/docs/specs/metrics-audit/telemetry-schema.md @@ -247,6 +247,13 @@ guarantees about the whole class: | Trust | `extension.id` and `extension.version` are derived from host-signed claims; `extension.source` and eligibility are checked against the installed record and verified source config, never from the request | | Review | Extension telemetry is reviewed when the extension is admitted to the official registry, under the same documentation, classification, and privacy rules as core fields. The eligibility rule above is what ties recording to that review | +Reviewed first-party event contracts: + +| Extension | `extension.event` | Trigger | Extension attributes | +|-----------|-------------------|---------|----------------------| +| `azure.ai.agents` | `local_client.route.selected` | `azd ai agent run` resolves the service and protocol profile; this precedes client availability, agent startup, and client launch | `ext.route`: fixed enum `inspector`, `playground`, or `suppressed`; suppression takes precedence | +| `azure.ai.inspector` | `inspector.funnel.stage` | The Inspector SPA sends `setViewReady` after mounting | `ext.stage`: fixed enum `ui_ready`; `ext.outcome`: fixed enum `succeeded`; this does not indicate agent connection | + Because `ext.usage` spans share the command's trace, they join the originating command in Kusto on `operation_Id`. See [ADR-001](../../architecture/adr-001-extension-telemetry-events.md) for