diff --git a/src/rovo-dev/analytics/events.ts b/src/rovo-dev/analytics/events.ts index e2bc7f70b..c4e04ebf0 100644 --- a/src/rovo-dev/analytics/events.ts +++ b/src/rovo-dev/analytics/events.ts @@ -257,6 +257,21 @@ export namespace Track { }; // TODO: rovodev metadata fields here are different from other events, reconcile later? + export type PromptWarningReason = 'rate_limit'; + + export type PromptWarning = { + action: 'rovoDevPromptWarning'; + subject: 'atlascode'; + attributes: { + rovoDevEnv: RovoDevEnv; + appInstanceId: string; + sessionId: string; + promptId: string; + reason: PromptWarningReason; + title?: string; + }; + }; + export type PerformanceEvent = { action: 'performanceEvent'; subject: 'atlascode'; @@ -285,4 +300,5 @@ export type TrackEvent = | Track.ReplayCompleted | Track.PerformanceEvent | Track.LocalServerPromptReceived - | Track.PromptCompleted; + | Track.PromptCompleted + | Track.PromptWarning; diff --git a/src/rovo-dev/rovoDevChatProvider.test.ts b/src/rovo-dev/rovoDevChatProvider.test.ts index f8a689f34..0af7430c9 100644 --- a/src/rovo-dev/rovoDevChatProvider.test.ts +++ b/src/rovo-dev/rovoDevChatProvider.test.ts @@ -1260,4 +1260,72 @@ describe('RovoDevChatProvider', () => { }); }); }); + + describe('rovoDevPromptWarning (rate limit)', () => { + beforeEach(() => { + chatProvider = new RovoDevChatProvider(true, mockTelemetryProvider); + chatProvider.setWebview(mockWebview); + chatProvider['_currentPromptId'] = 'prompt-A'; + mockTelemetryProvider.fireTelemetryEvent.mockClear(); + }); + + it('detects rate-limit warnings by title or message (case-insensitive)', () => { + expect(chatProvider['isRateLimitWarning']({ title: 'Rate limit exceeded' })).toBe(true); + expect(chatProvider['isRateLimitWarning']({ message: "We've hit a RATE LIMIT" })).toBe(true); + expect(chatProvider['isRateLimitWarning']({ title: 'Heads up', message: 'Model switched' })).toBe(false); + expect(chatProvider['isRateLimitWarning']({ title: 'current rate', message: 'limit hit' })).toBe(false); + expect(chatProvider['isRateLimitWarning']({})).toBe(false); + }); + + it('emits reason=rate_limit (with title), can fire repeatedly, and omits absent title', () => { + chatProvider['firePromptWarning']('rate_limit', 'Rate limit exceeded'); + expect(mockTelemetryProvider.fireTelemetryEvent).toHaveBeenCalledWith({ + action: 'rovoDevPromptWarning', + subject: 'atlascode', + attributes: { promptId: 'prompt-A', reason: 'rate_limit', title: 'Rate limit exceeded' }, + }); + + chatProvider['firePromptWarning']('rate_limit'); + expect(mockTelemetryProvider.fireTelemetryEvent).toHaveBeenCalledTimes(2); + expect(mockTelemetryProvider.fireTelemetryEvent.mock.calls[1][0].attributes).toEqual({ + promptId: 'prompt-A', + reason: 'rate_limit', + }); + }); + + it('does not emit outside Boysenberry mode or without a current promptId', () => { + const ideProvider = new RovoDevChatProvider(false, mockTelemetryProvider); + ideProvider['_currentPromptId'] = 'prompt-ide'; + ideProvider['firePromptWarning']('rate_limit', 'Rate limit exceeded'); + + chatProvider['_currentPromptId'] = ''; + chatProvider['firePromptWarning']('rate_limit', 'Rate limit exceeded'); + + expect(mockTelemetryProvider.fireTelemetryEvent).not.toHaveBeenCalled(); + }); + + it('fires only for rate-limit chat warnings, never for other warnings or replay', async () => { + await chatProvider['processRovoDevResponse']('chat', { + event_kind: 'warning', + message: 'Some models are slow', + title: 'Heads up', + } as any); + await chatProvider['processRovoDevResponse']('replay', { + event_kind: 'warning', + message: "We'll try again in 10 seconds.", + title: 'Rate limit exceeded', + } as any); + expect(mockTelemetryProvider.fireTelemetryEvent).not.toHaveBeenCalled(); + + await chatProvider['processRovoDevResponse']('chat', { + event_kind: 'warning', + message: "We'll try again in 10 seconds.", + title: 'Rate limit exceeded', + } as any); + expect(mockTelemetryProvider.fireTelemetryEvent).toHaveBeenCalledTimes(1); + const call = mockTelemetryProvider.fireTelemetryEvent.mock.calls[0][0]; + expect(call.action).toBe('rovoDevPromptWarning'); + expect((call.attributes as { reason?: string }).reason).toBe('rate_limit'); + }); + }); }); diff --git a/src/rovo-dev/rovoDevChatProvider.ts b/src/rovo-dev/rovoDevChatProvider.ts index 489edc24e..d1804132b 100644 --- a/src/rovo-dev/rovoDevChatProvider.ts +++ b/src/rovo-dev/rovoDevChatProvider.ts @@ -771,6 +771,10 @@ export class RovoDevChatProvider { } case 'warning': { + if (sourceApi !== 'replay' && this.isRateLimitWarning(response)) { + this.firePromptWarning('rate_limit', response.title); + } + const { text, link } = this.parseExceptionMessage(response.message); await webview.postMessage({ type: RovoDevProviderMessageType.ShowDialog, @@ -1201,6 +1205,34 @@ export class RovoDevChatProvider { }); } + private isRateLimitWarning(response: { title?: string; message?: string }): boolean { + return ( + !!response.title?.toLowerCase().includes('rate limit') || + !!response.message?.toLowerCase().includes('rate limit') + ); + } + + private firePromptWarning(reason: Track.PromptWarningReason, title?: string): void { + if (!this._isBoysenberry) { + return; + } + + const promptId = this._currentPromptId; + if (!promptId) { + return; + } + + this._telemetryProvider.fireTelemetryEvent({ + action: 'rovoDevPromptWarning', + subject: 'atlascode', + attributes: { + promptId, + reason, + ...(title !== undefined ? { title } : {}), + }, + }); + } + private async processError( error: Error, { diff --git a/src/rovo-dev/rovoDevTelemetryProvider.ts b/src/rovo-dev/rovoDevTelemetryProvider.ts index d0d46ba43..c46a4a925 100644 --- a/src/rovo-dev/rovoDevTelemetryProvider.ts +++ b/src/rovo-dev/rovoDevTelemetryProvider.ts @@ -36,7 +36,8 @@ export type TelemetryEvent = | PartialEvent | PartialEvent | PartialEvent - | PartialEvent; + | PartialEvent + | PartialEvent; export type TelemetryScreenEvent = 'rovoDevSessionHistoryPicker'; @@ -139,6 +140,7 @@ export class RovoDevTelemetryProvider { eventId === 'atlascode_rovoDevFileChangedAction' || eventId === 'rovoDevCreatePrButton_clicked' || eventId === 'atlascode_rovoDevRestartProcessAction' || // We want to log every restart attempt + eventId === 'atlascode_rovoDevPromptWarning' || // Otherwise, only allow if not fired yet !this._firedTelemetryForCurrentPrompt[eventId] );