diff --git a/packages/cloud/src/TelemetryClient.ts b/packages/cloud/src/TelemetryClient.ts index 1213530a95b..f4b05feed53 100644 --- a/packages/cloud/src/TelemetryClient.ts +++ b/packages/cloud/src/TelemetryClient.ts @@ -69,6 +69,10 @@ abstract class BaseTelemetryClient implements TelemetryClient { public abstract capture(event: TelemetryEvent): Promise + public captureException(_error: Error, _additionalProperties?: Record): void { + // No-op - exception capture is only supported by PostHog + } + public setProvider(provider: TelemetryPropertiesProvider): void { this.providerRef = new WeakRef(provider) } diff --git a/packages/telemetry/src/BaseTelemetryClient.ts b/packages/telemetry/src/BaseTelemetryClient.ts index 2eb308b414b..5f59e1643b7 100644 --- a/packages/telemetry/src/BaseTelemetryClient.ts +++ b/packages/telemetry/src/BaseTelemetryClient.ts @@ -59,6 +59,8 @@ export abstract class BaseTelemetryClient implements TelemetryClient { public abstract capture(event: TelemetryEvent): Promise + public abstract captureException(error: Error, additionalProperties?: Record): void + public setProvider(provider: TelemetryPropertiesProvider): void { this.providerRef = new WeakRef(provider) } diff --git a/packages/telemetry/src/PostHogTelemetryClient.ts b/packages/telemetry/src/PostHogTelemetryClient.ts index d7f632f1379..cea945f1580 100644 --- a/packages/telemetry/src/PostHogTelemetryClient.ts +++ b/packages/telemetry/src/PostHogTelemetryClient.ts @@ -61,6 +61,22 @@ export class PostHogTelemetryClient extends BaseTelemetryClient { }) } + public override captureException(error: Error, additionalProperties?: Record): void { + if (!this.isTelemetryEnabled()) { + if (this.debug) { + console.info(`[PostHogTelemetryClient#captureException] Skipping exception: ${error.message}`) + } + + return + } + + if (this.debug) { + console.info(`[PostHogTelemetryClient#captureException] ${error.message}`) + } + + this.client.captureException(error, this.distinctId, additionalProperties) + } + /** * Updates the telemetry state based on user preferences and VSCode settings. * Only enables telemetry if both VSCode global telemetry is enabled and diff --git a/packages/telemetry/src/TelemetryService.ts b/packages/telemetry/src/TelemetryService.ts index 8f4fbe09741..ff94b524f84 100644 --- a/packages/telemetry/src/TelemetryService.ts +++ b/packages/telemetry/src/TelemetryService.ts @@ -65,6 +65,19 @@ export class TelemetryService { this.clients.forEach((client) => client.capture({ event: eventName, properties })) } + /** + * Captures an exception using PostHog's error tracking + * @param error The error to capture + * @param additionalProperties Additional properties to include with the exception + */ + public captureException(error: Error, additionalProperties?: Record): void { + if (!this.isReady) { + return + } + + this.clients.forEach((client) => client.captureException(error, additionalProperties)) + } + public captureTaskCreated(taskId: string): void { this.captureEvent(TelemetryEventName.TASK_CREATED, { taskId }) } diff --git a/packages/types/src/telemetry.ts b/packages/types/src/telemetry.ts index 233edfe499e..eb934a09263 100644 --- a/packages/types/src/telemetry.ts +++ b/packages/types/src/telemetry.ts @@ -262,7 +262,44 @@ export interface TelemetryClient { setProvider(provider: TelemetryPropertiesProvider): void capture(options: TelemetryEvent): Promise + captureException(error: Error, additionalProperties?: Record): void updateTelemetryState(isOptedIn: boolean): void isTelemetryEnabled(): boolean shutdown(): Promise } + +/** + * Expected API error codes that should not be reported to telemetry. + * These are normal/expected errors that users can't do much about. + */ +export const EXPECTED_API_ERROR_CODES = new Set([ + 429, // Rate limit - expected when hitting API limits +]) + +/** + * Helper to check if an API error should be reported to telemetry. + * Filters out expected errors like rate limits. + * @param errorCode - The HTTP error code (if available) + * @returns true if the error should be reported, false if it should be filtered out + */ +export function shouldReportApiErrorToTelemetry(errorCode?: number): boolean { + if (errorCode === undefined) return true + return !EXPECTED_API_ERROR_CODES.has(errorCode) +} + +/** + * Generic API provider error class for structured error tracking via PostHog. + * Can be reused by any API provider. + */ +export class ApiProviderError extends Error { + constructor( + message: string, + public readonly provider: string, + public readonly modelId: string, + public readonly operation: string, + public readonly errorCode?: number, + ) { + super(message) + this.name = "ApiProviderError" + } +} diff --git a/src/api/providers/__tests__/openrouter.spec.ts b/src/api/providers/__tests__/openrouter.spec.ts index 157c5aacdd4..3347d96431e 100644 --- a/src/api/providers/__tests__/openrouter.spec.ts +++ b/src/api/providers/__tests__/openrouter.spec.ts @@ -9,10 +9,21 @@ import OpenAI from "openai" import { OpenRouterHandler } from "../openrouter" import { ApiHandlerOptions } from "../../../shared/api" import { Package } from "../../../shared/package" +import { ApiProviderError } from "@roo-code/types" // Mock dependencies vitest.mock("openai") vitest.mock("delay", () => ({ default: vitest.fn(() => Promise.resolve()) })) + +// Mock TelemetryService +const mockCaptureException = vitest.fn() +vitest.mock("@roo-code/telemetry", () => ({ + TelemetryService: { + instance: { + captureException: (...args: unknown[]) => mockCaptureException(...args), + }, + }, +})) vitest.mock("../fetchers/modelCache", () => ({ getModels: vitest.fn().mockImplementation(() => { return Promise.resolve({ @@ -267,7 +278,7 @@ describe("OpenRouterHandler", () => { ) }) - it("handles API errors", async () => { + it("handles API errors and captures telemetry", async () => { const handler = new OpenRouterHandler(mockOptions) const mockStream = { async *[Symbol.asyncIterator]() { @@ -282,6 +293,52 @@ describe("OpenRouterHandler", () => { const generator = handler.createMessage("test", []) await expect(generator.next()).rejects.toThrow("OpenRouter API Error 500: API Error") + + // Verify telemetry was captured + expect(mockCaptureException).toHaveBeenCalledWith(expect.any(ApiProviderError), { + provider: "OpenRouter", + modelId: mockOptions.openRouterModelId, + operation: "createMessage", + errorCode: 500, + }) + }) + + it("captures telemetry when createMessage throws an exception", async () => { + const handler = new OpenRouterHandler(mockOptions) + const mockCreate = vitest.fn().mockRejectedValue(new Error("Connection failed")) + ;(OpenAI as any).prototype.chat = { + completions: { create: mockCreate }, + } as any + + const generator = handler.createMessage("test", []) + await expect(generator.next()).rejects.toThrow() + + // Verify telemetry was captured + expect(mockCaptureException).toHaveBeenCalledWith(expect.any(ApiProviderError), { + provider: "OpenRouter", + modelId: mockOptions.openRouterModelId, + operation: "createMessage", + }) + }) + + it("does NOT capture telemetry for 429 rate limit errors", async () => { + const handler = new OpenRouterHandler(mockOptions) + const mockStream = { + async *[Symbol.asyncIterator]() { + yield { error: { message: "Rate limit exceeded", code: 429 } } + }, + } + + const mockCreate = vitest.fn().mockResolvedValue(mockStream) + ;(OpenAI as any).prototype.chat = { + completions: { create: mockCreate }, + } as any + + const generator = handler.createMessage("test", []) + await expect(generator.next()).rejects.toThrow("OpenRouter API Error 429: Rate limit exceeded") + + // Verify telemetry was NOT captured for 429 errors + expect(mockCaptureException).not.toHaveBeenCalled() }) it("yields tool_call_end events when finish_reason is tool_calls", async () => { @@ -384,7 +441,7 @@ describe("OpenRouterHandler", () => { ) }) - it("handles API errors", async () => { + it("handles API errors and captures telemetry", async () => { const handler = new OpenRouterHandler(mockOptions) const mockError = { error: { @@ -399,9 +456,17 @@ describe("OpenRouterHandler", () => { } as any await expect(handler.completePrompt("test prompt")).rejects.toThrow("OpenRouter API Error 500: API Error") + + // Verify telemetry was captured + expect(mockCaptureException).toHaveBeenCalledWith(expect.any(ApiProviderError), { + provider: "OpenRouter", + modelId: mockOptions.openRouterModelId, + operation: "completePrompt", + errorCode: 500, + }) }) - it("handles unexpected errors", async () => { + it("handles unexpected errors and captures telemetry", async () => { const handler = new OpenRouterHandler(mockOptions) const mockCreate = vitest.fn().mockRejectedValue(new Error("Unexpected error")) ;(OpenAI as any).prototype.chat = { @@ -409,6 +474,35 @@ describe("OpenRouterHandler", () => { } as any await expect(handler.completePrompt("test prompt")).rejects.toThrow("Unexpected error") + + // Verify telemetry was captured + expect(mockCaptureException).toHaveBeenCalledWith(expect.any(ApiProviderError), { + provider: "OpenRouter", + modelId: mockOptions.openRouterModelId, + operation: "completePrompt", + }) + }) + + it("does NOT capture telemetry for 429 rate limit errors", async () => { + const handler = new OpenRouterHandler(mockOptions) + const mockError = { + error: { + message: "Rate limit exceeded", + code: 429, + }, + } + + const mockCreate = vitest.fn().mockResolvedValue(mockError) + ;(OpenAI as any).prototype.chat = { + completions: { create: mockCreate }, + } as any + + await expect(handler.completePrompt("test prompt")).rejects.toThrow( + "OpenRouter API Error 429: Rate limit exceeded", + ) + + // Verify telemetry was NOT captured for 429 errors + expect(mockCaptureException).not.toHaveBeenCalled() }) }) }) diff --git a/src/api/providers/openrouter.ts b/src/api/providers/openrouter.ts index ef2c553b549..ec9e04c4feb 100644 --- a/src/api/providers/openrouter.ts +++ b/src/api/providers/openrouter.ts @@ -7,7 +7,10 @@ import { OPENROUTER_DEFAULT_PROVIDER_NAME, OPEN_ROUTER_PROMPT_CACHING_MODELS, DEEP_SEEK_DEFAULT_TEMPERATURE, + shouldReportApiErrorToTelemetry, + ApiProviderError, } from "@roo-code/types" +import { TelemetryService } from "@roo-code/telemetry" import { NativeToolCallParser } from "../../core/assistant-message/NativeToolCallParser" @@ -224,6 +227,15 @@ export class OpenRouterHandler extends BaseProvider implements SingleCompletionH try { stream = await this.client.chat.completions.create(completionParams, requestOptions) } catch (error) { + TelemetryService.instance.captureException( + new ApiProviderError( + error instanceof Error ? error.message : String(error), + this.providerName, + modelId, + "createMessage", + ), + { provider: this.providerName, modelId, operation: "createMessage" }, + ) throw handleOpenAIError(error, this.providerName) } @@ -248,6 +260,18 @@ export class OpenRouterHandler extends BaseProvider implements SingleCompletionH if ("error" in chunk) { const error = chunk.error as { message?: string; code?: number } console.error(`OpenRouter API Error: ${error?.code} - ${error?.message}`) + if (shouldReportApiErrorToTelemetry(error?.code)) { + TelemetryService.instance.captureException( + new ApiProviderError( + error?.message ?? "Unknown error", + this.providerName, + modelId, + "createMessage", + error?.code, + ), + { provider: this.providerName, modelId, operation: "createMessage", errorCode: error?.code }, + ) + } throw new Error(`OpenRouter API Error ${error?.code}: ${error?.message}`) } @@ -442,11 +466,32 @@ export class OpenRouterHandler extends BaseProvider implements SingleCompletionH try { response = await this.client.chat.completions.create(completionParams, requestOptions) } catch (error) { + TelemetryService.instance.captureException( + new ApiProviderError( + error instanceof Error ? error.message : String(error), + this.providerName, + modelId, + "completePrompt", + ), + { provider: this.providerName, modelId, operation: "completePrompt" }, + ) throw handleOpenAIError(error, this.providerName) } if ("error" in response) { const error = response.error as { message?: string; code?: number } + if (shouldReportApiErrorToTelemetry(error?.code)) { + TelemetryService.instance.captureException( + new ApiProviderError( + error?.message ?? "Unknown error", + this.providerName, + modelId, + "completePrompt", + error?.code, + ), + { provider: this.providerName, modelId, operation: "completePrompt", errorCode: error?.code }, + ) + } throw new Error(`OpenRouter API Error ${error?.code}: ${error?.message}`) } diff --git a/src/core/assistant-message/presentAssistantMessage.ts b/src/core/assistant-message/presentAssistantMessage.ts index bcbf0a35f77..84f00806da4 100644 --- a/src/core/assistant-message/presentAssistantMessage.ts +++ b/src/core/assistant-message/presentAssistantMessage.ts @@ -42,8 +42,6 @@ import { Task } from "../task/Task" import { codebaseSearchTool } from "../tools/CodebaseSearchTool" import { experiments, EXPERIMENT_IDS } from "../../shared/experiments" import { applyDiffTool as applyDiffToolClass } from "../tools/ApplyDiffTool" -import { isNativeProtocol } from "@roo-code/types" -import { resolveToolProtocol } from "../../utils/resolveToolProtocol" /** * Processes and presents assistant message content to the user interface. diff --git a/src/core/task/Task.ts b/src/core/task/Task.ts index 488562e8f9a..8ed9ab56405 100644 --- a/src/core/task/Task.ts +++ b/src/core/task/Task.ts @@ -849,6 +849,7 @@ export class Task extends EventEmitter implements TaskLike { role: "user", content: this.userMessageContent, } + const userMessageWithTs = { ...userMessage, ts: Date.now() } this.apiConversationHistory.push(userMessageWithTs as ApiMessage)