diff --git a/messages/en/provider-chain.json b/messages/en/provider-chain.json index b7ed57bb4..a347e8dc8 100644 --- a/messages/en/provider-chain.json +++ b/messages/en/provider-chain.json @@ -37,7 +37,8 @@ "systemError": "System Error", "concurrentLimit": "Concurrent Limit", "http2Fallback": "HTTP/2 Fallback", - "clientError": "Client Error" + "clientError": "Client Error", + "endpointPoolExhausted": "Endpoint Pool Exhausted" }, "reasons": { "request_success": "Success", @@ -48,7 +49,8 @@ "concurrent_limit_failed": "Concurrent Limit", "http2_fallback": "HTTP/2 Fallback", "session_reuse": "Session Reuse", - "initial_selection": "Initial Selection" + "initial_selection": "Initial Selection", + "endpoint_pool_exhausted": "Endpoint Pool Exhausted" }, "filterReasons": { "rate_limited": "Rate Limited", @@ -61,7 +63,9 @@ "context_1m_disabled": "1M Context Disabled", "model_not_supported": "Model Not Supported", "group_mismatch": "Group Mismatch", - "health_check_failed": "Health Check Failed" + "health_check_failed": "Health Check Failed", + "endpoint_circuit_open": "Endpoint Circuit Open", + "endpoint_disabled": "Endpoint Disabled" }, "details": { "selectionMethod": "Selection", @@ -185,6 +189,14 @@ "ruleMatchType": "Match Type: {matchType}", "ruleDescription": "Description: {description}", "ruleHasOverride": "Overrides: response={response}, statusCode={statusCode}", - "clientErrorNote": "This error is caused by client input and is not retried or counted in the circuit breaker." + "clientErrorNote": "This error is caused by client input and is not retried or counted in the circuit breaker.", + "endpointPoolExhausted": "Endpoint Pool Exhausted (all endpoints unavailable)", + "endpointStats": "Endpoint Filter Stats", + "endpointStatsTotal": "Total Endpoints: {count}", + "endpointStatsEnabled": "Enabled Endpoints: {count}", + "endpointStatsCircuitOpen": "Circuit-Open Endpoints: {count}", + "endpointStatsAvailable": "Available Endpoints: {count}", + "strictBlockNoEndpoints": "Strict mode: no endpoint candidates available, provider skipped without fallback", + "strictBlockSelectorError": "Strict mode: endpoint selector encountered an error, provider skipped without fallback" } } diff --git a/messages/en/settings/providers/filter.json b/messages/en/settings/providers/filter.json index 7512e5141..3d30c6830 100644 --- a/messages/en/settings/providers/filter.json +++ b/messages/en/settings/providers/filter.json @@ -1,5 +1,7 @@ { "circuitBroken": "Circuit Broken", + "keyCircuitBroken": "Key Circuit Broken", + "endpointCircuitBroken": "Endpoint Circuit Broken", "groups": { "all": "All", "default": "default", diff --git a/messages/en/settings/providers/list.json b/messages/en/settings/providers/list.json index 99b69c9fc..f1656c930 100644 --- a/messages/en/settings/providers/list.json +++ b/messages/en/settings/providers/list.json @@ -1,6 +1,8 @@ { "cancelButton": "Cancel", "circuitBroken": "Circuit Broken", + "keyCircuitBroken": "Key Circuit", + "endpointCircuitBroken": "Endpoint Circuit", "clipboardUnavailable": "Clipboard access is blocked in this environment. Select and copy the key manually.", "confirmDeleteMessage": "Are you sure you want to delete provider \"{name}\"? This action cannot be undone.", "confirmDeleteTitle": "Confirm Delete Provider?", diff --git a/messages/en/settings/providers/strings.json b/messages/en/settings/providers/strings.json index 252d426ed..486d742a9 100644 --- a/messages/en/settings/providers/strings.json +++ b/messages/en/settings/providers/strings.json @@ -112,6 +112,9 @@ "unhealthy": "Unhealthy", "unknown": "Unknown", "circuitOpen": "Circuit Open", - "circuitHalfOpen": "Circuit Half-Open" + "circuitHalfOpen": "Circuit Half-Open", + "resetCircuit": "Reset Circuit", + "resetCircuitSuccess": "Endpoint circuit breaker reset", + "resetCircuitFailed": "Failed to reset endpoint circuit breaker" } } diff --git a/messages/ja/provider-chain.json b/messages/ja/provider-chain.json index 4910f5e0f..37adb84f9 100644 --- a/messages/ja/provider-chain.json +++ b/messages/ja/provider-chain.json @@ -37,7 +37,8 @@ "systemError": "システムエラー", "concurrentLimit": "同時実行制限", "http2Fallback": "HTTP/2 フォールバック", - "clientError": "クライアントエラー" + "clientError": "クライアントエラー", + "endpointPoolExhausted": "エンドポイントプール枯渇" }, "reasons": { "request_success": "成功", @@ -48,7 +49,8 @@ "concurrent_limit_failed": "同時実行制限", "http2_fallback": "HTTP/2 フォールバック", "session_reuse": "セッション再利用", - "initial_selection": "初期選択" + "initial_selection": "初期選択", + "endpoint_pool_exhausted": "エンドポイントプール枯渇" }, "filterReasons": { "rate_limited": "レート制限", @@ -61,7 +63,9 @@ "context_1m_disabled": "1Mコンテキスト無効", "model_not_supported": "モデル非対応", "group_mismatch": "グループ不一致", - "health_check_failed": "ヘルスチェック失敗" + "health_check_failed": "ヘルスチェック失敗", + "endpoint_circuit_open": "エンドポイントサーキットオープン", + "endpoint_disabled": "エンドポイント無効" }, "details": { "selectionMethod": "選択方法", @@ -185,6 +189,14 @@ "ruleMatchType": "一致タイプ: {matchType}", "ruleDescription": "説明: {description}", "ruleHasOverride": "上書き: 応答={response} ステータスコード={statusCode}", - "clientErrorNote": "このエラーはクライアント入力が原因のため再試行せず、サーキットブレーカーにもカウントされません。" + "clientErrorNote": "このエラーはクライアント入力が原因のため再試行せず、サーキットブレーカーにもカウントされません。", + "endpointPoolExhausted": "エンドポイントプール枯渇（全エンドポイント利用不可）", + "endpointStats": "エンドポイントフィルタ統計", + "endpointStatsTotal": "総エンドポイント数: {count}", + "endpointStatsEnabled": "有効なエンドポイント: {count}", + "endpointStatsCircuitOpen": "サーキットオープンのエンドポイント: {count}", + "endpointStatsAvailable": "利用可能なエンドポイント: {count}", + "strictBlockNoEndpoints": "厳格モード：利用可能なエンドポイント候補がないため、フォールバックなしでプロバイダーをスキップ", + "strictBlockSelectorError": "厳格モード：エンドポイントセレクターでエラーが発生したため、フォールバックなしでプロバイダーをスキップ" } } diff --git a/messages/ja/settings/providers/filter.json b/messages/ja/settings/providers/filter.json index 119f91aac..8315ecac4 100644 --- a/messages/ja/settings/providers/filter.json +++ b/messages/ja/settings/providers/filter.json @@ -1,5 +1,7 @@ { "circuitBroken": "サーキットブレーカー", + "keyCircuitBroken": "キー遮断", + "endpointCircuitBroken": "エンドポイント遮断", "groups": { "all": "すべて", "default": "default", diff --git a/messages/ja/settings/providers/list.json b/messages/ja/settings/providers/list.json index 250012f9b..a0005c2df 100644 --- a/messages/ja/settings/providers/list.json +++ b/messages/ja/settings/providers/list.json @@ -1,6 +1,8 @@ { "cancelButton": "キャンセル", "circuitBroken": "遮断中", + "keyCircuitBroken": "キー遮断", + "endpointCircuitBroken": "エンドポイント遮断", "clipboardUnavailable": "この環境ではクリップボードを使用できません。手動でコピーしてください。", "confirmDeleteMessage": "プロバイダー \"{name}\" を削除してもよろしいですか？この操作は元に戻せません。", "confirmDeleteTitle": "プロバイダーの削除を確認しますか?", diff --git a/messages/ja/settings/providers/strings.json b/messages/ja/settings/providers/strings.json index f41bd00cd..26f2f6b9b 100644 --- a/messages/ja/settings/providers/strings.json +++ b/messages/ja/settings/providers/strings.json @@ -112,6 +112,9 @@ "unhealthy": "異常", "unknown": "不明", "circuitOpen": "サーキットオープン", - "circuitHalfOpen": "サーキット半開" + "circuitHalfOpen": "サーキット半開", + "resetCircuit": "サーキットリセット", + "resetCircuitSuccess": "エンドポイントのサーキットブレーカーをリセットしました", + "resetCircuitFailed": "エンドポイントのサーキットブレーカーのリセットに失敗しました" } } diff --git a/messages/ru/provider-chain.json b/messages/ru/provider-chain.json index 1f9e67efa..e37650b04 100644 --- a/messages/ru/provider-chain.json +++ b/messages/ru/provider-chain.json @@ -37,7 +37,8 @@ "systemError": "Системная ошибка", "concurrentLimit": "Лимит параллельных запросов", "http2Fallback": "Откат HTTP/2", - "clientError": "Ошибка клиента" + "clientError": "Ошибка клиента", + "endpointPoolExhausted": "Пул конечная точкаов исчерпан" }, "reasons": { "request_success": "Успешно", @@ -48,7 +49,8 @@ "concurrent_limit_failed": "Лимит параллельных запросов", "http2_fallback": "Откат HTTP/2", "session_reuse": "Повторное использование сессии", - "initial_selection": "Первоначальный выбор" + "initial_selection": "Первоначальный выбор", + "endpoint_pool_exhausted": "Пул конечная точкаов исчерпан" }, "filterReasons": { "rate_limited": "Ограничение скорости", @@ -61,7 +63,9 @@ "context_1m_disabled": "1M контекст отключен", "model_not_supported": "Модель не поддерживается", "group_mismatch": "Несоответствие группы", - "health_check_failed": "Проверка состояния не пройдена" + "health_check_failed": "Проверка состояния не пройдена", + "endpoint_circuit_open": "Автомат конечная точкаа открыт", + "endpoint_disabled": "Эндпоинт отключен" }, "details": { "selectionMethod": "Метод выбора", @@ -185,6 +189,14 @@ "ruleMatchType": "Тип совпадения: {matchType}", "ruleDescription": "Описание: {description}", "ruleHasOverride": "Переопределения: response={response}, statusCode={statusCode}", - "clientErrorNote": "Эта ошибка вызвана вводом клиента, не повторяется и не учитывается в автомате защиты." + "clientErrorNote": "Эта ошибка вызвана вводом клиента, не повторяется и не учитывается в автомате защиты.", + "endpointPoolExhausted": "Пул конечная точкаов исчерпан (все конечная точкаы недоступны)", + "endpointStats": "Статистика фильтрации конечная точкаов", + "endpointStatsTotal": "Всего конечная точкаов: {count}", + "endpointStatsEnabled": "Включено конечная точкаов: {count}", + "endpointStatsCircuitOpen": "Эндпоинтов с открытым автоматом: {count}", + "endpointStatsAvailable": "Доступных конечная точкаов: {count}", + "strictBlockNoEndpoints": "Строгий режим: нет доступных кандидатов конечная точкаов, провайдер пропущен без отката", + "strictBlockSelectorError": "Строгий режим: ошибка селектора конечная точкаов, провайдер пропущен без отката" } } diff --git a/messages/ru/settings/providers/filter.json b/messages/ru/settings/providers/filter.json index adecbee1b..77e4d0474 100644 --- a/messages/ru/settings/providers/filter.json +++ b/messages/ru/settings/providers/filter.json @@ -1,5 +1,7 @@ { "circuitBroken": "Сбой соединения", + "keyCircuitBroken": "Разрыв ключа", + "endpointCircuitBroken": "Разрыв конечной точки", "groups": { "all": "Все", "default": "default", diff --git a/messages/ru/settings/providers/list.json b/messages/ru/settings/providers/list.json index 71bbe2e38..7ae099a72 100644 --- a/messages/ru/settings/providers/list.json +++ b/messages/ru/settings/providers/list.json @@ -1,6 +1,8 @@ { "cancelButton": "Отмена", "circuitBroken": "Разорвано", + "keyCircuitBroken": "Разрыв ключа", + "endpointCircuitBroken": "Разрыв endpoint", "clipboardUnavailable": "Буфер обмена недоступен в этой среде. Скопируйте ключ вручную.", "confirmDeleteMessage": "Вы уверены, что хотите удалить провайдера \"{name}\"? Это действие нельзя отменить.", "confirmDeleteTitle": "Подтвердить удаление провайдера?", diff --git a/messages/ru/settings/providers/strings.json b/messages/ru/settings/providers/strings.json index 04f9f047e..7c9b932d8 100644 --- a/messages/ru/settings/providers/strings.json +++ b/messages/ru/settings/providers/strings.json @@ -112,6 +112,9 @@ "unhealthy": "Недоступен", "unknown": "Неизвестно", "circuitOpen": "Цепь открыта", - "circuitHalfOpen": "Цепь полуоткрыта" + "circuitHalfOpen": "Цепь полуоткрыта", + "resetCircuit": "Сброс цепи", + "resetCircuitSuccess": "Цепь эндпоинта сброшена", + "resetCircuitFailed": "Не удалось сбросить цепь эндпоинта" } } diff --git a/messages/zh-CN/provider-chain.json b/messages/zh-CN/provider-chain.json index 12f8f1bdf..fe75d85a1 100644 --- a/messages/zh-CN/provider-chain.json +++ b/messages/zh-CN/provider-chain.json @@ -37,7 +37,8 @@ "systemError": "系统错误", "concurrentLimit": "并发限制", "http2Fallback": "HTTP/2 回退", - "clientError": "客户端错误" + "clientError": "客户端错误", + "endpointPoolExhausted": "端点池耗尽" }, "reasons": { "request_success": "成功", @@ -48,7 +49,8 @@ "concurrent_limit_failed": "并发限制", "http2_fallback": "HTTP/2 回退", "session_reuse": "会话复用", - "initial_selection": "首次选择" + "initial_selection": "首次选择", + "endpoint_pool_exhausted": "端点池耗尽" }, "filterReasons": { "rate_limited": "速率限制", @@ -61,7 +63,9 @@ "context_1m_disabled": "1M上下文已禁用", "model_not_supported": "不支持该模型", "group_mismatch": "分组不匹配", - "health_check_failed": "健康检查失败" + "health_check_failed": "健康检查失败", + "endpoint_circuit_open": "端点已熔断", + "endpoint_disabled": "端点已禁用" }, "details": { "selectionMethod": "选择方式", @@ -185,6 +189,14 @@ "ruleMatchType": "匹配类型: {matchType}", "ruleDescription": "规则描述: {description}", "ruleHasOverride": "覆写配置: 响应体={response}, 状态码={statusCode}", - "clientErrorNote": "此错误由用户输入导致，不会重试，不计入熔断器。" + "clientErrorNote": "此错误由用户输入导致，不会重试，不计入熔断器。", + "endpointPoolExhausted": "端点池耗尽（所有端点不可用）", + "endpointStats": "端点过滤统计", + "endpointStatsTotal": "总端点数: {count}", + "endpointStatsEnabled": "已启用端点: {count}", + "endpointStatsCircuitOpen": "已熔断端点: {count}", + "endpointStatsAvailable": "可用端点: {count}", + "strictBlockNoEndpoints": "严格模式：无可用端点候选，跳过该供应商且不降级", + "strictBlockSelectorError": "严格模式：端点选择器发生错误，跳过该供应商且不降级" } } diff --git a/messages/zh-CN/settings/providers/filter.json b/messages/zh-CN/settings/providers/filter.json index 09e9792ba..320768094 100644 --- a/messages/zh-CN/settings/providers/filter.json +++ b/messages/zh-CN/settings/providers/filter.json @@ -10,6 +10,8 @@ "default": "default" }, "circuitBroken": "熔断", + "keyCircuitBroken": "Key 熔断", + "endpointCircuitBroken": "端点熔断", "mobileFilter": "筛选", "mobileFilterCount": "筛选 ({count})", "resetFilters": "重置筛选" diff --git a/messages/zh-CN/settings/providers/list.json b/messages/zh-CN/settings/providers/list.json index a6c0c17f3..235ffd874 100644 --- a/messages/zh-CN/settings/providers/list.json +++ b/messages/zh-CN/settings/providers/list.json @@ -5,6 +5,8 @@ "todayUsageLabel": "今日用量", "todayUsageCount": "{count} 次", "circuitBroken": "熔断中", + "keyCircuitBroken": "Key 熔断", + "endpointCircuitBroken": "端点熔断", "officialWebsite": "官网", "viewFullKey": "查看完整 API Key", "viewFullKeyDesc": "请妥善保管，不要泄露给他人", diff --git a/messages/zh-CN/settings/providers/strings.json b/messages/zh-CN/settings/providers/strings.json index 5c3de1c00..a30799180 100644 --- a/messages/zh-CN/settings/providers/strings.json +++ b/messages/zh-CN/settings/providers/strings.json @@ -112,6 +112,9 @@ "unhealthy": "故障", "unknown": "未知", "circuitOpen": "熔断开启", - "circuitHalfOpen": "熔断半开" + "circuitHalfOpen": "熔断半开", + "resetCircuit": "重置熔断", + "resetCircuitSuccess": "端点熔断已重置", + "resetCircuitFailed": "重置端点熔断失败" } } diff --git a/messages/zh-TW/provider-chain.json b/messages/zh-TW/provider-chain.json index cdd98fc06..04aa28488 100644 --- a/messages/zh-TW/provider-chain.json +++ b/messages/zh-TW/provider-chain.json @@ -37,7 +37,8 @@ "systemError": "系統錯誤", "concurrentLimit": "並發限制", "http2Fallback": "HTTP/2 回退", - "clientError": "客戶端錯誤" + "clientError": "客戶端錯誤", + "endpointPoolExhausted": "端點池耗盡" }, "reasons": { "request_success": "成功", @@ -48,7 +49,8 @@ "concurrent_limit_failed": "並發限制", "http2_fallback": "HTTP/2 回退", "session_reuse": "會話複用", - "initial_selection": "首次選擇" + "initial_selection": "首次選擇", + "endpoint_pool_exhausted": "端點池耗盡" }, "filterReasons": { "rate_limited": "速率限制", @@ -61,7 +63,9 @@ "context_1m_disabled": "1M上下文已停用", "model_not_supported": "不支援該模型", "group_mismatch": "分組不匹配", - "health_check_failed": "健康檢查失敗" + "health_check_failed": "健康檢查失敗", + "endpoint_circuit_open": "端點已熔斷", + "endpoint_disabled": "端點已停用" }, "details": { "selectionMethod": "選擇方式", @@ -185,6 +189,14 @@ "ruleMatchType": "匹配類型: {matchType}", "ruleDescription": "規則描述: {description}", "ruleHasOverride": "覆寫設定: 回應體={response}, 狀態碼={statusCode}", - "clientErrorNote": "此錯誤由使用者輸入導致，不會重試，不計入熔斷器。" + "clientErrorNote": "此錯誤由使用者輸入導致，不會重試，不計入熔斷器。", + "endpointPoolExhausted": "端點池耗盡（所有端點不可用）", + "endpointStats": "端點過濾統計", + "endpointStatsTotal": "總端點數: {count}", + "endpointStatsEnabled": "已啟用端點: {count}", + "endpointStatsCircuitOpen": "已熔斷端點: {count}", + "endpointStatsAvailable": "可用端點: {count}", + "strictBlockNoEndpoints": "嚴格模式：無可用端點候選，跳過該供應商且不降級", + "strictBlockSelectorError": "嚴格模式：端點選擇器發生錯誤，跳過該供應商且不降級" } } diff --git a/messages/zh-TW/settings/providers/filter.json b/messages/zh-TW/settings/providers/filter.json index 765aa2961..45fa361df 100644 --- a/messages/zh-TW/settings/providers/filter.json +++ b/messages/zh-TW/settings/providers/filter.json @@ -1,5 +1,7 @@ { "circuitBroken": "熔斷", + "keyCircuitBroken": "Key 熔斷", + "endpointCircuitBroken": "端點熔斷", "groups": { "all": "所有", "default": "default", diff --git a/messages/zh-TW/settings/providers/list.json b/messages/zh-TW/settings/providers/list.json index 620ce129f..bcf7c571d 100644 --- a/messages/zh-TW/settings/providers/list.json +++ b/messages/zh-TW/settings/providers/list.json @@ -1,6 +1,8 @@ { "cancelButton": "關閉", "circuitBroken": "熔斷中", + "keyCircuitBroken": "Key 熔斷", + "endpointCircuitBroken": "端點熔斷", "clipboardUnavailable": "目前環境無法使用剪貼簿，請手動選取複製。", "confirmDeleteMessage": "確定要刪除供應商 \"{name}\" 嗎？此操作無法撤銷。", "confirmDeleteTitle": "確認刪除供應商？", diff --git a/messages/zh-TW/settings/providers/strings.json b/messages/zh-TW/settings/providers/strings.json index bfa07f2ea..1b89133e0 100644 --- a/messages/zh-TW/settings/providers/strings.json +++ b/messages/zh-TW/settings/providers/strings.json @@ -112,6 +112,9 @@ "unhealthy": "故障", "unknown": "未知", "circuitOpen": "熔斷開啟", - "circuitHalfOpen": "熔斷半開" + "circuitHalfOpen": "熔斷半開", + "resetCircuit": "重置熔斷", + "resetCircuitSuccess": "端點熔斷已重置", + "resetCircuitFailed": "重置端點熔斷失敗" } } diff --git a/src/actions/provider-endpoints.ts b/src/actions/provider-endpoints.ts index d446e9098..9b4977f54 100644 --- a/src/actions/provider-endpoints.ts +++ b/src/actions/provider-endpoints.ts @@ -123,6 +123,10 @@ const SetVendorTypeManualOpenSchema = z.object({ manualOpen: z.boolean(), }); +const BatchGetEndpointCircuitInfoSchema = z.object({ + endpointIds: z.array(EndpointIdSchema).max(500), +}); + async function getAdminSession() { const session = await getSession(); if (!session || session.user.role !== "admin") { @@ -547,6 +551,59 @@ export async function getEndpointCircuitInfo(input: unknown): Promise< } } +export async function batchGetEndpointCircuitInfo(input: unknown): Promise< + ActionResult< + Array<{ + endpointId: number; + circuitState: "closed" | "open" | "half-open"; + failureCount: number; + circuitOpenUntil: number | null; + }> + > +> { + try { + const session = await getAdminSession(); + if (!session) { + return { + ok: false, + error: "无权限执行此操作", + errorCode: ERROR_CODES.PERMISSION_DENIED, + }; + } + + const parsed = BatchGetEndpointCircuitInfoSchema.safeParse(input); + if (!parsed.success) { + return { + ok: false, + error: formatZodError(parsed.error), + errorCode: extractZodErrorCode(parsed.error), + }; + } + + if (parsed.data.endpointIds.length === 0) { + return { ok: true, data: [] }; + } + + const results = await Promise.all( + parsed.data.endpointIds.map(async (endpointId) => { + const { health } = await getEndpointHealthInfo(endpointId); + return { + endpointId, + circuitState: health.circuitState, + failureCount: health.failureCount, + circuitOpenUntil: health.circuitOpenUntil, + }; + }) + ); + + return { ok: true, data: results }; + } catch (error) { + logger.error("batchGetEndpointCircuitInfo:error", error); + const message = error instanceof Error ? error.message : "批量获取端点熔断状态失败"; + return { ok: false, error: message, errorCode: ERROR_CODES.OPERATION_FAILED }; + } +} + export async function resetEndpointCircuit(input: unknown): Promise { try { const session = await getAdminSession(); diff --git a/src/app/[locale]/settings/providers/_components/endpoint-status.ts b/src/app/[locale]/settings/providers/_components/endpoint-status.ts index b7425beb3..e8a97f81c 100644 --- a/src/app/[locale]/settings/providers/_components/endpoint-status.ts +++ b/src/app/[locale]/settings/providers/_components/endpoint-status.ts @@ -19,6 +19,22 @@ export type EndpointStatusToken = | "circuit-open" | "circuit-half-open"; +/** + * Source of incident for unified status semantics. + * - "provider": Incident originates from provider-level health check + * - "endpoint": Incident originates from endpoint-level health check (circuit breaker) + */ +export type IncidentSource = "provider" | "endpoint"; + +/** + * Resolved display status for endpoint with unified semantics. + */ +export interface EndpointDisplayStatus { + status: string; + source: IncidentSource; + priority: number; +} + export interface EndpointStatusModel { status: EndpointStatusToken; labelKey: string; @@ -105,3 +121,47 @@ export function getEndpointStatusModel( borderColor: "border-slate-400/30", }; } + +/** + * Resolves the display status for an endpoint with unified semantics. + * + * Priority order: + * 1. circuit-open (priority 0) - Circuit breaker has opened + * 2. circuit-half-open (priority 1) - Circuit breaker is testing recovery + * 3. enabled (priority 2) - Circuit closed and endpoint is enabled + * 4. disabled (priority 3) - Circuit closed but endpoint is disabled + * + * @param endpoint - Endpoint data with optional isEnabled property + * @param circuitState - Current circuit breaker state + * @returns Display status with source indicator and priority + */ +export function resolveEndpointDisplayStatus( + endpoint: { lastProbeOk: boolean | null; isEnabled?: boolean | null }, + circuitState?: EndpointCircuitState | null +): EndpointDisplayStatus { + // Priority 0: Circuit Open + if (circuitState === "open") { + return { + status: "circuit-open", + source: "endpoint", + priority: 0, + }; + } + + // Priority 1: Circuit Half-Open + if (circuitState === "half-open") { + return { + status: "circuit-half-open", + source: "endpoint", + priority: 1, + }; + } + + // Priority 2/3: Circuit Closed - check enabled/disabled (no circuit incident) + const isExplicitlyDisabled = endpoint.isEnabled === false; + return { + status: isExplicitlyDisabled ? "disabled" : "enabled", + source: "provider", + priority: isExplicitlyDisabled ? 3 : 2, + }; +} diff --git a/src/app/[locale]/settings/providers/_components/provider-endpoints-table.tsx b/src/app/[locale]/settings/providers/_components/provider-endpoints-table.tsx index 3b246b6dd..3da614132 100644 --- a/src/app/[locale]/settings/providers/_components/provider-endpoints-table.tsx +++ b/src/app/[locale]/settings/providers/_components/provider-endpoints-table.tsx @@ -1,17 +1,19 @@ "use client"; import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; -import { Edit2, Loader2, MoreHorizontal, Play, Plus, Trash2 } from "lucide-react"; +import { Edit2, Loader2, MoreHorizontal, Play, Plus, RotateCcw, Trash2 } from "lucide-react"; import { useTranslations } from "next-intl"; import { useEffect, useMemo, useState } from "react"; import { toast } from "sonner"; import { addProviderEndpoint, + batchGetEndpointCircuitInfo, editProviderEndpoint, getProviderEndpoints, getProviderEndpointsByVendor, probeProviderEndpoint, removeProviderEndpoint, + resetEndpointCircuit, } from "@/actions/provider-endpoints"; import { Badge } from "@/components/ui/badge"; import { Button } from "@/components/ui/button"; @@ -58,6 +60,7 @@ import { import { getErrorMessage } from "@/lib/utils/error-messages"; import type { ProviderEndpoint, ProviderType } from "@/types/provider"; import { EndpointLatencySparkline } from "./endpoint-latency-sparkline"; +import type { EndpointCircuitState } from "./endpoint-status"; import { UrlPreview } from "./forms/url-preview"; // ============================================================================ @@ -127,6 +130,24 @@ export function ProviderEndpointsTable({ }); }, [rawEndpoints]); + // Fetch circuit breaker states for all endpoints in batch + const endpointIds = useMemo(() => endpoints.map((ep) => ep.id), [endpoints]); + const { data: circuitInfoMap = {} } = useQuery({ + queryKey: ["endpoint-circuit-info", endpointIds.toSorted((a, b) => a - b).join(",")], + queryFn: async () => { + if (endpointIds.length === 0) return {}; + const res = await batchGetEndpointCircuitInfo({ endpointIds }); + if (!res.ok || !res.data) return {}; + const map: Record = {}; + for (const item of res.data) { + map[item.endpointId] = item.circuitState as EndpointCircuitState; + } + return map; + }, + enabled: endpointIds.length > 0, + staleTime: 15_000, + }); + if (isLoading) { return
{t("keyLoading")}
; } @@ -160,6 +181,7 @@ export function ProviderEndpointsTable({ tTypes={tTypes} readOnly={readOnly} hideTypeColumn={hideTypeColumn} + circuitState={circuitInfoMap[endpoint.id] ?? null} /> ))} @@ -177,13 +199,16 @@ function EndpointRow({ tTypes, readOnly, hideTypeColumn, + circuitState, }: { endpoint: ProviderEndpoint; tTypes: ReturnType; readOnly: boolean; hideTypeColumn: boolean; + circuitState: EndpointCircuitState | null; }) { const t = useTranslations("settings.providers"); + const tStatus = useTranslations("settings.providers.endpointStatus"); const tCommon = useTranslations("settings.common"); const queryClient = useQueryClient(); const [isProbing, setIsProbing] = useState(false); @@ -255,6 +280,24 @@ function EndpointRow({ }, }); + const resetCircuitMutation = useMutation({ + mutationFn: async () => { + const res = await resetEndpointCircuit({ endpointId: endpoint.id }); + if (!res.ok) throw new Error(res.error); + return res; + }, + onSuccess: () => { + queryClient.invalidateQueries({ queryKey: ["endpoint-circuit-info"] }); + queryClient.invalidateQueries({ queryKey: ["provider-endpoints"] }); + toast.success(tStatus("resetCircuitSuccess")); + }, + onError: () => { + toast.error(tStatus("resetCircuitFailed")); + }, + }); + + const isCircuitTripped = circuitState === "open" || circuitState === "half-open"; + return ( {!hideTypeColumn && ( @@ -277,8 +320,17 @@ function EndpointRow({ {endpoint.url} -
- {endpoint.isEnabled ? ( +
+ {circuitState === "open" ? ( + {tStatus("circuitOpen")} + ) : circuitState === "half-open" ? ( + + {tStatus("circuitHalfOpen")} + + ) : endpoint.isEnabled ? ( + {isCircuitTripped && ( + resetCircuitMutation.mutate()} + disabled={resetCircuitMutation.isPending} + > + + {tStatus("resetCircuit")} + + )} { diff --git a/src/app/[locale]/settings/providers/_components/provider-list.tsx b/src/app/[locale]/settings/providers/_components/provider-list.tsx index 3b67ebf0a..b70899b7e 100644 --- a/src/app/[locale]/settings/providers/_components/provider-list.tsx +++ b/src/app/[locale]/settings/providers/_components/provider-list.tsx @@ -4,6 +4,7 @@ import { useTranslations } from "next-intl"; import type { CurrencyCode } from "@/lib/utils/currency"; import type { ProviderDisplay, ProviderStatisticsMap } from "@/types/provider"; import type { User } from "@/types/user"; +import type { EndpointCircuitInfoMap } from "./provider-manager"; import { ProviderRichListItem } from "./provider-rich-list-item"; interface ProviderListProps { @@ -19,6 +20,8 @@ interface ProviderListProps { recoveryMinutes: number | null; } >; + /** Endpoint-level circuit breaker info, keyed by provider ID */ + endpointCircuitInfo?: EndpointCircuitInfoMap; statistics?: ProviderStatisticsMap; statisticsLoading?: boolean; currencyCode?: CurrencyCode; @@ -36,6 +39,7 @@ export function ProviderList({ providers, currentUser, healthStatus, + endpointCircuitInfo = {}, statistics = {}, statisticsLoading = false, currencyCode = "USD", @@ -70,6 +74,7 @@ export function ProviderList({ provider={provider} currentUser={currentUser} healthStatus={healthStatus[provider.id]} + endpointCircuitInfo={endpointCircuitInfo[provider.id]} statistics={statistics[provider.id]} statisticsLoading={statisticsLoading} currencyCode={currencyCode} diff --git a/src/app/[locale]/settings/providers/_components/provider-manager.tsx b/src/app/[locale]/settings/providers/_components/provider-manager.tsx index fce2c8761..0b373c4db 100644 --- a/src/app/[locale]/settings/providers/_components/provider-manager.tsx +++ b/src/app/[locale]/settings/providers/_components/provider-manager.tsx @@ -30,6 +30,17 @@ import { ProviderSortDropdown, type SortKey } from "./provider-sort-dropdown"; import { ProviderTypeFilter } from "./provider-type-filter"; import { ProviderVendorView } from "./provider-vendor-view"; +/** Per-endpoint circuit breaker state, keyed by provider ID */ +export type EndpointCircuitInfoMap = Record< + number, + Array<{ + endpointId: number; + circuitState: "closed" | "open" | "half-open"; + failureCount: number; + circuitOpenUntil: number | null; + }> +>; + interface ProviderManagerProps { providers: ProviderDisplay[]; currentUser?: User; @@ -43,6 +54,8 @@ interface ProviderManagerProps { recoveryMinutes: number | null; } >; + /** Endpoint-level circuit breaker info, keyed by provider ID */ + endpointCircuitInfo?: EndpointCircuitInfoMap; statistics?: ProviderStatisticsMap; statisticsLoading?: boolean; currencyCode?: CurrencyCode; @@ -56,6 +69,7 @@ export function ProviderManager({ providers, currentUser, healthStatus, + endpointCircuitInfo = {}, statistics = {}, statisticsLoading = false, currencyCode = "USD", @@ -86,10 +100,27 @@ export function ProviderManager({ const [batchDialogOpen, setBatchDialogOpen] = useState(false); const [batchActionMode, setBatchActionMode] = useState(null); - // Count providers with circuit breaker open + // Helper: check if a provider has any circuit open (key-level or endpoint-level) + const hasAnyCircuitOpen = useCallback( + (providerId: number): boolean => { + // Key-level circuit open + if (healthStatus[providerId]?.circuitState === "open") { + return true; + } + // Endpoint-level circuit open + const endpoints = endpointCircuitInfo[providerId]; + if (Array.isArray(endpoints) && endpoints.some((ep) => ep.circuitState === "open")) { + return true; + } + return false; + }, + [healthStatus, endpointCircuitInfo] + ); + + // Count providers with circuit breaker open (key-level or endpoint-level, deduplicated) const circuitBrokenCount = useMemo(() => { - return providers.filter((p) => healthStatus[p.id]?.circuitState === "open").length; - }, [providers, healthStatus]); + return providers.filter((p) => hasAnyCircuitOpen(p.id)).length; + }, [providers, hasAnyCircuitOpen]); const activeFilterCount = useMemo(() => { let count = 0; @@ -190,9 +221,9 @@ export function ProviderManager({ }); } - // Filter by circuit breaker state + // Filter by circuit breaker state (key-level or endpoint-level) if (circuitBrokenFilter) { - result = result.filter((p) => healthStatus[p.id]?.circuitState === "open"); + result = result.filter((p) => hasAnyCircuitOpen(p.id)); } // 排序 @@ -232,7 +263,7 @@ export function ProviderManager({ statusFilter, groupFilter, circuitBrokenFilter, - healthStatus, + hasAnyCircuitOpen, ]); // Batch selection handlers @@ -572,6 +603,7 @@ export function ProviderManager({ providers={filteredProviders} currentUser={currentUser} healthStatus={healthStatus} + endpointCircuitInfo={endpointCircuitInfo} statistics={statistics} statisticsLoading={statisticsLoading} currencyCode={currencyCode} diff --git a/src/app/[locale]/settings/providers/_components/provider-rich-list-item.tsx b/src/app/[locale]/settings/providers/_components/provider-rich-list-item.tsx index 47359c90e..572e4b267 100644 --- a/src/app/[locale]/settings/providers/_components/provider-rich-list-item.tsx +++ b/src/app/[locale]/settings/providers/_components/provider-rich-list-item.tsx @@ -78,6 +78,13 @@ interface ProviderRichListItemProps { circuitOpenUntil: number | null; recoveryMinutes: number | null; }; + /** Endpoint-level circuit breaker info for this provider */ + endpointCircuitInfo?: Array<{ + endpointId: number; + circuitState: "closed" | "open" | "half-open"; + failureCount: number; + circuitOpenUntil: number | null; + }>; statistics?: ProviderStatistics; statisticsLoading?: boolean; currencyCode?: CurrencyCode; @@ -98,6 +105,7 @@ export function ProviderRichListItem({ provider, currentUser, healthStatus, + endpointCircuitInfo = [], statistics, statisticsLoading = false, currencyCode = "USD", @@ -500,10 +508,21 @@ export function ProviderRichListItem({ ) : ( {PROVIDER_GROUP.DEFAULT} )} + {/* Key-level circuit badge */} {healthStatus?.circuitState === "open" && ( - {tList("circuitBroken")} + {tList("keyCircuitBroken")} + + )} + {/* Endpoint-level circuit badge */} + {endpointCircuitInfo?.some((ep) => ep.circuitState === "open") && ( + + + {tList("endpointCircuitBroken")} )}
@@ -688,10 +707,21 @@ export function ProviderRichListItem({ {PROVIDER_GROUP.DEFAULT} )} + {/* Key-level circuit badge */} {healthStatus?.circuitState === "open" && ( - {tList("circuitBroken")} + {tList("keyCircuitBroken")} + + )} + {/* Endpoint-level circuit badge */} + {endpointCircuitInfo?.some((ep) => ep.circuitState === "open") && ( + + + {tList("endpointCircuitBroken")} )}
diff --git a/src/app/[locale]/settings/providers/_components/provider-vendor-view.tsx b/src/app/[locale]/settings/providers/_components/provider-vendor-view.tsx index a0d1e2160..a4566315b 100644 --- a/src/app/[locale]/settings/providers/_components/provider-vendor-view.tsx +++ b/src/app/[locale]/settings/providers/_components/provider-vendor-view.tsx @@ -26,6 +26,7 @@ import { getErrorMessage } from "@/lib/utils/error-messages"; import type { ProviderDisplay, ProviderVendor } from "@/types/provider"; import type { User } from "@/types/user"; import { ProviderEndpointsSection } from "./provider-endpoints-table"; + import { VendorKeysCompactList } from "./vendor-keys-compact-list"; interface ProviderVendorViewProps { diff --git a/src/app/v1/_lib/proxy/forwarder.ts b/src/app/v1/_lib/proxy/forwarder.ts index d7fd898b7..ad52c6b9d 100644 --- a/src/app/v1/_lib/proxy/forwarder.ts +++ b/src/app/v1/_lib/proxy/forwarder.ts @@ -18,7 +18,10 @@ import { PROVIDER_DEFAULTS, PROVIDER_LIMITS } from "@/lib/constants/provider.con import { recordEndpointFailure, recordEndpointSuccess } from "@/lib/endpoint-circuit-breaker"; import { applyGeminiGoogleSearchOverrideWithAudit } from "@/lib/gemini/provider-overrides"; import { logger } from "@/lib/logger"; -import { getPreferredProviderEndpoints } from "@/lib/provider-endpoints/endpoint-selector"; +import { + getEndpointFilterStats, + getPreferredProviderEndpoints, +} from "@/lib/provider-endpoints/endpoint-selector"; import { getGlobalAgentPool, getProxyAgentForProvider } from "@/lib/proxy-agent"; import { SessionManager } from "@/lib/session-manager"; import { CONTEXT_1M_BETA_HEADER, shouldApplyContext1m } from "@/lib/special-attributes"; @@ -28,6 +31,7 @@ import { } from "@/lib/vendor-type-circuit-breaker"; import { updateMessageRequestDetails } from "@/repository/message"; import type { CacheTtlPreference, CacheTtlResolved } from "@/types/cache"; +import type { ProviderChainItem } from "@/types/message"; import type { ClaudeMetadataUserIdInjectionSpecialSetting } from "@/types/special-settings"; import { GeminiAuth } from "../gemini/auth"; @@ -475,6 +479,10 @@ export class ProxyForwarder { if (endpointCandidates.length === 0) { if (shouldEnforceStrictEndpointPool) { + const strictBlockCause = endpointSelectionError + ? "selector_error" + : "no_endpoint_candidates"; + logger.warn( "ProxyForwarder: Strict endpoint policy blocked legacy provider.url fallback", { @@ -483,12 +491,42 @@ export class ProxyForwarder { providerType: currentProvider.providerType, requestPath, reason: "strict_blocked_legacy_fallback", - strictBlockCause: endpointSelectionError - ? "selector_error" - : "no_endpoint_candidates", + strictBlockCause, selectorError: endpointSelectionError?.message, } ); + + // Record endpoint pool exhaustion in provider chain for audit trail + const exhaustionContext: Record = { strictBlockCause }; + if (endpointSelectionError) { + exhaustionContext.selectorError = endpointSelectionError.message; + } + + // Collect endpoint filter stats for no_endpoint_candidates (selector_error has no data) + let filterStats: ProviderChainItem["endpointFilterStats"]; + if (!endpointSelectionError) { + try { + const stats = await getEndpointFilterStats({ + vendorId: providerVendorId, + providerType: currentProvider.providerType, + }); + filterStats = stats; + } catch (statsError) { + logger.warn("[ProxyForwarder] Failed to collect endpoint filter stats", { + providerId: currentProvider.id, + vendorId: providerVendorId, + error: statsError instanceof Error ? statsError.message : String(statsError), + }); + } + } + + session.addProviderToChain(currentProvider, { + reason: "endpoint_pool_exhausted", + strictBlockCause: strictBlockCause as ProviderChainItem["strictBlockCause"], + ...(filterStats ? { endpointFilterStats: filterStats } : {}), + errorMessage: endpointSelectionError?.message, + }); + failedProviderIds.push(currentProvider.id); attemptCount = maxAttemptsPerProvider; } else { diff --git a/src/app/v1/_lib/proxy/session.ts b/src/app/v1/_lib/proxy/session.ts index 000ffc364..5e93b76f3 100644 --- a/src/app/v1/_lib/proxy/session.ts +++ b/src/app/v1/_lib/proxy/session.ts @@ -459,7 +459,8 @@ export class ProxySession { | "retry_with_official_instructions" // Codex instructions 自动重试（官方） | "retry_with_cached_instructions" // Codex instructions 智能重试（缓存） | "client_error_non_retryable" // 不可重试的客户端错误（Prompt 超限、内容过滤、PDF 限制、Thinking 格式） - | "http2_fallback"; // HTTP/2 协议错误，回退到 HTTP/1.1（不切换供应商、不计入熔断器） + | "http2_fallback" // HTTP/2 协议错误，回退到 HTTP/1.1（不切换供应商、不计入熔断器） + | "endpoint_pool_exhausted"; // 端点池耗尽（strict endpoint policy 阻止了 fallback） selectionMethod?: | "session_reuse" | "weighted_random" @@ -476,6 +477,8 @@ export class ProxySession { circuitFailureThreshold?: number; // 熔断阈值 errorDetails?: ProviderChainItem["errorDetails"]; // 结构化错误详情 decisionContext?: ProviderChainItem["decisionContext"]; + strictBlockCause?: ProviderChainItem["strictBlockCause"]; // endpoint pool exhaustion cause + endpointFilterStats?: ProviderChainItem["endpointFilterStats"]; // endpoint filter statistics } ): void { const item: ProviderChainItem = { @@ -502,6 +505,8 @@ export class ProxySession { circuitFailureThreshold: metadata?.circuitFailureThreshold, errorDetails: metadata?.errorDetails, // 结构化错误详情 decisionContext: metadata?.decisionContext, + strictBlockCause: metadata?.strictBlockCause, + endpointFilterStats: metadata?.endpointFilterStats, }; // 避免重复添加同一个供应商（除非是重试，即有 attemptNumber） diff --git a/src/lib/endpoint-circuit-breaker.ts b/src/lib/endpoint-circuit-breaker.ts index 06ce237d3..161dbca0d 100644 --- a/src/lib/endpoint-circuit-breaker.ts +++ b/src/lib/endpoint-circuit-breaker.ts @@ -146,12 +146,28 @@ export async function recordEndpointFailure(endpointId: number, error: Error): P health.circuitOpenUntil = Date.now() + config.openDuration; health.halfOpenSuccessCount = 0; + const retryAt = new Date(health.circuitOpenUntil).toISOString(); + logger.warn("[EndpointCircuitBreaker] Endpoint circuit opened", { endpointId, failureCount: health.failureCount, threshold: config.failureThreshold, errorMessage: error.message, }); + + // Async alert (non-blocking) + triggerEndpointCircuitBreakerAlert( + endpointId, + health.failureCount, + retryAt, + error.message + ).catch((err) => { + logger.error({ + action: "trigger_endpoint_circuit_breaker_alert_error", + endpointId, + error: err instanceof Error ? err.message : String(err), + }); + }); } persistStateToRedis(endpointId, health); @@ -194,3 +210,65 @@ export async function resetEndpointCircuit(endpointId: number): Promise { await deleteEndpointCircuitState(endpointId); } + +/** + * Alert data for endpoint circuit breaker events. + */ +export interface EndpointCircuitAlertData { + endpointId: number; + failureCount: number; + retryAt: string; + lastError: string; + endpointUrl?: string; +} + +/** + * Trigger circuit breaker alert for an endpoint. + * Looks up endpoint info to enrich the alert data, then delegates to sendCircuitBreakerAlert. + */ +export async function triggerEndpointCircuitBreakerAlert( + endpointId: number, + failureCount: number, + retryAt: string, + lastError: string +): Promise { + try { + const { sendCircuitBreakerAlert } = await import("@/lib/notification/notifier"); + + // Try to enrich with endpoint URL and vendor info from database + let endpointUrl: string | undefined; + let vendorId = 0; + let endpointLabel = ""; + try { + const { findProviderEndpointById } = await import("@/repository"); + const endpoint = await findProviderEndpointById(endpointId); + if (endpoint) { + endpointUrl = endpoint.url; + vendorId = endpoint.vendorId; + endpointLabel = endpoint.label || ""; + } + } catch (lookupError) { + logger.warn("[EndpointCircuitBreaker] Failed to enrich alert with endpoint info", { + endpointId, + error: lookupError instanceof Error ? lookupError.message : String(lookupError), + }); + } + + await sendCircuitBreakerAlert({ + providerId: vendorId, + providerName: endpointLabel || `endpoint:${endpointId}`, + failureCount, + retryAt, + lastError, + incidentSource: "endpoint", + endpointId, + endpointUrl, + }); + } catch (error) { + logger.error({ + action: "endpoint_circuit_breaker_alert_error", + endpointId, + error: error instanceof Error ? error.message : String(error), + }); + } +} diff --git a/src/lib/notification/notifier.ts b/src/lib/notification/notifier.ts index e1ca4e56c..06c7009d6 100644 --- a/src/lib/notification/notifier.ts +++ b/src/lib/notification/notifier.ts @@ -24,14 +24,18 @@ export async function sendCircuitBreakerAlert(data: CircuitBreakerAlertData): Pr // 防止 5 分钟内重复告警 const redisClient = getRedisClient(); + const source = data.incidentSource ?? "provider"; + const dedupSuffix = + source === "endpoint" && data.endpointId != null ? `endpoint:${data.endpointId}` : source; if (redisClient) { - const cacheKey = `circuit-breaker-alert:${data.providerId}`; + const cacheKey = `circuit-breaker-alert:${data.providerId}:${dedupSuffix}`; const cached = await redisClient.get(cacheKey); if (cached) { logger.info({ action: "circuit_breaker_alert_suppressed", providerId: data.providerId, + incidentSource: source, reason: "duplicate_within_5min", }); return; @@ -112,11 +116,13 @@ export async function sendCircuitBreakerAlert(data: CircuitBreakerAlertData): Pr logger.info({ action: "circuit_breaker_alert_sent", providerId: data.providerId, + incidentSource: source, }); } catch (error) { logger.error({ action: "send_circuit_breaker_alert_error", providerId: data.providerId, + incidentSource: data.incidentSource ?? "provider", error: error instanceof Error ? error.message : String(error), }); } diff --git a/src/lib/provider-endpoints/endpoint-selector.ts b/src/lib/provider-endpoints/endpoint-selector.ts index 246fadfcb..cda4ddd3a 100644 --- a/src/lib/provider-endpoints/endpoint-selector.ts +++ b/src/lib/provider-endpoints/endpoint-selector.ts @@ -53,6 +53,38 @@ export async function getPreferredProviderEndpoints(input: { return rankProviderEndpoints(candidates); } +export interface EndpointFilterStats { + total: number; + enabled: number; + circuitOpen: number; + available: number; +} + +/** + * Collect endpoint filter statistics for a given vendor/type. + * + * Used for audit trail when all endpoints are exhausted (strict block). + * Returns null only when the raw endpoint query itself fails. + */ +export async function getEndpointFilterStats(input: { + vendorId: number; + providerType: ProviderType; +}): Promise { + const endpoints = await findProviderEndpointsByVendorAndType(input.vendorId, input.providerType); + const total = endpoints.length; + const enabled = endpoints.filter((e) => e.isEnabled && !e.deletedAt).length; + + const circuitResults = await Promise.all( + endpoints + .filter((e) => e.isEnabled && !e.deletedAt) + .map(async (e) => isEndpointCircuitOpen(e.id)) + ); + const circuitOpen = circuitResults.filter(Boolean).length; + const available = enabled - circuitOpen; + + return { total, enabled, circuitOpen, available }; +} + export async function pickBestProviderEndpoint(input: { vendorId: number; providerType: ProviderType; diff --git a/src/lib/utils/provider-chain-formatter.test.ts b/src/lib/utils/provider-chain-formatter.test.ts index 99310e5d1..8caf5ed97 100644 --- a/src/lib/utils/provider-chain-formatter.test.ts +++ b/src/lib/utils/provider-chain-formatter.test.ts @@ -1,5 +1,27 @@ import { describe, expect, test } from "vitest"; -import { formatProbability, formatProbabilityCompact } from "./provider-chain-formatter"; +import type { ProviderChainItem } from "@/types/message"; +import { + formatProbability, + formatProbabilityCompact, + formatProviderDescription, + formatProviderSummary, + formatProviderTimeline, +} from "./provider-chain-formatter"; + +/** + * Simple mock t() function: returns the key followed by interpolated values. + * Format: "key" or "key [k1=v1, k2=v2]" when values are provided. + * This is enough to verify the formatter calls t() with the right keys and values. + */ +function mockT(key: string, values?: Record): string { + if (!values || Object.keys(values).length === 0) { + return key; + } + const pairs = Object.entries(values) + .map(([k, v]) => `${k}=${v}`) + .join(", "); + return `${key} [${pairs}]`; +} describe("formatProbability", () => { test("formats 0.5 as 50.0%", () => { @@ -62,3 +84,230 @@ describe("formatProbabilityCompact", () => { expect(formatProbabilityCompact(Number.NaN)).toBeNull(); }); }); + +// ============================================================================= +// endpoint_pool_exhausted reason tests +// ============================================================================= + +describe("endpoint_pool_exhausted", () => { + // --------------------------------------------------------------------------- + // Shared fixtures + // --------------------------------------------------------------------------- + const baseExhaustedItem: ProviderChainItem = { + id: 1, + name: "provider-a", + reason: "endpoint_pool_exhausted", + timestamp: 1000, + endpointFilterStats: { + total: 5, + enabled: 4, + circuitOpen: 3, + available: 0, + }, + strictBlockCause: "no_endpoint_candidates", + }; + + const exhaustedWithSelectorError: ProviderChainItem = { + id: 1, + name: "provider-a", + reason: "endpoint_pool_exhausted", + timestamp: 1000, + endpointFilterStats: { + total: 3, + enabled: 2, + circuitOpen: 1, + available: 0, + }, + strictBlockCause: "selector_error", + errorMessage: "endpoint selector threw an unexpected error", + }; + + const exhaustedNoStats: ProviderChainItem = { + id: 1, + name: "provider-a", + reason: "endpoint_pool_exhausted", + timestamp: 1000, + }; + + // --------------------------------------------------------------------------- + // getProviderStatus / isActualRequest (tested implicitly through formatters) + // --------------------------------------------------------------------------- + + describe("formatProviderSummary", () => { + test("renders exhausted item in chain path with failure mark", () => { + const chain: ProviderChainItem[] = [baseExhaustedItem]; + const result = formatProviderSummary(chain, mockT); + + // Should show a failure status marker for this provider + expect(result).toContain("provider-a"); + expect(result).toContain("✗"); + }); + + test("renders exhausted alongside successful retry in multi-provider chain", () => { + const chain: ProviderChainItem[] = [ + baseExhaustedItem, + { + id: 2, + name: "provider-b", + reason: "request_success", + statusCode: 200, + timestamp: 2000, + }, + ]; + const result = formatProviderSummary(chain, mockT); + + expect(result).toContain("provider-a"); + expect(result).toContain("provider-b"); + // provider-a should be failure, provider-b should be success + expect(result).toMatch(/provider-a\(.*\).*provider-b\(.*\)/); + }); + }); + + describe("formatProviderDescription", () => { + test("shows endpoint pool exhausted label in request chain", () => { + const chain: ProviderChainItem[] = [baseExhaustedItem]; + const result = formatProviderDescription(chain, mockT); + + expect(result).toContain("provider-a"); + expect(result).toContain("description.endpointPoolExhausted"); + }); + + test("handles exhausted item when it is the only item (no initial_selection)", () => { + const chain: ProviderChainItem[] = [baseExhaustedItem]; + const result = formatProviderDescription(chain, mockT); + + // Should not crash, should produce something reasonable + expect(result.length).toBeGreaterThan(0); + }); + }); + + describe("formatProviderTimeline", () => { + test("renders endpoint pool exhausted with filter stats", () => { + const chain: ProviderChainItem[] = [ + { + id: 1, + name: "provider-a", + reason: "initial_selection", + timestamp: 0, + decisionContext: { + totalProviders: 3, + enabledProviders: 3, + targetType: "claude", + groupFilterApplied: false, + beforeHealthCheck: 3, + afterHealthCheck: 3, + priorityLevels: [1], + selectedPriority: 1, + candidatesAtPriority: [], + }, + }, + baseExhaustedItem, + ]; + + const { timeline } = formatProviderTimeline(chain, mockT); + + // Should contain the title for endpoint pool exhausted + expect(timeline).toContain("timeline.endpointPoolExhausted"); + + // Should contain filter stats breakdown with values + expect(timeline).toContain("timeline.endpointStatsTotal [count=5]"); + expect(timeline).toContain("timeline.endpointStatsEnabled [count=4]"); + expect(timeline).toContain("timeline.endpointStatsCircuitOpen [count=3]"); + expect(timeline).toContain("timeline.endpointStatsAvailable [count=0]"); + }); + + test("renders strictBlockCause = no_endpoint_candidates", () => { + const chain: ProviderChainItem[] = [baseExhaustedItem]; + const { timeline } = formatProviderTimeline(chain, mockT); + + expect(timeline).toContain("timeline.strictBlockNoEndpoints"); + }); + + test("renders strictBlockCause = selector_error with error message", () => { + const chain: ProviderChainItem[] = [exhaustedWithSelectorError]; + const { timeline } = formatProviderTimeline(chain, mockT); + + expect(timeline).toContain("timeline.strictBlockSelectorError"); + // The error message is passed through t("timeline.error", { error: ... }) + expect(timeline).toContain("error=endpoint selector threw an unexpected error"); + }); + + test("degrades gracefully when endpointFilterStats is missing", () => { + const chain: ProviderChainItem[] = [exhaustedNoStats]; + const { timeline } = formatProviderTimeline(chain, mockT); + + // Should still render without crashing + expect(timeline).toContain("timeline.endpointPoolExhausted"); + // Provider name is embedded in the mockT output as a value + expect(timeline).toContain("provider=provider-a"); + // Should NOT contain stats section + expect(timeline).not.toContain("timeline.endpointStatsTotal"); + }); + + test("computes totalDuration correctly with exhausted items", () => { + const chain: ProviderChainItem[] = [ + { + id: 1, + name: "provider-a", + reason: "initial_selection", + timestamp: 0, + decisionContext: { + totalProviders: 1, + enabledProviders: 1, + targetType: "claude", + groupFilterApplied: false, + beforeHealthCheck: 1, + afterHealthCheck: 1, + priorityLevels: [1], + selectedPriority: 1, + candidatesAtPriority: [], + }, + }, + { ...baseExhaustedItem, timestamp: 500 }, + ]; + const { totalDuration } = formatProviderTimeline(chain, mockT); + expect(totalDuration).toBe(500); + }); + }); +}); + +// ============================================================================= +// Unknown reason graceful degradation +// ============================================================================= + +describe("unknown reason graceful degradation", () => { + const unknownItem: ProviderChainItem = { + id: 99, + name: "provider-x", + // @ts-expect-error -- intentionally testing an unknown reason string + reason: "some_future_reason_not_yet_defined", + timestamp: 1000, + }; + + test("formatProviderSummary does not throw for unknown reason", () => { + expect(() => formatProviderSummary([unknownItem], mockT)).not.toThrow(); + }); + + test("formatProviderDescription does not throw for unknown reason", () => { + expect(() => formatProviderDescription([unknownItem], mockT)).not.toThrow(); + }); + + test("formatProviderTimeline does not throw for unknown reason", () => { + expect(() => formatProviderTimeline([unknownItem], mockT)).not.toThrow(); + const { timeline } = formatProviderTimeline([unknownItem], mockT); + // Should include the provider name and the raw reason as fallback + expect(timeline).toContain("provider-x"); + expect(timeline).toContain("some_future_reason_not_yet_defined"); + }); + + test("formatProviderTimeline renders unknown reason with no reason field", () => { + const noReasonItem: ProviderChainItem = { + id: 99, + name: "provider-y", + timestamp: 1000, + }; + const { timeline } = formatProviderTimeline([noReasonItem], mockT); + expect(timeline).toContain("provider-y"); + expect(timeline).toContain("timeline.unknown"); + }); +}); diff --git a/src/lib/utils/provider-chain-formatter.ts b/src/lib/utils/provider-chain-formatter.ts index 0c9257ee6..5369bf1b0 100644 --- a/src/lib/utils/provider-chain-formatter.ts +++ b/src/lib/utils/provider-chain-formatter.ts @@ -63,7 +63,8 @@ function getProviderStatus(item: ProviderChainItem): "✓" | "✗" | "⚡" | " if ( item.reason === "retry_failed" || item.reason === "system_error" || - item.reason === "client_error_non_retryable" + item.reason === "client_error_non_retryable" || + item.reason === "endpoint_pool_exhausted" ) { return "✗"; } @@ -90,7 +91,8 @@ function isActualRequest(item: ProviderChainItem): boolean { if ( item.reason === "retry_failed" || item.reason === "system_error" || - item.reason === "client_error_non_retryable" + item.reason === "client_error_non_retryable" || + item.reason === "endpoint_pool_exhausted" ) { return true; } @@ -309,6 +311,8 @@ export function formatProviderDescription( desc += ` ${t("description.http2Fallback")}`; } else if (item.reason === "client_error_non_retryable") { desc += ` ${t("description.clientError")}`; + } else if (item.reason === "endpoint_pool_exhausted") { + desc += ` ${t("description.endpointPoolExhausted")}`; } desc += "\n"; @@ -710,6 +714,34 @@ export function formatProviderTimeline( continue; } + // === 端点池耗尽 === + if (item.reason === "endpoint_pool_exhausted") { + timeline += `${t("timeline.endpointPoolExhausted")}\n\n`; + timeline += `${t("timeline.provider", { provider: item.name })}\n`; + + // 端点过滤统计 + if (item.endpointFilterStats && typeof item.endpointFilterStats.total === "number") { + const stats = item.endpointFilterStats; + timeline += `\n${t("timeline.endpointStats")}:\n`; + timeline += `${t("timeline.endpointStatsTotal", { count: stats.total })}\n`; + timeline += `${t("timeline.endpointStatsEnabled", { count: stats.enabled })}\n`; + timeline += `${t("timeline.endpointStatsCircuitOpen", { count: stats.circuitOpen })}\n`; + timeline += `${t("timeline.endpointStatsAvailable", { count: stats.available })}\n`; + } + + // 严格模式阻止原因 + if (item.strictBlockCause === "no_endpoint_candidates") { + timeline += `\n${t("timeline.strictBlockNoEndpoints")}`; + } else if (item.strictBlockCause === "selector_error") { + timeline += `\n${t("timeline.strictBlockSelectorError")}`; + if (item.errorMessage) { + timeline += `\n${t("timeline.error", { error: item.errorMessage })}`; + } + } + + continue; + } + // 并发限制失败 if (item.reason === "concurrent_limit_failed") { timeline += `${t("timeline.attemptFailed", { attempt: actualAttemptNumber ?? 0 })}\n\n`; diff --git a/src/lib/webhook/templates/circuit-breaker.ts b/src/lib/webhook/templates/circuit-breaker.ts index 7238d0ea6..2fe92f67f 100644 --- a/src/lib/webhook/templates/circuit-breaker.ts +++ b/src/lib/webhook/templates/circuit-breaker.ts @@ -5,6 +5,8 @@ export function buildCircuitBreakerMessage( data: CircuitBreakerAlertData, timezone?: string ): StructuredMessage { + const isEndpoint = data.incidentSource === "endpoint"; + const fields = [ { label: "失败次数", value: `${data.failureCount} 次` }, { label: "预计恢复", value: formatDateTime(data.retryAt, timezone || "UTC") }, @@ -14,9 +16,24 @@ export function buildCircuitBreakerMessage( fields.push({ label: "最后错误", value: data.lastError }); } + // Add endpoint-specific fields + if (isEndpoint) { + if (data.endpointId !== undefined) { + fields.push({ label: "端点ID", value: String(data.endpointId) }); + } + if (data.endpointUrl) { + fields.push({ label: "端点地址", value: data.endpointUrl }); + } + } + + const title = isEndpoint ? "端点熔断告警" : "供应商熔断告警"; + const description = isEndpoint + ? `供应商 ${data.providerName} 的端点 (ID: ${data.endpointId ?? "N/A"}) 已触发熔断保护` + : `供应商 ${data.providerName} (ID: ${data.providerId}) 已触发熔断保护`; + return { header: { - title: "供应商熔断告警", + title, icon: "🔌", level: "error", }, @@ -25,7 +42,7 @@ export function buildCircuitBreakerMessage( content: [ { type: "quote", - value: `供应商 ${data.providerName} (ID: ${data.providerId}) 已触发熔断保护`, + value: description, }, ], }, diff --git a/src/lib/webhook/templates/placeholders.ts b/src/lib/webhook/templates/placeholders.ts index 7bfd405e2..a380596c6 100644 --- a/src/lib/webhook/templates/placeholders.ts +++ b/src/lib/webhook/templates/placeholders.ts @@ -38,6 +38,13 @@ export const TEMPLATE_PLACEHOLDERS = { { key: "{{failure_count}}", label: "失败次数", description: "连续失败计数" }, { key: "{{retry_at}}", label: "恢复时间", description: "预计恢复时间" }, { key: "{{last_error}}", label: "错误信息", description: "最后一次错误详情" }, + { + key: "{{incident_source}}", + label: "熔断来源", + description: "provider(Key 熔断) 或 endpoint(Endpoint 熔断)", + }, + { key: "{{endpoint_id}}", label: "端点ID", description: "触发熔断的端点 ID" }, + { key: "{{endpoint_url}}", label: "端点地址", description: "触发熔断的端点 URL" }, ], daily_leaderboard: [ { key: "{{date}}", label: "统计日期", description: "YYYY-MM-DD 格式" }, @@ -91,6 +98,9 @@ export function buildTemplateVariables(params: { values["{{failure_count}}"] = cb?.failureCount !== undefined ? String(cb.failureCount) : ""; values["{{retry_at}}"] = cb?.retryAt ?? ""; values["{{last_error}}"] = cb?.lastError ?? ""; + values["{{incident_source}}"] = cb?.incidentSource ?? "provider"; + values["{{endpoint_id}}"] = cb?.endpointId !== undefined ? String(cb.endpointId) : ""; + values["{{endpoint_url}}"] = cb?.endpointUrl ?? ""; } if (notificationType === "daily_leaderboard") { diff --git a/src/lib/webhook/types.ts b/src/lib/webhook/types.ts index 019f1993e..ea4e24809 100644 --- a/src/lib/webhook/types.ts +++ b/src/lib/webhook/types.ts @@ -45,6 +45,12 @@ export interface CircuitBreakerAlertData { failureCount: number; retryAt: string; lastError?: string; + /** Incident source: 'provider' for key circuit, 'endpoint' for endpoint circuit */ + incidentSource?: "provider" | "endpoint"; + /** Endpoint ID when incidentSource is 'endpoint' */ + endpointId?: number; + /** Endpoint URL when incidentSource is 'endpoint' */ + endpointUrl?: string; } export interface DailyLeaderboardEntry { diff --git a/src/types/message.ts b/src/types/message.ts index e5b20b62b..ee3784ed8 100644 --- a/src/types/message.ts +++ b/src/types/message.ts @@ -32,7 +32,8 @@ export interface ProviderChainItem { | "retry_with_official_instructions" // Codex instructions 自动重试（官方） | "retry_with_cached_instructions" // Codex instructions 智能重试（缓存） | "client_error_non_retryable" // 不可重试的客户端错误（Prompt 超限、内容过滤、PDF 限制、Thinking 格式） - | "http2_fallback"; // HTTP/2 协议错误，回退到 HTTP/1.1（不切换供应商、不计入熔断器） + | "http2_fallback" // HTTP/2 协议错误，回退到 HTTP/1.1（不切换供应商、不计入熔断器） + | "endpoint_pool_exhausted"; // 端点池耗尽（所有端点熔断或不可用，严格模式阻止降级） // === 选择方法（细化） === selectionMethod?: @@ -54,6 +55,15 @@ export interface ProviderChainItem { circuitFailureCount?: number; // 失败计数（包含本次失败） circuitFailureThreshold?: number; // 熔断阈值 + // 端点池耗尽详情（endpoint_pool_exhausted 专用） + strictBlockCause?: "selector_error" | "no_endpoint_candidates"; // 严格模式阻止降级的原因 + endpointFilterStats?: { + total: number; // 总端点数 + enabled: number; // 启用的端点数 + circuitOpen: number; // 熔断的端点数 + available: number; // 可用的端点数（通过熔断检查后） + }; + // 时间戳和尝试信息 timestamp?: number; attemptNumber?: number; // 第几次尝试（用于标识重试） diff --git a/tests/unit/actions/provider-endpoints.test.ts b/tests/unit/actions/provider-endpoints.test.ts index 6ec569be2..22f8c7369 100644 --- a/tests/unit/actions/provider-endpoints.test.ts +++ b/tests/unit/actions/provider-endpoints.test.ts @@ -259,4 +259,97 @@ describe("provider-endpoints actions", () => { expect(res.ok).toBe(true); expect(tryDeleteProviderVendorIfEmptyMock).toHaveBeenCalledWith(123); }); + + describe("batchGetEndpointCircuitInfo", () => { + it("returns circuit info for multiple endpoints", async () => { + getSessionMock.mockResolvedValue({ user: { role: "admin" } }); + + const { getEndpointHealthInfo } = await import("@/lib/endpoint-circuit-breaker"); + vi.mocked(getEndpointHealthInfo) + .mockResolvedValueOnce({ + health: { + failureCount: 0, + lastFailureTime: null, + circuitState: "closed" as const, + circuitOpenUntil: null, + halfOpenSuccessCount: 0, + }, + config: { failureThreshold: 3, openDuration: 300000, halfOpenSuccessThreshold: 1 }, + }) + .mockResolvedValueOnce({ + health: { + failureCount: 5, + lastFailureTime: Date.now(), + circuitState: "open" as const, + circuitOpenUntil: Date.now() + 60000, + halfOpenSuccessCount: 0, + }, + config: { failureThreshold: 3, openDuration: 300000, halfOpenSuccessThreshold: 1 }, + }) + .mockResolvedValueOnce({ + health: { + failureCount: 1, + lastFailureTime: Date.now() - 1000, + circuitState: "half-open" as const, + circuitOpenUntil: null, + halfOpenSuccessCount: 0, + }, + config: { failureThreshold: 3, openDuration: 300000, halfOpenSuccessThreshold: 1 }, + }); + + const { batchGetEndpointCircuitInfo } = await import("@/actions/provider-endpoints"); + const res = await batchGetEndpointCircuitInfo({ endpointIds: [1, 2, 3] }); + + expect(res.ok).toBe(true); + expect(res.data).toHaveLength(3); + expect(res.data?.[0]).toEqual({ + endpointId: 1, + circuitState: "closed", + failureCount: 0, + circuitOpenUntil: null, + }); + expect(res.data?.[1]).toEqual({ + endpointId: 2, + circuitState: "open", + failureCount: 5, + circuitOpenUntil: expect.any(Number), + }); + expect(res.data?.[2]).toEqual({ + endpointId: 3, + circuitState: "half-open", + failureCount: 1, + circuitOpenUntil: null, + }); + }); + + it("returns empty array for empty input", async () => { + getSessionMock.mockResolvedValue({ user: { role: "admin" } }); + + const { batchGetEndpointCircuitInfo } = await import("@/actions/provider-endpoints"); + const res = await batchGetEndpointCircuitInfo({ endpointIds: [] }); + + expect(res.ok).toBe(true); + expect(res.data).toEqual([]); + }); + + it("requires admin session", async () => { + getSessionMock.mockResolvedValue({ user: { role: "user" } }); + + const { batchGetEndpointCircuitInfo } = await import("@/actions/provider-endpoints"); + const res = await batchGetEndpointCircuitInfo({ endpointIds: [1, 2] }); + + expect(res.ok).toBe(false); + expect(res.errorCode).toBe("PERMISSION_DENIED"); + }); + + it("validates endpointIds are positive integers", async () => { + getSessionMock.mockResolvedValue({ user: { role: "admin" } }); + + const { batchGetEndpointCircuitInfo } = await import("@/actions/provider-endpoints"); + const res = await batchGetEndpointCircuitInfo({ endpointIds: [0, -1, 1] }); + + expect(res.ok).toBe(false); + expect(res.errorCode).toBe("MIN_VALUE"); + }); + }); }); diff --git a/tests/unit/lib/endpoint-circuit-breaker.test.ts b/tests/unit/lib/endpoint-circuit-breaker.test.ts index f45a31771..90b938f9f 100644 --- a/tests/unit/lib/endpoint-circuit-breaker.test.ts +++ b/tests/unit/lib/endpoint-circuit-breaker.test.ts @@ -121,4 +121,83 @@ describe("endpoint-circuit-breaker", () => { ]?.[1] as SavedEndpointCircuitState; expect(lastState.failureCount).toBe(0); }); + + test("triggerEndpointCircuitBreakerAlert should call sendCircuitBreakerAlert", async () => { + vi.resetModules(); + + const sendAlertMock = vi.fn(async () => {}); + vi.doMock("@/lib/logger", () => ({ logger: createLoggerMock() })); + vi.doMock("@/lib/notification/notifier", () => ({ + sendCircuitBreakerAlert: sendAlertMock, + })); + vi.doMock("@/repository", () => ({ + findProviderEndpointById: vi.fn(async () => null), + })); + + const { triggerEndpointCircuitBreakerAlert } = await import("@/lib/endpoint-circuit-breaker"); + + await triggerEndpointCircuitBreakerAlert( + 5, + 3, + "2026-01-01T00:05:00.000Z", + "connection refused" + ); + + expect(sendAlertMock).toHaveBeenCalledTimes(1); + expect(sendAlertMock).toHaveBeenCalledWith({ + providerId: 0, + providerName: "endpoint:5", + failureCount: 3, + retryAt: "2026-01-01T00:05:00.000Z", + lastError: "connection refused", + incidentSource: "endpoint", + endpointId: 5, + endpointUrl: undefined, + }); + }); + + test("triggerEndpointCircuitBreakerAlert should include endpointUrl when available", async () => { + vi.resetModules(); + + const sendAlertMock = vi.fn(async () => {}); + vi.doMock("@/lib/notification/notifier", () => ({ + sendCircuitBreakerAlert: sendAlertMock, + })); + vi.doMock("@/repository", () => ({ + findProviderEndpointById: vi.fn(async () => ({ + id: 10, + url: "https://custom.example.com/v1/chat/completions", + vendorId: 1, + providerType: "openai", + label: "Custom Endpoint", + sortOrder: 0, + isEnabled: true, + lastProbedAt: null, + lastProbeOk: null, + lastProbeStatusCode: null, + lastProbeLatencyMs: null, + lastProbeErrorType: null, + lastProbeErrorMessage: null, + createdAt: new Date(), + updatedAt: new Date(), + deletedAt: null, + })), + })); + + const { triggerEndpointCircuitBreakerAlert } = await import("@/lib/endpoint-circuit-breaker"); + + await triggerEndpointCircuitBreakerAlert(10, 3, "2026-01-01T00:05:00.000Z", "timeout"); + + expect(sendAlertMock).toHaveBeenCalledTimes(1); + expect(sendAlertMock).toHaveBeenCalledWith({ + providerId: 1, + providerName: "Custom Endpoint", + failureCount: 3, + retryAt: "2026-01-01T00:05:00.000Z", + lastError: "timeout", + incidentSource: "endpoint", + endpointId: 10, + endpointUrl: "https://custom.example.com/v1/chat/completions", + }); + }); }); diff --git a/tests/unit/lib/provider-endpoints/endpoint-selector.test.ts b/tests/unit/lib/provider-endpoints/endpoint-selector.test.ts index 49f279fcf..8aa1291c0 100644 --- a/tests/unit/lib/provider-endpoints/endpoint-selector.test.ts +++ b/tests/unit/lib/provider-endpoints/endpoint-selector.test.ts @@ -154,3 +154,93 @@ describe("provider-endpoints: endpoint-selector", () => { expect(isOpenMock).not.toHaveBeenCalled(); }); }); + +describe("getEndpointFilterStats", () => { + test("should correctly count total, enabled, circuitOpen, and available endpoints", async () => { + vi.resetModules(); + + const endpoints: ProviderEndpoint[] = [ + makeEndpoint({ id: 1, isEnabled: true, lastProbeOk: true }), + makeEndpoint({ id: 2, isEnabled: true, lastProbeOk: true }), + makeEndpoint({ id: 3, isEnabled: true, lastProbeOk: false }), + makeEndpoint({ id: 4, isEnabled: false }), + makeEndpoint({ id: 5, deletedAt: new Date(1) }), + ]; + + const findMock = vi.fn(async () => endpoints); + // id=2 is circuit open + const isOpenMock = vi.fn(async (endpointId: number) => endpointId === 2); + + vi.doMock("@/repository", () => ({ + findProviderEndpointsByVendorAndType: findMock, + })); + vi.doMock("@/lib/endpoint-circuit-breaker", () => ({ + isEndpointCircuitOpen: isOpenMock, + })); + + const { getEndpointFilterStats } = await import("@/lib/provider-endpoints/endpoint-selector"); + const stats = await getEndpointFilterStats({ vendorId: 10, providerType: "claude" }); + + expect(findMock).toHaveBeenCalledWith(10, "claude"); + expect(stats).toEqual({ + total: 5, // all endpoints + enabled: 3, // id=1,2,3 (isEnabled && !deletedAt) + circuitOpen: 1, // id=2 + available: 2, // enabled - circuitOpen = 3 - 1 + }); + }); + + test("should return all zeros when no endpoints exist", async () => { + vi.resetModules(); + + const findMock = vi.fn(async () => []); + const isOpenMock = vi.fn(async () => false); + + vi.doMock("@/repository", () => ({ + findProviderEndpointsByVendorAndType: findMock, + })); + vi.doMock("@/lib/endpoint-circuit-breaker", () => ({ + isEndpointCircuitOpen: isOpenMock, + })); + + const { getEndpointFilterStats } = await import("@/lib/provider-endpoints/endpoint-selector"); + const stats = await getEndpointFilterStats({ vendorId: 99, providerType: "codex" }); + + expect(stats).toEqual({ + total: 0, + enabled: 0, + circuitOpen: 0, + available: 0, + }); + expect(isOpenMock).not.toHaveBeenCalled(); + }); + + test("should count all enabled endpoints as circuitOpen when all are open", async () => { + vi.resetModules(); + + const endpoints: ProviderEndpoint[] = [ + makeEndpoint({ id: 1, isEnabled: true }), + makeEndpoint({ id: 2, isEnabled: true }), + ]; + + const findMock = vi.fn(async () => endpoints); + const isOpenMock = vi.fn(async () => true); + + vi.doMock("@/repository", () => ({ + findProviderEndpointsByVendorAndType: findMock, + })); + vi.doMock("@/lib/endpoint-circuit-breaker", () => ({ + isEndpointCircuitOpen: isOpenMock, + })); + + const { getEndpointFilterStats } = await import("@/lib/provider-endpoints/endpoint-selector"); + const stats = await getEndpointFilterStats({ vendorId: 1, providerType: "openai-compatible" }); + + expect(stats).toEqual({ + total: 2, + enabled: 2, + circuitOpen: 2, + available: 0, + }); + }); +}); diff --git a/tests/unit/proxy/proxy-forwarder-endpoint-audit.test.ts b/tests/unit/proxy/proxy-forwarder-endpoint-audit.test.ts index 18b38957e..6891fff23 100644 --- a/tests/unit/proxy/proxy-forwarder-endpoint-audit.test.ts +++ b/tests/unit/proxy/proxy-forwarder-endpoint-audit.test.ts @@ -3,6 +3,7 @@ import { beforeEach, describe, expect, test, vi } from "vitest"; const mocks = vi.hoisted(() => { return { getPreferredProviderEndpoints: vi.fn(), + getEndpointFilterStats: vi.fn(async () => null), recordEndpointSuccess: vi.fn(async () => {}), recordEndpointFailure: vi.fn(async () => {}), recordSuccess: vi.fn(), @@ -31,6 +32,7 @@ vi.mock("@/lib/logger", () => ({ vi.mock("@/lib/provider-endpoints/endpoint-selector", () => ({ getPreferredProviderEndpoints: mocks.getPreferredProviderEndpoints, + getEndpointFilterStats: mocks.getEndpointFilterStats, })); vi.mock("@/lib/endpoint-circuit-breaker", () => ({ @@ -505,4 +507,186 @@ describe("ProxyForwarder - endpoint audit", () => { const warnMessages = vi.mocked(logger.warn).mock.calls.map(([message]) => message); expect(warnMessages).not.toContain("[ProxyForwarder] Failed to load provider endpoints"); }); + + test("endpoint pool exhausted (no_endpoint_candidates) should record endpoint_pool_exhausted in provider chain", async () => { + const requestPath = "/v1/messages"; + const session = createSession(new URL(`https://example.com${requestPath}`)); + const provider = createProvider({ + providerType: "claude", + providerVendorId: 123, + url: "https://provider.example.com/v1/messages", + }); + session.setProvider(provider); + + // Return empty array => no_endpoint_candidates + mocks.getPreferredProviderEndpoints.mockResolvedValueOnce([]); + mocks.getEndpointFilterStats.mockResolvedValueOnce({ + total: 3, + enabled: 2, + circuitOpen: 2, + available: 0, + }); + + const doForward = vi.spyOn( + ProxyForwarder as unknown as { doForward: (...args: unknown[]) => unknown }, + "doForward" + ); + + await expect(ProxyForwarder.send(session)).rejects.toThrow(); + + expect(doForward).not.toHaveBeenCalled(); + + const chain = session.getProviderChain(); + const exhaustedItem = chain.find((item) => item.reason === "endpoint_pool_exhausted"); + expect(exhaustedItem).toBeDefined(); + expect(exhaustedItem).toEqual( + expect.objectContaining({ + id: provider.id, + name: provider.name, + vendorId: 123, + providerType: "claude", + reason: "endpoint_pool_exhausted", + strictBlockCause: "no_endpoint_candidates", + }) + ); + + // endpointFilterStats should be present at top level + expect(exhaustedItem!.endpointFilterStats).toEqual({ + total: 3, + enabled: 2, + circuitOpen: 2, + available: 0, + }); + + // errorMessage should be undefined for no_endpoint_candidates (no exception) + expect(exhaustedItem!.errorMessage).toBeUndefined(); + }); + + test("endpoint pool exhausted (selector_error) should record endpoint_pool_exhausted with selectorError in decisionContext", async () => { + const requestPath = "/v1/responses"; + const session = createSession(new URL(`https://example.com${requestPath}`)); + const provider = createProvider({ + providerType: "codex", + providerVendorId: 456, + url: "https://provider.example.com/v1/responses", + }); + session.setProvider(provider); + + // Throw error => selector_error cause + mocks.getPreferredProviderEndpoints.mockRejectedValueOnce(new Error("Redis connection lost")); + + const doForward = vi.spyOn( + ProxyForwarder as unknown as { doForward: (...args: unknown[]) => unknown }, + "doForward" + ); + + await expect(ProxyForwarder.send(session)).rejects.toThrow(); + + expect(doForward).not.toHaveBeenCalled(); + + const chain = session.getProviderChain(); + const exhaustedItem = chain.find((item) => item.reason === "endpoint_pool_exhausted"); + expect(exhaustedItem).toBeDefined(); + expect(exhaustedItem).toEqual( + expect.objectContaining({ + id: provider.id, + name: provider.name, + vendorId: 456, + providerType: "codex", + reason: "endpoint_pool_exhausted", + strictBlockCause: "selector_error", + }) + ); + + // selector_error should NOT call getEndpointFilterStats (exception path, no data available) + // endpointFilterStats should be undefined for selector_error + expect(exhaustedItem!.endpointFilterStats).toBeUndefined(); + + // errorMessage should contain the selector error message + expect(exhaustedItem!.errorMessage).toBe("Redis connection lost"); + }); + + test("selector_error and no_endpoint_candidates are correctly distinguished in provider chain", async () => { + // Test 1: selector_error (exception thrown) + const session1 = createSession(new URL("https://example.com/v1/chat/completions")); + const provider1 = createProvider({ + id: 10, + name: "p-selector-err", + providerType: "openai-compatible", + providerVendorId: 789, + }); + session1.setProvider(provider1); + mocks.getPreferredProviderEndpoints.mockRejectedValueOnce(new Error("timeout")); + + await expect(ProxyForwarder.send(session1)).rejects.toThrow(); + + const chain1 = session1.getProviderChain(); + const item1 = chain1.find((i) => i.reason === "endpoint_pool_exhausted"); + expect(item1).toBeDefined(); + expect(item1!.strictBlockCause).toBe("selector_error"); + expect(item1!.endpointFilterStats).toBeUndefined(); + expect(item1!.errorMessage).toBe("timeout"); + + // Test 2: no_endpoint_candidates (empty array returned) + const session2 = createSession(new URL("https://example.com/v1/chat/completions")); + const provider2 = createProvider({ + id: 20, + name: "p-empty-pool", + providerType: "openai-compatible", + providerVendorId: 789, + }); + session2.setProvider(provider2); + mocks.getPreferredProviderEndpoints.mockResolvedValueOnce([]); + mocks.getEndpointFilterStats.mockResolvedValueOnce({ + total: 5, + enabled: 3, + circuitOpen: 3, + available: 0, + }); + + await expect(ProxyForwarder.send(session2)).rejects.toThrow(); + + const chain2 = session2.getProviderChain(); + const item2 = chain2.find((i) => i.reason === "endpoint_pool_exhausted"); + expect(item2).toBeDefined(); + expect(item2!.strictBlockCause).toBe("no_endpoint_candidates"); + expect(item2!.endpointFilterStats).toEqual({ + total: 5, + enabled: 3, + circuitOpen: 3, + available: 0, + }); + expect(item2!.errorMessage).toBeUndefined(); + }); + + test("endpointFilterStats should gracefully handle getEndpointFilterStats failure", async () => { + const requestPath = "/v1/messages"; + const session = createSession(new URL(`https://example.com${requestPath}`)); + const provider = createProvider({ + providerType: "claude", + providerVendorId: 123, + url: "https://provider.example.com/v1/messages", + }); + session.setProvider(provider); + + mocks.getPreferredProviderEndpoints.mockResolvedValueOnce([]); + // Stats call fails - should not break the flow + mocks.getEndpointFilterStats.mockRejectedValueOnce(new Error("DB unavailable")); + + const doForward = vi.spyOn( + ProxyForwarder as unknown as { doForward: (...args: unknown[]) => unknown }, + "doForward" + ); + + await expect(ProxyForwarder.send(session)).rejects.toThrow(); + + expect(doForward).not.toHaveBeenCalled(); + + const chain = session.getProviderChain(); + const exhaustedItem = chain.find((item) => item.reason === "endpoint_pool_exhausted"); + expect(exhaustedItem).toBeDefined(); + expect(exhaustedItem!.strictBlockCause).toBe("no_endpoint_candidates"); + // endpointFilterStats should be undefined when stats call fails + expect(exhaustedItem!.endpointFilterStats).toBeUndefined(); + }); }); diff --git a/tests/unit/settings/providers/endpoint-status.test.ts b/tests/unit/settings/providers/endpoint-status.test.ts index 7352bc31f..1f7e6c7de 100644 --- a/tests/unit/settings/providers/endpoint-status.test.ts +++ b/tests/unit/settings/providers/endpoint-status.test.ts @@ -2,6 +2,8 @@ import { describe, expect, it } from "vitest"; import { type EndpointCircuitState, getEndpointStatusModel, + type IncidentSource, + resolveEndpointDisplayStatus, } from "@/app/[locale]/settings/providers/_components/endpoint-status"; import { AlertTriangle, Ban, CheckCircle2, HelpCircle, XCircle } from "lucide-react"; @@ -99,3 +101,178 @@ describe("getEndpointStatusModel", () => { }); }); }); + +describe("IncidentSource", () => { + it("should have correct type values", () => { + const source: IncidentSource = "provider"; + expect(source).toBe("provider"); + + const endpointSource: IncidentSource = "endpoint"; + expect(endpointSource).toBe("endpoint"); + }); +}); + +describe("resolveEndpointDisplayStatus", () => { + const createEndpoint = (lastProbeOk: boolean | null, isEnabled?: boolean) => + ({ lastProbeOk, isEnabled }) as { lastProbeOk: boolean | null; isEnabled?: boolean }; + + describe("Priority: circuit-open", () => { + it("should return circuit-open with endpoint source when circuit is open", () => { + const endpoint = createEndpoint(true); + const result = resolveEndpointDisplayStatus(endpoint, "open"); + + expect(result).toEqual({ + status: "circuit-open", + source: "endpoint", + priority: 0, + }); + }); + + it("should return circuit-open even when probe is failed", () => { + const endpoint = createEndpoint(false); + const result = resolveEndpointDisplayStatus(endpoint, "open"); + + expect(result).toEqual({ + status: "circuit-open", + source: "endpoint", + priority: 0, + }); + }); + }); + + describe("Priority: circuit-half-open", () => { + it("should return circuit-half-open with endpoint source when circuit is half-open", () => { + const endpoint = createEndpoint(false); + const result = resolveEndpointDisplayStatus(endpoint, "half-open"); + + expect(result).toEqual({ + status: "circuit-half-open", + source: "endpoint", + priority: 1, + }); + }); + + it("should return circuit-half-open even when probe is ok", () => { + const endpoint = createEndpoint(true); + const result = resolveEndpointDisplayStatus(endpoint, "half-open"); + + expect(result).toEqual({ + status: "circuit-half-open", + source: "endpoint", + priority: 1, + }); + }); + }); + + describe("Priority: enabled/disabled (circuit-closed)", () => { + it("should return enabled when circuit is closed and endpoint is enabled", () => { + const endpoint = createEndpoint(true, true); + const result = resolveEndpointDisplayStatus(endpoint, "closed"); + + expect(result).toEqual({ + status: "enabled", + source: "provider", + priority: 2, + }); + }); + + it("should return disabled when circuit is closed and endpoint is disabled", () => { + const endpoint = createEndpoint(true, false); + const result = resolveEndpointDisplayStatus(endpoint, "closed"); + + expect(result).toEqual({ + status: "disabled", + source: "provider", + priority: 3, + }); + }); + + it("should return enabled when circuit is closed and isEnabled is undefined", () => { + const endpoint = createEndpoint(true, undefined); + const result = resolveEndpointDisplayStatus(endpoint, "closed"); + + expect(result).toEqual({ + status: "enabled", + source: "provider", + priority: 2, + }); + }); + + it("should return enabled when circuit is closed and isEnabled is null", () => { + const endpoint = createEndpoint(true, null as unknown as undefined); + const result = resolveEndpointDisplayStatus(endpoint, "closed"); + + expect(result).toEqual({ + status: "enabled", + source: "provider", + priority: 2, + }); + }); + }); + + describe("Priority ordering", () => { + it("should have circuit-open (0) > circuit-half-open (1) > enabled (2) > disabled (3)", () => { + const endpoint = createEndpoint(true, true); + + const openResult = resolveEndpointDisplayStatus(endpoint, "open"); + const halfOpenResult = resolveEndpointDisplayStatus(endpoint, "half-open"); + const enabledResult = resolveEndpointDisplayStatus(endpoint, "closed"); + + const disabledEndpoint = createEndpoint(true, false); + const disabledResult = resolveEndpointDisplayStatus(disabledEndpoint, "closed"); + + expect(openResult.priority).toBe(0); + expect(halfOpenResult.priority).toBe(1); + expect(enabledResult.priority).toBe(2); + expect(disabledResult.priority).toBe(3); + }); + }); + + describe("Null/undefined circuit state", () => { + it("should return enabled when circuit is null and endpoint is enabled", () => { + const endpoint = createEndpoint(true, true); + const result = resolveEndpointDisplayStatus(endpoint, null); + + expect(result).toEqual({ + status: "enabled", + source: "provider", + priority: 2, + }); + }); + + it("should return enabled when circuit is undefined", () => { + const endpoint = createEndpoint(true, true); + const result = resolveEndpointDisplayStatus(endpoint, undefined); + + expect(result).toEqual({ + status: "enabled", + source: "provider", + priority: 2, + }); + }); + }); + + describe("Edge cases", () => { + it("should handle endpoint without isEnabled property", () => { + const endpoint = { lastProbeOk: true } as { lastProbeOk: boolean | null }; + const result = resolveEndpointDisplayStatus(endpoint, "closed"); + + expect(result).toEqual({ + status: "enabled", + source: "provider", + priority: 2, + }); + }); + + it("should return circuit-open when probe is null and circuit is open", () => { + const endpoint = createEndpoint(null); + const result = resolveEndpointDisplayStatus(endpoint, "open"); + + expect(result).toEqual({ + status: "circuit-open", + source: "endpoint", + priority: 0, + }); + }); + }); +}); diff --git a/tests/unit/settings/providers/provider-manager.test.tsx b/tests/unit/settings/providers/provider-manager.test.tsx new file mode 100644 index 000000000..be36972cb --- /dev/null +++ b/tests/unit/settings/providers/provider-manager.test.tsx @@ -0,0 +1,441 @@ +/** + * @vitest-environment happy-dom + */ + +import { NextIntlClientProvider } from "next-intl"; +import { type ReactNode, act } from "react"; +import { createRoot } from "react-dom/client"; +import { beforeEach, describe, expect, test, vi } from "vitest"; +import type { ProviderDisplay } from "@/types/provider"; +import enMessages from "../../../../messages/en"; + +// --------------------------------------------------------------------------- +// Mocks -- keep them minimal, only stub what provider-manager.tsx touches +// --------------------------------------------------------------------------- + +vi.mock("@/lib/hooks/use-debounce", () => ({ + useDebounce: (value: string, _delay: number) => value, +})); + +// Batch-edit subcomponents (heavy, irrelevant to this test scope) +vi.mock("@/app/[locale]/settings/providers/_components/batch-edit", () => ({ + ProviderBatchActions: () => null, + ProviderBatchDialog: () => null, + ProviderBatchToolbar: () => null, +})); + +// ProviderList -- render a simple list so we can inspect filtered output +vi.mock("@/app/[locale]/settings/providers/_components/provider-list", () => ({ + ProviderList: ({ providers }: { providers: ProviderDisplay[] }) => ( +
    + {providers.map((p) => ( +
  • + {p.name} +
    • + ))} +
+ ), +})); + +// ProviderVendorView -- not under test +vi.mock("@/app/[locale]/settings/providers/_components/provider-vendor-view", () => ({ + ProviderVendorView: () => null, +})); + +// ProviderTypeFilter +vi.mock("@/app/[locale]/settings/providers/_components/provider-type-filter", () => ({ + ProviderTypeFilter: () => null, +})); + +// ProviderSortDropdown +vi.mock("@/app/[locale]/settings/providers/_components/provider-sort-dropdown", () => ({ + ProviderSortDropdown: () => null, +})); + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +function makeProvider(overrides: Partial = {}): ProviderDisplay { + return { + id: 1, + name: "Provider A", + url: "https://api.example.com", + maskedKey: "sk-***", + isEnabled: true, + weight: 1, + priority: 1, + costMultiplier: 1, + groupTag: null, + groupPriorities: null, + providerType: "claude", + providerVendorId: null, + preserveClientIp: false, + modelRedirects: null, + allowedModels: null, + mcpPassthroughType: "none", + mcpPassthroughUrl: null, + limit5hUsd: null, + limitDailyUsd: null, + dailyResetMode: "fixed", + dailyResetTime: "00:00", + limitWeeklyUsd: null, + limitMonthlyUsd: null, + limitTotalUsd: null, + limitConcurrentSessions: 1, + maxRetryAttempts: null, + circuitBreakerFailureThreshold: 1, + circuitBreakerOpenDuration: 60, + circuitBreakerHalfOpenSuccessThreshold: 1, + proxyUrl: null, + proxyFallbackToDirect: false, + firstByteTimeoutStreamingMs: 0, + streamingIdleTimeoutMs: 0, + requestTimeoutNonStreamingMs: 0, + websiteUrl: null, + faviconUrl: null, + cacheTtlPreference: null, + context1mPreference: null, + codexReasoningEffortPreference: null, + codexReasoningSummaryPreference: null, + codexTextVerbosityPreference: null, + codexParallelToolCallsPreference: null, + anthropicMaxTokensPreference: null, + anthropicThinkingBudgetPreference: null, + geminiGoogleSearchPreference: null, + tpm: null, + rpm: null, + rpd: null, + cc: null, + createdAt: "2026-01-01", + updatedAt: "2026-01-01", + ...overrides, + }; +} + +function renderWithProviders(node: ReactNode) { + const container = document.createElement("div"); + document.body.appendChild(container); + const root = createRoot(container); + + act(() => { + root.render( + + {node} + + ); + }); + + return { + unmount: () => { + act(() => root.unmount()); + container.remove(); + }, + container, + }; +} + +// --------------------------------------------------------------------------- +// Tests +// --------------------------------------------------------------------------- + +// Lazy-import after mocks are established +let ProviderManager: typeof import("@/app/[locale]/settings/providers/_components/provider-manager").ProviderManager; + +beforeEach(async () => { + vi.clearAllMocks(); + while (document.body.firstChild) { + document.body.removeChild(document.body.firstChild); + } + // Dynamic import to ensure mocks take effect + const mod = await import("@/app/[locale]/settings/providers/_components/provider-manager"); + ProviderManager = mod.ProviderManager; +}); + +describe("ProviderManager circuitBrokenCount with endpoint circuits", () => { + const providers = [ + makeProvider({ id: 1, name: "Provider A" }), + makeProvider({ id: 2, name: "Provider B" }), + makeProvider({ id: 3, name: "Provider C" }), + ]; + + test("counts only key-level circuit breaker when no endpointCircuitInfo", () => { + const healthStatus = { + 1: { + circuitState: "open" as const, + failureCount: 5, + lastFailureTime: Date.now(), + circuitOpenUntil: Date.now() + 60000, + recoveryMinutes: 1, + }, + }; + + const { unmount, container } = renderWithProviders( + + ); + + // The circuit broken count should show 1 (only Provider A has key-level open) + const text = container.textContent || ""; + expect(text).toContain("(1)"); + + unmount(); + }); + + test("counts providers with endpoint-level circuit open in addition to key-level", () => { + // Provider 1: key-level circuit open + // Provider 2: healthy key, but has an endpoint circuit open + // Provider 3: all healthy + const healthStatus = { + 1: { + circuitState: "open" as const, + failureCount: 5, + lastFailureTime: Date.now(), + circuitOpenUntil: Date.now() + 60000, + recoveryMinutes: 1, + }, + }; + + const endpointCircuitInfo: Record< + number, + Array<{ + endpointId: number; + circuitState: "closed" | "open" | "half-open"; + failureCount: number; + circuitOpenUntil: number | null; + }> + > = { + 2: [ + { + endpointId: 10, + circuitState: "open", + failureCount: 3, + circuitOpenUntil: Date.now() + 60000, + }, + { + endpointId: 11, + circuitState: "closed", + failureCount: 0, + circuitOpenUntil: null, + }, + ], + }; + + const { unmount, container } = renderWithProviders( + + ); + + // Count should be 2: Provider A (key open) + Provider B (endpoint open) + const text = container.textContent || ""; + expect(text).toContain("(2)"); + + unmount(); + }); + + test("does not double-count provider with both key and endpoint circuits open", () => { + const healthStatus = { + 1: { + circuitState: "open" as const, + failureCount: 5, + lastFailureTime: Date.now(), + circuitOpenUntil: Date.now() + 60000, + recoveryMinutes: 1, + }, + }; + + const endpointCircuitInfo: Record< + number, + Array<{ + endpointId: number; + circuitState: "closed" | "open" | "half-open"; + failureCount: number; + circuitOpenUntil: number | null; + }> + > = { + 1: [ + { + endpointId: 10, + circuitState: "open", + failureCount: 3, + circuitOpenUntil: Date.now() + 60000, + }, + ], + }; + + const { unmount, container } = renderWithProviders( + + ); + + // Should still be 1 -- provider 1 has both, but count is deduplicated + const text = container.textContent || ""; + expect(text).toContain("(1)"); + + unmount(); + }); + + test("circuit broken filter includes providers with endpoint circuits open", () => { + // Use a state-based approach: + // We'll set circuitBrokenFilter active programmatically by clicking the toggle. + // Provider 2 only has an endpoint circuit open (no key circuit). + + const healthStatus = {}; + const endpointCircuitInfo: Record< + number, + Array<{ + endpointId: number; + circuitState: "closed" | "open" | "half-open"; + failureCount: number; + circuitOpenUntil: number | null; + }> + > = { + 2: [ + { + endpointId: 10, + circuitState: "open", + failureCount: 3, + circuitOpenUntil: Date.now() + 60000, + }, + ], + }; + + const { unmount, container } = renderWithProviders( + + ); + + // Circuit broken count should be 1 (Provider B has endpoint open) + const text = container.textContent || ""; + expect(text).toContain("(1)"); + + // Find and click the circuit broken toggle + const toggle = container.querySelector("#circuit-broken-filter"); + expect(toggle).not.toBeNull(); + + act(() => { + toggle!.dispatchEvent(new MouseEvent("click", { bubbles: true })); + }); + + // After activating the filter, only Provider B should be shown + const listItems = container.querySelectorAll("[data-testid^='provider-']"); + const providerNames = Array.from(listItems).map((el) => el.textContent); + expect(providerNames).toContain("Provider B"); + expect(providerNames).not.toContain("Provider A"); + expect(providerNames).not.toContain("Provider C"); + + unmount(); + }); + + test("shows zero circuit broken count when no circuits are open", () => { + const healthStatus = {}; + + const { unmount, container } = renderWithProviders( + + ); + + // When count is 0, the circuit broken section should NOT be rendered + const toggleDesktop = container.querySelector("#circuit-broken-filter"); + expect(toggleDesktop).toBeNull(); + + unmount(); + }); + + test("endpointCircuitInfo defaults to empty when not provided", () => { + const healthStatus = {}; + + const { unmount, container } = renderWithProviders( + + ); + + // No circuit broken UI should appear + const toggleDesktop = container.querySelector("#circuit-broken-filter"); + expect(toggleDesktop).toBeNull(); + + unmount(); + }); +}); + +describe("ProviderManager layered circuit labels", () => { + const providers = [ + makeProvider({ id: 1, name: "Provider Key Broken" }), + makeProvider({ id: 2, name: "Provider Endpoint Broken" }), + makeProvider({ id: 3, name: "Provider Both Broken" }), + ]; + + test("counts all providers with any circuit open for layered labels", () => { + const healthStatus = { + 1: { + circuitState: "open" as const, + failureCount: 5, + lastFailureTime: Date.now(), + circuitOpenUntil: Date.now() + 60000, + recoveryMinutes: 1, + }, + 3: { + circuitState: "open" as const, + failureCount: 3, + lastFailureTime: Date.now(), + circuitOpenUntil: Date.now() + 30000, + recoveryMinutes: 0.5, + }, + }; + + const endpointCircuitInfo = { + 2: [ + { + endpointId: 20, + circuitState: "open" as const, + failureCount: 2, + circuitOpenUntil: Date.now() + 60000, + }, + ], + 3: [ + { + endpointId: 30, + circuitState: "open" as const, + failureCount: 4, + circuitOpenUntil: Date.now() + 60000, + }, + ], + }; + + const { unmount, container } = renderWithProviders( + + ); + + // The circuit broken count should be 3 (all three providers have some form of circuit open) + const text = container.textContent || ""; + expect(text).toContain("(3)"); + + unmount(); + }); +}); diff --git a/tests/unit/settings/providers/provider-vendor-view-circuit-ui.test.tsx b/tests/unit/settings/providers/provider-vendor-view-circuit-ui.test.tsx index c4f286f5a..1656f373b 100644 --- a/tests/unit/settings/providers/provider-vendor-view-circuit-ui.test.tsx +++ b/tests/unit/settings/providers/provider-vendor-view-circuit-ui.test.tsx @@ -39,7 +39,7 @@ const providerEndpointsActionMocks = vi.hoisted(() => ({ sortOrder: 0, isEnabled: true, lastProbedAt: null, - lastOk: null, + lastProbeOk: null, lastLatencyMs: null, createdAt: "2026-01-01", updatedAt: "2026-01-01", @@ -59,6 +59,8 @@ const providerEndpointsActionMocks = vi.hoisted(() => ({ probeProviderEndpoint: vi.fn(async () => ({ ok: true, data: { result: { ok: true } } })), removeProviderEndpoint: vi.fn(async () => ({ ok: true })), removeProviderVendor: vi.fn(async () => ({ ok: true })), + batchGetEndpointCircuitInfo: vi.fn(async () => ({ ok: true, data: [] })), + resetEndpointCircuit: vi.fn(async () => ({ ok: true })), })); vi.mock("@/actions/provider-endpoints", () => providerEndpointsActionMocks); @@ -129,12 +131,16 @@ function makeProviderDisplay(overrides: Partial = {}): Provider codexReasoningSummaryPreference: null, codexTextVerbosityPreference: null, codexParallelToolCallsPreference: null, + anthropicMaxTokensPreference: null, + anthropicThinkingBudgetPreference: null, + geminiGoogleSearchPreference: null, tpm: null, rpm: null, rpd: null, cc: null, createdAt: "2026-01-01", updatedAt: "2026-01-01", + groupPriorities: null, ...overrides, }; } @@ -340,3 +346,347 @@ describe("ProviderVendorView endpoints table", () => { unmount(); }); }); + +// ============================================================================ +// NEW: Circuit breaker display and reset action tests +// ============================================================================ + +describe("Endpoint circuit breaker badge display", () => { + beforeEach(() => { + queryClient = new QueryClient({ + defaultOptions: { + queries: { retry: false }, + mutations: { retry: false }, + }, + }); + vi.clearAllMocks(); + while (document.body.firstChild) { + document.body.removeChild(document.body.firstChild); + } + }); + + test("shows circuit-open badge when batchGetEndpointCircuitInfo returns open state", async () => { + // Mock: endpoint 1 circuit is open + providerEndpointsActionMocks.batchGetEndpointCircuitInfo.mockResolvedValue({ + ok: true, + data: [ + { + endpointId: 1, + circuitState: "open", + failureCount: 5, + circuitOpenUntil: Date.now() + 60000, + }, + ], + }); + + const { unmount } = renderWithProviders( + + ); + + await flushTicks(8); + + // Expect "Circuit Open" text to be rendered in the status cell + const bodyText = document.body.textContent || ""; + expect(bodyText).toContain("Circuit Open"); + + unmount(); + }); + + test("shows circuit-half-open badge when batchGetEndpointCircuitInfo returns half-open state", async () => { + providerEndpointsActionMocks.batchGetEndpointCircuitInfo.mockResolvedValue({ + ok: true, + data: [ + { + endpointId: 1, + circuitState: "half-open", + failureCount: 3, + circuitOpenUntil: null, + }, + ], + }); + + const { unmount } = renderWithProviders( + + ); + + await flushTicks(8); + + const bodyText = document.body.textContent || ""; + expect(bodyText).toContain("Circuit Half-Open"); + + unmount(); + }); + + test("does not show circuit badge when circuit is closed", async () => { + providerEndpointsActionMocks.batchGetEndpointCircuitInfo.mockResolvedValue({ + ok: true, + data: [ + { + endpointId: 1, + circuitState: "closed", + failureCount: 0, + circuitOpenUntil: null, + }, + ], + }); + + const { unmount } = renderWithProviders( + + ); + + await flushTicks(8); + + const bodyText = document.body.textContent || ""; + expect(bodyText).not.toContain("Circuit Open"); + expect(bodyText).not.toContain("Circuit Half-Open"); + + unmount(); + }); + + test("uses resolveEndpointDisplayStatus to determine badge when circuit is open and endpoint is enabled", async () => { + // Endpoint is enabled but circuit is open => circuit-open takes priority + providerEndpointsActionMocks.batchGetEndpointCircuitInfo.mockResolvedValue({ + ok: true, + data: [ + { + endpointId: 1, + circuitState: "open", + failureCount: 5, + circuitOpenUntil: Date.now() + 60000, + }, + ], + }); + + const { unmount } = renderWithProviders( + + ); + + await flushTicks(8); + + // Circuit-open badge should take priority over enabled badge + const bodyText = document.body.textContent || ""; + expect(bodyText).toContain("Circuit Open"); + + unmount(); + }); +}); + +describe("Endpoint circuit breaker reset action", () => { + beforeEach(() => { + queryClient = new QueryClient({ + defaultOptions: { + queries: { retry: false }, + mutations: { retry: false }, + }, + }); + vi.clearAllMocks(); + while (document.body.firstChild) { + document.body.removeChild(document.body.firstChild); + } + }); + + test("reset endpoint circuit action is available when circuit is open", async () => { + // Mock circuit state as open + providerEndpointsActionMocks.batchGetEndpointCircuitInfo.mockResolvedValue({ + ok: true, + data: [ + { + endpointId: 1, + circuitState: "open", + failureCount: 5, + circuitOpenUntil: Date.now() + 60000, + }, + ], + }); + providerEndpointsActionMocks.resetEndpointCircuit.mockResolvedValue({ ok: true }); + + const { unmount } = renderWithProviders( + + ); + + await flushTicks(8); + + // Verify the circuit state query was called with correct endpoint + expect(providerEndpointsActionMocks.batchGetEndpointCircuitInfo).toHaveBeenCalled(); + + // Verify circuit badge is shown + expect(document.body.textContent || "").toContain("Circuit Open"); + + // Call the reset action directly and verify it succeeds + const resetResult = await providerEndpointsActionMocks.resetEndpointCircuit({ endpointId: 1 }); + expect(resetResult.ok).toBe(true); + + unmount(); + }); + + test("resetEndpointCircuit action is called correctly when circuit is open", async () => { + providerEndpointsActionMocks.batchGetEndpointCircuitInfo.mockResolvedValue({ + ok: true, + data: [ + { + endpointId: 1, + circuitState: "open", + failureCount: 5, + circuitOpenUntil: Date.now() + 60000, + }, + ], + }); + providerEndpointsActionMocks.resetEndpointCircuit.mockResolvedValue({ ok: true }); + + const { unmount } = renderWithProviders( + + ); + + await flushTicks(8); + + // Verify the circuit state query was called with correct endpoint + expect(providerEndpointsActionMocks.batchGetEndpointCircuitInfo).toHaveBeenCalled(); + + // Verify circuit badge is shown + expect(document.body.textContent || "").toContain("Circuit Open"); + + // The reset endpoint circuit action is available and will be called when user clicks + // Verify the mock is properly set up to handle the call + const resetResult = await providerEndpointsActionMocks.resetEndpointCircuit({ endpointId: 1 }); + expect(resetResult.ok).toBe(true); + expect(providerEndpointsActionMocks.resetEndpointCircuit).toHaveBeenCalledWith( + expect.objectContaining({ endpointId: 1 }) + ); + + unmount(); + }); + + test("resetEndpointCircuit action handles failure correctly", async () => { + providerEndpointsActionMocks.batchGetEndpointCircuitInfo.mockResolvedValue({ + ok: true, + data: [ + { + endpointId: 1, + circuitState: "open", + failureCount: 5, + circuitOpenUntil: Date.now() + 60000, + }, + ], + }); + providerEndpointsActionMocks.resetEndpointCircuit.mockResolvedValue({ + ok: false, + error: "Failed to reset circuit", + }); + + const { unmount } = renderWithProviders( + + ); + + await flushTicks(8); + + // Verify the circuit state query was called + expect(providerEndpointsActionMocks.batchGetEndpointCircuitInfo).toHaveBeenCalled(); + + // Verify circuit badge is shown + expect(document.body.textContent || "").toContain("Circuit Open"); + + // Call the reset action and expect failure + const resetResult = await providerEndpointsActionMocks.resetEndpointCircuit({ endpointId: 1 }); + expect(resetResult.ok).toBe(false); + expect(resetResult.error).toBe("Failed to reset circuit"); + + unmount(); + }); + + test("reset circuit action not needed when circuit is closed", async () => { + providerEndpointsActionMocks.batchGetEndpointCircuitInfo.mockResolvedValue({ + ok: true, + data: [ + { + endpointId: 1, + circuitState: "closed", + failureCount: 0, + circuitOpenUntil: null, + }, + ], + }); + + const { unmount } = renderWithProviders( + + ); + + await flushTicks(8); + + // Verify circuit badge is NOT shown (only enabled badge) + expect(document.body.textContent || "").not.toContain("Circuit Open"); + expect(document.body.textContent || "").not.toContain("Circuit Half-Open"); + + // Circuit is closed, so reset action is not expected to be needed + // Verify no circuit badges are present + const circuitBadges = document.querySelectorAll( + '[class*="text-rose-500"], [class*="text-amber-500"]' + ); + expect(circuitBadges.length).toBe(0); + + unmount(); + }); +}); diff --git a/tests/unit/webhook/notifier-circuit-breaker.test.ts b/tests/unit/webhook/notifier-circuit-breaker.test.ts new file mode 100644 index 000000000..92b6bbbdd --- /dev/null +++ b/tests/unit/webhook/notifier-circuit-breaker.test.ts @@ -0,0 +1,181 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import type { CircuitBreakerAlertData } from "@/lib/webhook/types"; + +describe("sendCircuitBreakerAlert", () => { + const mockRedisGet = vi.fn(); + const mockRedisSet = vi.fn(); + const mockAddNotificationJob = vi.fn(async () => {}); + const mockAddNotificationJobForTarget = vi.fn(async () => {}); + + beforeEach(() => { + vi.resetModules(); + + vi.doMock("@/lib/redis/client", () => ({ + getRedisClient: vi.fn(() => ({ + get: mockRedisGet, + set: mockRedisSet, + })), + })); + + vi.doMock("@/repository/notifications", () => ({ + getNotificationSettings: vi.fn(async () => ({ + enabled: true, + circuitBreakerEnabled: true, + useLegacyMode: true, + circuitBreakerWebhook: "https://qyapi.weixin.qq.com/cgi-bin/webhook/send?key=xxx", + })), + })); + + vi.doMock("@/lib/notification/notification-queue", () => ({ + addNotificationJob: mockAddNotificationJob, + addNotificationJobForTarget: mockAddNotificationJobForTarget, + })); + + vi.doMock("@/lib/logger", () => ({ + logger: { + debug: vi.fn(), + info: vi.fn(), + warn: vi.fn(), + error: vi.fn(), + trace: vi.fn(), + fatal: vi.fn(), + }, + })); + }); + + afterEach(() => { + vi.clearAllMocks(); + }); + + describe("dedup key with incidentSource", () => { + it("should use provider dedup key when incidentSource is provider", async () => { + mockRedisGet.mockResolvedValue(null); // No cached alert + + const { sendCircuitBreakerAlert } = await import("@/lib/notification/notifier"); + + const data: CircuitBreakerAlertData = { + providerName: "OpenAI", + providerId: 1, + failureCount: 5, + retryAt: "2025-01-02T12:30:00Z", + incidentSource: "provider", + }; + + await sendCircuitBreakerAlert(data); + + // Should use dedup key with provider source + expect(mockRedisSet).toHaveBeenCalledWith("circuit-breaker-alert:1:provider", "1", "EX", 300); + }); + + it("should use endpoint dedup key when incidentSource is endpoint", async () => { + mockRedisGet.mockResolvedValue(null); + + const { sendCircuitBreakerAlert } = await import("@/lib/notification/notifier"); + + const data: CircuitBreakerAlertData = { + providerName: "OpenAI", + providerId: 1, + failureCount: 3, + retryAt: "2025-01-02T13:00:00Z", + incidentSource: "endpoint", + endpointId: 42, + endpointUrl: "https://api.openai.com/v1", + }; + + await sendCircuitBreakerAlert(data); + + // Should use dedup key with endpoint source including endpointId + expect(mockRedisSet).toHaveBeenCalledWith( + "circuit-breaker-alert:1:endpoint:42", + "1", + "EX", + 300 + ); + }); + + it("should dedup independently for same provider with different sources", async () => { + // Provider alert is cached + mockRedisGet.mockResolvedValueOnce("1"); + // Endpoint alert is NOT cached + mockRedisGet.mockResolvedValueOnce(null); + + const { sendCircuitBreakerAlert } = await import("@/lib/notification/notifier"); + + const providerData: CircuitBreakerAlertData = { + providerName: "OpenAI", + providerId: 1, + failureCount: 5, + retryAt: "2025-01-02T12:30:00Z", + incidentSource: "provider", + }; + + const endpointData: CircuitBreakerAlertData = { + providerName: "OpenAI", + providerId: 1, + failureCount: 3, + retryAt: "2025-01-02T13:00:00Z", + incidentSource: "endpoint", + endpointId: 42, + endpointUrl: "https://api.openai.com/v1", + }; + + await sendCircuitBreakerAlert(providerData); + await sendCircuitBreakerAlert(endpointData); + + // Provider alert should be suppressed (cached) + expect(mockRedisSet).toHaveBeenCalledTimes(1); + // That one call should be for endpoint source + expect(mockRedisSet).toHaveBeenCalledWith( + "circuit-breaker-alert:1:endpoint:42", + "1", + "EX", + 300 + ); + }); + + it("should default to provider source when incidentSource is undefined", async () => { + mockRedisGet.mockResolvedValue(null); + + const { sendCircuitBreakerAlert } = await import("@/lib/notification/notifier"); + + const data: CircuitBreakerAlertData = { + providerName: "Anthropic", + providerId: 2, + failureCount: 3, + retryAt: "2025-01-02T13:00:00Z", + // incidentSource is undefined - should default to provider + }; + + await sendCircuitBreakerAlert(data); + + // Should use dedup key with default provider source + expect(mockRedisSet).toHaveBeenCalledWith("circuit-breaker-alert:2:provider", "1", "EX", 300); + }); + + it("should suppress endpoint alert when same endpointId was recently alerted", async () => { + // First call: not cached + mockRedisGet.mockResolvedValueOnce(null); + // Second call: cached + mockRedisGet.mockResolvedValueOnce("1"); + + const { sendCircuitBreakerAlert } = await import("@/lib/notification/notifier"); + + const data: CircuitBreakerAlertData = { + providerName: "OpenAI", + providerId: 1, + failureCount: 3, + retryAt: "2025-01-02T13:00:00Z", + incidentSource: "endpoint", + endpointId: 42, + }; + + await sendCircuitBreakerAlert(data); + await sendCircuitBreakerAlert(data); + + // Only first call should have set cache + expect(mockRedisSet).toHaveBeenCalledTimes(1); + // Should have checked cache twice + expect(mockRedisGet).toHaveBeenCalledTimes(2); + }); + }); +}); diff --git a/tests/unit/webhook/templates/placeholders.test.ts b/tests/unit/webhook/templates/placeholders.test.ts index 5ab2e9df4..999c7171d 100644 --- a/tests/unit/webhook/templates/placeholders.test.ts +++ b/tests/unit/webhook/templates/placeholders.test.ts @@ -71,6 +71,57 @@ describe("Webhook Template Placeholders", () => { expect(vars["{{sections}}"]).toContain("footer"); }); + it("buildTemplateVariables should include endpoint variables when source is endpoint", () => { + const message: StructuredMessage = { + header: { title: "端点熔断告警", level: "error" }, + sections: [], + timestamp: new Date("2025-01-02T12:00:00Z"), + }; + + const vars = buildTemplateVariables({ + message, + notificationType: "circuit_breaker", + data: { + providerName: "OpenAI", + providerId: 1, + failureCount: 5, + retryAt: "2025-01-02T13:00:00Z", + lastError: "connection refused", + incidentSource: "endpoint", + endpointId: 42, + endpointUrl: "https://api.openai.com/v1/chat/completions", + }, + }); + + expect(vars["{{incident_source}}"]).toBe("endpoint"); + expect(vars["{{endpoint_id}}"]).toBe("42"); + expect(vars["{{endpoint_url}}"]).toBe("https://api.openai.com/v1/chat/completions"); + }); + + it("buildTemplateVariables should have empty endpoint variables when source is provider", () => { + const message: StructuredMessage = { + header: { title: "供应商熔断告警", level: "error" }, + sections: [], + timestamp: new Date("2025-01-02T12:00:00Z"), + }; + + const vars = buildTemplateVariables({ + message, + notificationType: "circuit_breaker", + data: { + providerName: "Anthropic", + providerId: 2, + failureCount: 3, + retryAt: "2025-01-02T13:00:00Z", + incidentSource: "provider", + }, + }); + + expect(vars["{{incident_source}}"]).toBe("provider"); + expect(vars["{{endpoint_id}}"]).toBe(""); + expect(vars["{{endpoint_url}}"]).toBe(""); + }); + it("buildTemplateVariables should handle daily_leaderboard entries JSON stringify errors", () => { // 构造循环引用，验证 safeJsonStringify 降级 // eslint-disable-next-line @typescript-eslint/no-explicit-any diff --git a/tests/unit/webhook/templates/templates.test.ts b/tests/unit/webhook/templates/templates.test.ts index 2dcd1d294..e6eccf603 100644 --- a/tests/unit/webhook/templates/templates.test.ts +++ b/tests/unit/webhook/templates/templates.test.ts @@ -42,6 +42,77 @@ describe("Message Templates", () => { const message = buildCircuitBreakerMessage(data); expect(message.header.level).toBe("error"); }); + + it("should default to provider source when incidentSource is not set", () => { + const data: CircuitBreakerAlertData = { + providerName: "OpenAI", + providerId: 1, + failureCount: 5, + retryAt: "2025-01-02T12:30:00Z", + }; + + const message = buildCircuitBreakerMessage(data); + const sectionsStr = JSON.stringify(message.sections); + expect(sectionsStr).toContain("OpenAI"); + // Default should use provider-style title + expect(message.header.title).toMatch(/供应商/); + }); + + it("should produce provider-specific message when incidentSource is provider", () => { + const data: CircuitBreakerAlertData = { + providerName: "Anthropic", + providerId: 2, + failureCount: 3, + retryAt: "2025-01-02T13:00:00Z", + incidentSource: "provider", + }; + + const message = buildCircuitBreakerMessage(data); + expect(message.header.title).toMatch(/供应商/); + const sectionsStr = JSON.stringify(message.sections); + expect(sectionsStr).toContain("Anthropic"); + expect(sectionsStr).toContain("ID: 2"); + }); + + it("should produce endpoint-specific message when incidentSource is endpoint", () => { + const data: CircuitBreakerAlertData = { + providerName: "OpenAI", + providerId: 1, + failureCount: 3, + retryAt: "2025-01-02T13:00:00Z", + incidentSource: "endpoint", + endpointId: 42, + endpointUrl: "https://api.openai.com/v1", + }; + + const message = buildCircuitBreakerMessage(data); + expect(message.header.title).toMatch(/端点/); + const sectionsStr = JSON.stringify(message.sections); + expect(sectionsStr).toContain("42"); + expect(sectionsStr).toContain("https://api.openai.com/v1"); + }); + + it("should include endpoint fields in details when source is endpoint", () => { + const data: CircuitBreakerAlertData = { + providerName: "OpenAI", + providerId: 1, + failureCount: 5, + retryAt: "2025-01-02T12:30:00Z", + lastError: "Connection refused", + incidentSource: "endpoint", + endpointId: 99, + endpointUrl: "https://custom-proxy.example.com/v1", + }; + + const message = buildCircuitBreakerMessage(data); + const sectionsStr = JSON.stringify(message.sections); + // Should still include failure count and error + expect(sectionsStr).toContain("5"); + expect(sectionsStr).toContain("Connection refused"); + // Should include endpoint-specific info + expect(sectionsStr).toContain("99"); + expect(sectionsStr).toContain("https://custom-proxy.example.com/v1"); + }); }); describe("buildCostAlertMessage", () => {