From 183ee60b9731f85edcecbad72357bec2dd3983b1 Mon Sep 17 00:00:00 2001 From: ding113 Date: Tue, 10 Feb 2026 13:46:27 +0800 Subject: [PATCH] feat(leaderboard): add provider avg-cost metrics and cache-hit model drilldown Provider ranking adds avgCostPerRequest and avgCostPerMillionTokens with null-safe division guards. Cache-hit ranking gains per-provider expandable model-level breakdown (modelStats) via a second grouped query keyed by billingModelSource. LeaderboardTable now supports generic expandable rows. API route formats new fields immutably via spread pattern. i18n keys added for all 5 locales. 16 new tests cover repository semantics, API formatting, and multi-provider model grouping. --- messages/en/dashboard.json | 6 +- messages/ja/dashboard.json | 6 +- messages/ru/dashboard.json | 6 +- messages/zh-CN/dashboard.json | 6 +- messages/zh-TW/dashboard.json | 6 +- .../_components/leaderboard-table.tsx | 86 +++- .../_components/leaderboard-view.tsx | 96 +++- src/app/api/leaderboard/route.ts | 44 +- src/repository/leaderboard.ts | 92 +++- tests/unit/api/leaderboard-route.test.ts | 115 +++++ .../leaderboard-provider-metrics.test.ts | 473 ++++++++++++++++++ 11 files changed, 893 insertions(+), 43 deletions(-) create mode 100644 tests/unit/repository/leaderboard-provider-metrics.test.ts diff --git a/messages/en/dashboard.json b/messages/en/dashboard.json index d30cd49d8..f26229c75 100644 --- a/messages/en/dashboard.json +++ b/messages/en/dashboard.json @@ -410,8 +410,12 @@ "successRate": "Success Rate", "avgResponseTime": "Avg Response Time", "avgTtfbMs": "Avg TTFB", - "avgTokensPerSecond": "Avg tok/s" + "avgTokensPerSecond": "Avg tok/s", + "avgCostPerRequest": "Avg Cost/Req", + "avgCostPerMillionTokens": "Avg Cost/1M Tokens" }, + "expandModelStats": "Expand model details", + "collapseModelStats": "Collapse model details", "states": { "loading": "Loading...", "noData": "No data available", diff --git a/messages/ja/dashboard.json b/messages/ja/dashboard.json index c73664cf5..fd6a42f10 100644 --- a/messages/ja/dashboard.json +++ b/messages/ja/dashboard.json @@ -410,8 +410,12 @@ "successRate": "成功率(%)", "avgResponseTime": "平均応答時間", "avgTtfbMs": "平均TTFB", - "avgTokensPerSecond": "平均トークン/秒" + "avgTokensPerSecond": "平均トークン/秒", + "avgCostPerRequest": "平均リクエスト単価", + "avgCostPerMillionTokens": "100万トークンあたりコスト" }, + "expandModelStats": "モデル詳細を展開", + "collapseModelStats": "モデル詳細を折りたたむ", "states": { "loading": "読み込み中...", "noData": "データなし", diff --git a/messages/ru/dashboard.json b/messages/ru/dashboard.json index 0a0dc47fc..48337f945 100644 --- a/messages/ru/dashboard.json +++ b/messages/ru/dashboard.json @@ -410,8 +410,12 @@ "successRate": "Процент успеха", "avgResponseTime": "Среднее время ответа", "avgTtfbMs": "Средний TTFB", - "avgTokensPerSecond": "Средн. ток/с" + "avgTokensPerSecond": "Средн. ток/с", + "avgCostPerRequest": "Ср. стоимость/запрос", + "avgCostPerMillionTokens": "Ср. стоимость/1М токенов" }, + "expandModelStats": "Развернуть модели", + "collapseModelStats": "Свернуть модели", "states": { "loading": "Загрузка...", "noData": "Нет данных", diff --git a/messages/zh-CN/dashboard.json b/messages/zh-CN/dashboard.json index 21dbf23c9..823caee5a 100644 --- a/messages/zh-CN/dashboard.json +++ b/messages/zh-CN/dashboard.json @@ -410,8 +410,12 @@ "successRate": "成功率", "avgResponseTime": "平均响应时间", "avgTtfbMs": "平均 TTFB", - "avgTokensPerSecond": "平均输出速率" + "avgTokensPerSecond": "平均输出速率", + "avgCostPerRequest": "平均单次请求成本", + "avgCostPerMillionTokens": "平均百万 Token 成本" }, + "expandModelStats": "展开模型详情", + "collapseModelStats": "收起模型详情", "states": { "loading": "加载中...", "noData": "暂无数据", diff --git a/messages/zh-TW/dashboard.json b/messages/zh-TW/dashboard.json index 75eda170c..ce0739f82 100644 --- a/messages/zh-TW/dashboard.json +++ b/messages/zh-TW/dashboard.json @@ -410,8 +410,12 @@ "successRate": "成功率（%）", "avgResponseTime": "平均回覆時間", "avgTtfbMs": "平均 TTFB（ms）", - "avgTokensPerSecond": "平均輸出速率" + "avgTokensPerSecond": "平均輸出速率", + "avgCostPerRequest": "平均每次請求成本", + "avgCostPerMillionTokens": "平均每百萬 Token 成本" }, + "expandModelStats": "展開模型詳情", + "collapseModelStats": "收起模型詳情", "states": { "loading": "載入中...", "noData": "暫無資料", diff --git a/src/app/[locale]/dashboard/leaderboard/_components/leaderboard-table.tsx b/src/app/[locale]/dashboard/leaderboard/_components/leaderboard-table.tsx index 292137ffe..d161bd537 100644 --- a/src/app/[locale]/dashboard/leaderboard/_components/leaderboard-table.tsx +++ b/src/app/[locale]/dashboard/leaderboard/_components/leaderboard-table.tsx @@ -1,8 +1,17 @@ "use client"; -import { ArrowDown, ArrowUp, ArrowUpDown, Award, Medal, Trophy } from "lucide-react"; +import { + ArrowDown, + ArrowUp, + ArrowUpDown, + Award, + ChevronDown, + ChevronRight, + Medal, + Trophy, +} from "lucide-react"; import { useTranslations } from "next-intl"; -import { useMemo, useState } from "react"; +import { Fragment, useMemo, useState } from "react"; import { Badge } from "@/components/ui/badge"; import { Card, CardContent } from "@/components/ui/card"; import { @@ -32,6 +41,7 @@ interface LeaderboardTableProps { period: LeaderboardPeriod; columns: ColumnDef[]; // 不包含"排名"列，组件会自动添加 getRowKey?: (row: T, index: number) => string | number; + renderExpandedContent?: (row: T, index: number) => React.ReactNode | null; } export function LeaderboardTable({ @@ -39,6 +49,7 @@ export function LeaderboardTable({ period, columns, getRowKey, + renderExpandedContent, }: LeaderboardTableProps) { const t = useTranslations("dashboard.leaderboard"); @@ -46,6 +57,20 @@ export function LeaderboardTable({ const [sortKey, setSortKey] = useState(null); const [sortDirection, setSortDirection] = useState(null); + // 展开行状态 + const [expandedRows, setExpandedRows] = useState>(new Set()); + const toggleRow = (key: string | number) => { + setExpandedRows((prev) => { + const next = new Set(prev); + if (next.has(key)) { + next.delete(key); + } else { + next.add(key); + } + return next; + }); + }; + // 判断列是否需要加粗 const getShouldBold = (col: ColumnDef) => { const isActiveSortColumn = sortKey === col.sortKey && sortDirection !== null; @@ -204,22 +229,53 @@ export function LeaderboardTable({ const rank = index + 1; const isTopThree = rank <= 3; const rowKey = getRowKey ? (getRowKey(row, index) ?? index) : index; + const hasExpandable = renderExpandedContent != null; + const expandedContent = hasExpandable ? renderExpandedContent(row, index) : null; + const isExpanded = expandedRows.has(rowKey); return ( - - {getRankBadge(rank)} - {columns.map((col, idx) => { - const shouldBold = getShouldBold(col); - return ( - - {col.cell(row, index)} + + toggleRow(rowKey) : undefined + } + > + +
+ {hasExpandable && expandedContent ? ( + isExpanded ? ( + + ) : ( + + ) + ) : null} + {getRankBadge(rank)} +
+ + {columns.map((col, idx) => { + const shouldBold = getShouldBold(col); + return ( + + {col.cell(row, index)} + + ); + })} + + {isExpanded && expandedContent && ( + + + {expandedContent} - ); - })} - + + )} + ); })} diff --git a/src/app/[locale]/dashboard/leaderboard/_components/leaderboard-view.tsx b/src/app/[locale]/dashboard/leaderboard/_components/leaderboard-view.tsx index 5b4c0463e..ccc13eb22 100644 --- a/src/app/[locale]/dashboard/leaderboard/_components/leaderboard-view.tsx +++ b/src/app/[locale]/dashboard/leaderboard/_components/leaderboard-view.tsx @@ -14,6 +14,7 @@ import type { DateRangeParams, LeaderboardEntry, LeaderboardPeriod, + ModelCacheHitStat, ModelLeaderboardEntry, ProviderCacheHitRateLeaderboardEntry, ProviderLeaderboardEntry, @@ -28,7 +29,11 @@ interface LeaderboardViewProps { type LeaderboardScope = "user" | "provider" | "providerCacheHitRate" | "model"; type UserEntry = LeaderboardEntry & { totalCostFormatted?: string }; -type ProviderEntry = ProviderLeaderboardEntry & { totalCostFormatted?: string }; +type ProviderEntry = ProviderLeaderboardEntry & { + totalCostFormatted?: string; + avgCostPerRequestFormatted?: string | null; + avgCostPerMillionTokensFormatted?: string | null; +}; type ProviderCacheHitRateEntry = ProviderCacheHitRateLeaderboardEntry; type ModelEntry = ModelLeaderboardEntry & { totalCostFormatted?: string }; type AnyEntry = UserEntry | ProviderEntry | ProviderCacheHitRateEntry | ModelEntry; @@ -163,7 +168,7 @@ export function LeaderboardView({ isAdmin }: LeaderboardViewProps) { scope === "user" ? 5 : scope === "provider" - ? 8 + ? 10 : scope === "providerCacheHitRate" ? 8 : scope === "model" @@ -265,6 +270,28 @@ export function LeaderboardView({ isAdmin }: LeaderboardViewProps) { sortKey: "avgTokensPerSecond", getValue: (row) => (row as ProviderEntry).avgTokensPerSecond ?? 0, }, + { + header: t("columns.avgCostPerRequest"), + className: "text-right font-mono", + cell: (row) => { + const r = row as ProviderEntry; + if (r.avgCostPerRequest == null) return "-"; + return r.avgCostPerRequestFormatted ?? r.avgCostPerRequest.toFixed(4); + }, + sortKey: "avgCostPerRequest", + getValue: (row) => (row as ProviderEntry).avgCostPerRequest ?? 0, + }, + { + header: t("columns.avgCostPerMillionTokens"), + className: "text-right font-mono", + cell: (row) => { + const r = row as ProviderEntry; + if (r.avgCostPerMillionTokens == null) return "-"; + return r.avgCostPerMillionTokensFormatted ?? r.avgCostPerMillionTokens.toFixed(2); + }, + sortKey: "avgCostPerMillionTokens", + getValue: (row) => (row as ProviderEntry).avgCostPerMillionTokens ?? 0, + }, ]; const providerCacheHitRateColumns: ColumnDef[] = [ @@ -484,7 +511,70 @@ export function LeaderboardView({ isAdmin }: LeaderboardViewProps) { ) : ( - + { + const entry = row as ProviderCacheHitRateEntry & { + modelStats?: ModelCacheHitStat[]; + }; + if (!entry.modelStats || entry.modelStats.length === 0) return null; + return ( +
+
+ {t("expandModelStats")} +
+ + + + + + + + + + + + {entry.modelStats.map((ms) => { + const rate = (ms.cacheHitRate ?? 0) * 100; + const colorClass = + rate >= 85 + ? "text-green-600 dark:text-green-400" + : rate >= 60 + ? "text-yellow-600 dark:text-yellow-400" + : "text-orange-600 dark:text-orange-400"; + return ( + + + + + + + + ); + })} + +
{t("columns.model")}{t("columns.requests")} + {t("columns.cacheReadTokens")} + {t("columns.totalTokens")}{t("columns.cacheHitRate")}
{ms.model} + {ms.totalRequests.toLocaleString()} + + {formatTokenAmount(ms.cacheReadTokens)} + + {formatTokenAmount(ms.totalInputTokens)} + + {rate.toFixed(1)}% +
+
+ ); + } + : undefined + } + /> )} diff --git a/src/app/api/leaderboard/route.ts b/src/app/api/leaderboard/route.ts index 4d1ac419d..e703a1e71 100644 --- a/src/app/api/leaderboard/route.ts +++ b/src/app/api/leaderboard/route.ts @@ -160,17 +160,41 @@ export async function GET(request: NextRequest) { totalCostFormatted: formatCurrency(entry.totalCost, systemSettings.currencyDisplay), }; - if (typeof (entry as { cacheCreationCost?: unknown }).cacheCreationCost === "number") { - return { - ...base, - cacheCreationCostFormatted: formatCurrency( - (entry as { cacheCreationCost: number }).cacheCreationCost, - systemSettings.currencyDisplay - ), - }; - } + // Provider scope: add avgCost formatted fields + const typedEntry = entry as { + avgCostPerRequest?: number | null; + avgCostPerMillionTokens?: number | null; + cacheCreationCost?: number; + }; - return base; + const providerFields = + typeof typedEntry.avgCostPerRequest !== "undefined" + ? { + avgCostPerRequestFormatted: + typedEntry.avgCostPerRequest != null + ? formatCurrency(typedEntry.avgCostPerRequest, systemSettings.currencyDisplay) + : null, + avgCostPerMillionTokensFormatted: + typedEntry.avgCostPerMillionTokens != null + ? formatCurrency( + typedEntry.avgCostPerMillionTokens, + systemSettings.currencyDisplay + ) + : null, + } + : {}; + + const cacheFields = + typeof typedEntry.cacheCreationCost === "number" + ? { + cacheCreationCostFormatted: formatCurrency( + typedEntry.cacheCreationCost, + systemSettings.currencyDisplay + ), + } + : {}; + + return { ...base, ...providerFields, ...cacheFields }; }); logger.info("Leaderboard API: Access granted", { diff --git a/src/repository/leaderboard.ts b/src/repository/leaderboard.ts index d173dfeff..b9b578fa6 100644 --- a/src/repository/leaderboard.ts +++ b/src/repository/leaderboard.ts @@ -41,6 +41,19 @@ export interface ProviderLeaderboardEntry { successRate: number; // 0-1 之间的小数，UI 层负责格式化为百分比 avgTtfbMs: number; // 毫秒 avgTokensPerSecond: number; // tok/s（仅统计流式且可计算的请求） + avgCostPerRequest: number | null; // totalCost / totalRequests, null when totalRequests === 0 + avgCostPerMillionTokens: number | null; // totalCost * 1_000_000 / totalTokens, null when totalTokens === 0 +} + +/** + * 供应商缓存命中率 - 模型级统计 + */ +export interface ModelCacheHitStat { + model: string; + totalRequests: number; + cacheReadTokens: number; + totalInputTokens: number; + cacheHitRate: number; // 0-1 } /** @@ -58,6 +71,7 @@ export interface ProviderCacheHitRateLeaderboardEntry { /** @deprecated Use totalInputTokens instead */ totalTokens: number; cacheHitRate: number; // 0-1 之间的小数，UI 层负责格式化为百分比 + modelStats: ModelCacheHitStat[]; } /** @@ -413,16 +427,23 @@ async function findProviderLeaderboardWithTimezone( .groupBy(messageRequest.providerId, providers.name) .orderBy(desc(sql`sum(${messageRequest.costUsd})`)); - return rankings.map((entry) => ({ - providerId: entry.providerId, - providerName: entry.providerName, - totalRequests: entry.totalRequests, - totalCost: parseFloat(entry.totalCost), - totalTokens: entry.totalTokens, - successRate: entry.successRate ?? 0, - avgTtfbMs: entry.avgTtfbMs ?? 0, - avgTokensPerSecond: entry.avgTokensPerSecond ?? 0, - })); + return rankings.map((entry) => { + const totalCost = parseFloat(entry.totalCost); + const totalRequests = entry.totalRequests; + const totalTokens = entry.totalTokens; + return { + providerId: entry.providerId, + providerName: entry.providerName, + totalRequests, + totalCost, + totalTokens, + successRate: entry.successRate ?? 0, + avgTtfbMs: entry.avgTtfbMs ?? 0, + avgTokensPerSecond: entry.avgTokensPerSecond ?? 0, + avgCostPerRequest: totalRequests > 0 ? totalCost / totalRequests : null, + avgCostPerMillionTokens: totalTokens > 0 ? (totalCost * 1_000_000) / totalTokens : null, + }; + }); } /** @@ -488,6 +509,56 @@ async function findProviderCacheHitRateLeaderboardWithTimezone( .groupBy(messageRequest.providerId, providers.name) .orderBy(desc(cacheHitRateExpr), desc(sql`count(*)`)); + // Model-level cache hit breakdown per provider + const systemSettings = await getSystemSettings(); + const billingModelSource = systemSettings.billingModelSource; + const modelField = + billingModelSource === "original" + ? sql`COALESCE(${messageRequest.originalModel}, ${messageRequest.model})` + : sql`COALESCE(${messageRequest.model}, ${messageRequest.originalModel})`; + + const modelTotalInput = sql`COALESCE(sum(${totalInputTokensExpr})::double precision, 0::double precision)`; + const modelCacheRead = sql`COALESCE(sum(COALESCE(${messageRequest.cacheReadInputTokens}, 0))::double precision, 0::double precision)`; + const modelCacheHitRate = sql`COALESCE( + ${modelCacheRead} / NULLIF(${modelTotalInput}, 0::double precision), + 0::double precision + )`; + + const modelRows = await db + .select({ + providerId: messageRequest.providerId, + model: modelField, + totalRequests: sql`count(*)::double precision`, + cacheReadTokens: modelCacheRead, + totalInputTokens: modelTotalInput, + cacheHitRate: modelCacheHitRate, + }) + .from(messageRequest) + .innerJoin( + providers, + and(sql`${messageRequest.providerId} = ${providers.id}`, isNull(providers.deletedAt)) + ) + .where( + and(...whereConditions.filter((c): c is NonNullable<(typeof whereConditions)[number]> => !!c)) + ) + .groupBy(messageRequest.providerId, modelField) + .orderBy(desc(modelCacheHitRate), desc(sql`count(*)`)); + + // Group model stats by providerId + const modelStatsByProvider = new Map(); + for (const row of modelRows) { + if (!row.model || row.model.trim() === "") continue; + const stats = modelStatsByProvider.get(row.providerId) ?? []; + stats.push({ + model: row.model, + totalRequests: row.totalRequests, + cacheReadTokens: row.cacheReadTokens, + totalInputTokens: row.totalInputTokens, + cacheHitRate: Math.min(Math.max(row.cacheHitRate ?? 0, 0), 1), + }); + modelStatsByProvider.set(row.providerId, stats); + } + return rankings.map((entry) => ({ providerId: entry.providerId, providerName: entry.providerName, @@ -498,6 +569,7 @@ async function findProviderCacheHitRateLeaderboardWithTimezone( totalInputTokens: entry.totalInputTokens, totalTokens: entry.totalInputTokens, // deprecated, for backward compatibility cacheHitRate: Math.min(Math.max(entry.cacheHitRate ?? 0, 0), 1), + modelStats: modelStatsByProvider.get(entry.providerId) ?? [], })); } diff --git a/tests/unit/api/leaderboard-route.test.ts b/tests/unit/api/leaderboard-route.test.ts index ce37a4823..a4d9385a8 100644 --- a/tests/unit/api/leaderboard-route.test.ts +++ b/tests/unit/api/leaderboard-route.test.ts @@ -83,4 +83,119 @@ describe("GET /api/leaderboard", () => { expect(options.userTags).toBeUndefined(); expect(options.userGroups).toBeUndefined(); }); + + describe("additive provider fields", () => { + it("includes avgCostPerRequest and avgCostPerMillionTokens in provider scope response", async () => { + mocks.getSession.mockResolvedValue({ user: { id: 1, name: "u", role: "admin" } }); + mocks.getLeaderboardWithCache.mockResolvedValue([ + { + providerId: 1, + providerName: "test-provider", + totalRequests: 100, + totalCost: 5.0, + totalTokens: 500000, + successRate: 0.95, + avgTtfbMs: 200, + avgTokensPerSecond: 50, + avgCostPerRequest: 0.05, + avgCostPerMillionTokens: 10.0, + }, + ]); + + const { GET } = await import("@/app/api/leaderboard/route"); + const url = "http://localhost/api/leaderboard?scope=provider&period=daily"; + const response = await GET({ nextUrl: new URL(url) } as any); + const body = await response.json(); + + expect(response.status).toBe(200); + expect(body).toHaveLength(1); + + const entry = body[0]; + // Additive fields must be present + expect(entry).toHaveProperty("avgCostPerRequest", 0.05); + expect(entry).toHaveProperty("avgCostPerMillionTokens", 10.0); + // Formatted variants should exist + expect(entry).toHaveProperty("avgCostPerRequestFormatted"); + expect(entry).toHaveProperty("avgCostPerMillionTokensFormatted"); + // Existing fields must still be present + expect(entry).toHaveProperty("totalCostFormatted"); + expect(entry).toHaveProperty("providerId", 1); + expect(entry).toHaveProperty("providerName", "test-provider"); + }); + + it("formats null avgCost fields without error", async () => { + mocks.getSession.mockResolvedValue({ user: { id: 1, name: "u", role: "admin" } }); + mocks.getLeaderboardWithCache.mockResolvedValue([ + { + providerId: 2, + providerName: "zero-provider", + totalRequests: 0, + totalCost: 0, + totalTokens: 0, + successRate: 0, + avgTtfbMs: 0, + avgTokensPerSecond: 0, + avgCostPerRequest: null, + avgCostPerMillionTokens: null, + }, + ]); + + const { GET } = await import("@/app/api/leaderboard/route"); + const url = "http://localhost/api/leaderboard?scope=provider&period=daily"; + const response = await GET({ nextUrl: new URL(url) } as any); + const body = await response.json(); + + expect(response.status).toBe(200); + const entry = body[0]; + expect(entry.avgCostPerRequest).toBeNull(); + expect(entry.avgCostPerMillionTokens).toBeNull(); + }); + + it("includes modelStats in providerCacheHitRate scope response", async () => { + mocks.getSession.mockResolvedValue({ user: { id: 1, name: "u", role: "admin" } }); + mocks.getLeaderboardWithCache.mockResolvedValue([ + { + providerId: 1, + providerName: "cache-provider", + totalRequests: 50, + cacheReadTokens: 10000, + totalCost: 2.5, + cacheCreationCost: 1.0, + totalInputTokens: 20000, + totalTokens: 20000, + cacheHitRate: 0.5, + modelStats: [ + { + model: "claude-3-opus", + totalRequests: 30, + cacheReadTokens: 8000, + totalInputTokens: 15000, + cacheHitRate: 0.53, + }, + { + model: "claude-3-sonnet", + totalRequests: 20, + cacheReadTokens: 2000, + totalInputTokens: 5000, + cacheHitRate: 0.4, + }, + ], + }, + ]); + + const { GET } = await import("@/app/api/leaderboard/route"); + const url = "http://localhost/api/leaderboard?scope=providerCacheHitRate&period=daily"; + const response = await GET({ nextUrl: new URL(url) } as any); + const body = await response.json(); + + expect(response.status).toBe(200); + expect(body).toHaveLength(1); + + const entry = body[0]; + expect(entry).toHaveProperty("modelStats"); + expect(entry.modelStats).toHaveLength(2); + expect(entry.modelStats[0]).toHaveProperty("model", "claude-3-opus"); + expect(entry.modelStats[0]).toHaveProperty("cacheHitRate", 0.53); + }); + }); }); diff --git a/tests/unit/repository/leaderboard-provider-metrics.test.ts b/tests/unit/repository/leaderboard-provider-metrics.test.ts new file mode 100644 index 000000000..14d05177b --- /dev/null +++ b/tests/unit/repository/leaderboard-provider-metrics.test.ts @@ -0,0 +1,473 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +/** + * GREEN tests for provider leaderboard average cost metrics and cache-hit model breakdown. + * + * These tests verify the semantic contracts: + * - avgCostPerRequest = totalCost / totalRequests (null when totalRequests === 0) + * - avgCostPerMillionTokens = totalCost * 1_000_000 / totalTokens (null when totalTokens === 0) + * - ProviderCacheHitRateLeaderboardEntry.modelStats: nested model-level breakdown + */ + +const createChainMock = (resolvedData: unknown[]) => ({ + from: vi.fn().mockReturnThis(), + innerJoin: vi.fn().mockReturnThis(), + where: vi.fn().mockReturnThis(), + groupBy: vi.fn().mockReturnThis(), + orderBy: vi.fn().mockResolvedValue(resolvedData), +}); + +// Track select calls to return different chains for different queries +let selectCallIndex = 0; +let chainMocks: ReturnType[] = []; + +const mockSelect = vi.fn(() => { + const chain = chainMocks[selectCallIndex] ?? createChainMock([]); + selectCallIndex++; + return chain; +}); + +const mocks = vi.hoisted(() => ({ + resolveSystemTimezone: vi.fn(), + getSystemSettings: vi.fn(), +})); + +vi.mock("@/drizzle/db", () => ({ + db: { + select: (...args: unknown[]) => mockSelect(...args), + }, +})); + +vi.mock("@/drizzle/schema", () => ({ + messageRequest: { + deletedAt: "deletedAt", + providerId: "providerId", + userId: "userId", + costUsd: "costUsd", + inputTokens: "inputTokens", + outputTokens: "outputTokens", + cacheCreationInputTokens: "cacheCreationInputTokens", + cacheReadInputTokens: "cacheReadInputTokens", + errorMessage: "errorMessage", + blockedBy: "blockedBy", + createdAt: "createdAt", + ttfbMs: "ttfbMs", + durationMs: "durationMs", + model: "model", + originalModel: "originalModel", + }, + providers: { + id: "id", + name: "name", + deletedAt: "deletedAt", + providerType: "providerType", + }, + users: {}, +})); + +vi.mock("@/lib/utils/timezone", () => ({ + resolveSystemTimezone: mocks.resolveSystemTimezone, +})); + +vi.mock("@/repository/system-config", () => ({ + getSystemSettings: mocks.getSystemSettings, +})); + +describe("Provider Leaderboard Average Cost Metrics", () => { + beforeEach(() => { + vi.resetModules(); + selectCallIndex = 0; + chainMocks = []; + mockSelect.mockClear(); + mocks.resolveSystemTimezone.mockResolvedValue("UTC"); + mocks.getSystemSettings.mockResolvedValue({ billingModelSource: "redirected" }); + }); + + it("computes avgCostPerRequest = totalCost / totalRequests for valid denominators", async () => { + chainMocks = [ + createChainMock([ + { + providerId: 1, + providerName: "test-provider", + totalRequests: 100, + totalCost: "5.0", + totalTokens: 500000, + successRate: 0.95, + avgTtfbMs: 200, + avgTokensPerSecond: 50, + }, + ]), + ]; + + const { findDailyProviderLeaderboard } = await import("@/repository/leaderboard"); + const result = await findDailyProviderLeaderboard(); + + expect(result).toHaveLength(1); + const entry = result[0]; + expect(entry).toHaveProperty("avgCostPerRequest"); + expect(entry.avgCostPerRequest).toBeCloseTo(5.0 / 100); + + type HasAvgCostPerRequest = { avgCostPerRequest: number | null }; + const _typeCheck: HasAvgCostPerRequest = {} as Awaited< + ReturnType + >[number]; + expect(_typeCheck).toBeDefined(); + }); + + it("computes avgCostPerMillionTokens = totalCost * 1_000_000 / totalTokens for valid denominators", async () => { + chainMocks = [ + createChainMock([ + { + providerId: 1, + providerName: "test-provider", + totalRequests: 100, + totalCost: "5.0", + totalTokens: 500000, + successRate: 0.95, + avgTtfbMs: 200, + avgTokensPerSecond: 50, + }, + ]), + ]; + + const { findDailyProviderLeaderboard } = await import("@/repository/leaderboard"); + const result = await findDailyProviderLeaderboard(); + + expect(result).toHaveLength(1); + const entry = result[0]; + expect(entry).toHaveProperty("avgCostPerMillionTokens"); + expect(entry.avgCostPerMillionTokens).toBeCloseTo((5.0 * 1_000_000) / 500000); + }); + + it("returns null for avgCostPerRequest when totalRequests is 0", async () => { + chainMocks = [ + createChainMock([ + { + providerId: 1, + providerName: "zero-provider", + totalRequests: 0, + totalCost: "0", + totalTokens: 0, + successRate: 0, + avgTtfbMs: 0, + avgTokensPerSecond: 0, + }, + ]), + ]; + + const { findDailyProviderLeaderboard } = await import("@/repository/leaderboard"); + const result = await findDailyProviderLeaderboard(); + + expect(result).toHaveLength(1); + expect(result[0].avgCostPerRequest).toBeNull(); + }); + + it("returns null for avgCostPerMillionTokens when totalTokens is 0", async () => { + chainMocks = [ + createChainMock([ + { + providerId: 1, + providerName: "zero-provider", + totalRequests: 5, + totalCost: "1.0", + totalTokens: 0, + successRate: 0, + avgTtfbMs: 0, + avgTokensPerSecond: 0, + }, + ]), + ]; + + const { findDailyProviderLeaderboard } = await import("@/repository/leaderboard"); + const result = await findDailyProviderLeaderboard(); + + expect(result).toHaveLength(1); + expect(result[0].avgCostPerMillionTokens).toBeNull(); + }); + + it("preserves provider sort order by totalCost descending", async () => { + chainMocks = [ + createChainMock([ + { + providerId: 1, + providerName: "expensive", + totalRequests: 100, + totalCost: "10.0", + totalTokens: 500000, + successRate: 0.95, + avgTtfbMs: 200, + avgTokensPerSecond: 50, + }, + { + providerId: 2, + providerName: "cheap", + totalRequests: 50, + totalCost: "2.0", + totalTokens: 100000, + successRate: 0.9, + avgTtfbMs: 300, + avgTokensPerSecond: 40, + }, + ]), + ]; + + const { findDailyProviderLeaderboard } = await import("@/repository/leaderboard"); + const result = await findDailyProviderLeaderboard(); + + expect(result).toHaveLength(2); + expect(result[0].totalCost).toBeGreaterThanOrEqual(result[1].totalCost); + }); +}); + +describe("Provider Cache Hit Rate Model Breakdown", () => { + beforeEach(() => { + vi.resetModules(); + selectCallIndex = 0; + chainMocks = []; + mockSelect.mockClear(); + mocks.resolveSystemTimezone.mockResolvedValue("UTC"); + mocks.getSystemSettings.mockResolvedValue({ billingModelSource: "redirected" }); + }); + + it("includes modelStats field on cache-hit leaderboard entries", async () => { + chainMocks = [ + createChainMock([ + { + providerId: 1, + providerName: "cache-provider", + totalRequests: 50, + totalCost: "2.5", + cacheReadTokens: 10000, + cacheCreationCost: "1.0", + totalInputTokens: 20000, + cacheHitRate: 0.5, + }, + ]), + createChainMock([ + { + providerId: 1, + model: "claude-3-opus", + totalRequests: 30, + cacheReadTokens: 8000, + totalInputTokens: 15000, + cacheHitRate: 0.53, + }, + { + providerId: 1, + model: "claude-3-sonnet", + totalRequests: 20, + cacheReadTokens: 2000, + totalInputTokens: 5000, + cacheHitRate: 0.4, + }, + ]), + ]; + + const { findDailyProviderCacheHitRateLeaderboard } = await import("@/repository/leaderboard"); + const result = await findDailyProviderCacheHitRateLeaderboard(); + + expect(result).toHaveLength(1); + const entry = result[0]; + expect(entry).toHaveProperty("modelStats"); + expect(Array.isArray(entry.modelStats)).toBe(true); + expect(entry.modelStats).toHaveLength(2); + expect(entry.modelStats[0].model).toBe("claude-3-opus"); + }); + + it("provider cache hit ranking sort stability preserved after adding modelStats", async () => { + chainMocks = [ + createChainMock([ + { + providerId: 1, + providerName: "high-cache", + totalRequests: 50, + totalCost: "2.5", + cacheReadTokens: 15000, + cacheCreationCost: "1.0", + totalInputTokens: 20000, + cacheHitRate: 0.75, + }, + { + providerId: 2, + providerName: "low-cache", + totalRequests: 30, + totalCost: "1.0", + cacheReadTokens: 2000, + cacheCreationCost: "0.5", + totalInputTokens: 10000, + cacheHitRate: 0.2, + }, + ]), + createChainMock([]), + ]; + + const { findDailyProviderCacheHitRateLeaderboard } = await import("@/repository/leaderboard"); + const result = await findDailyProviderCacheHitRateLeaderboard(); + + expect(result).toHaveLength(2); + expect(result[0].cacheHitRate).toBeGreaterThanOrEqual(result[1].cacheHitRate); + }); + + it("model breakdown excludes empty model names and has deterministic order", async () => { + chainMocks = [ + createChainMock([ + { + providerId: 1, + providerName: "provider-a", + totalRequests: 50, + totalCost: "2.5", + cacheReadTokens: 10000, + cacheCreationCost: "1.0", + totalInputTokens: 20000, + cacheHitRate: 0.5, + }, + ]), + createChainMock([ + { + providerId: 1, + model: "claude-3-opus", + totalRequests: 30, + cacheReadTokens: 8000, + totalInputTokens: 15000, + cacheHitRate: 0.53, + }, + { + providerId: 1, + model: "", + totalRequests: 5, + cacheReadTokens: 100, + totalInputTokens: 500, + cacheHitRate: 0.2, + }, + { + providerId: 1, + model: "claude-3-sonnet", + totalRequests: 15, + cacheReadTokens: 1900, + totalInputTokens: 4500, + cacheHitRate: 0.42, + }, + ]), + ]; + + const { findDailyProviderCacheHitRateLeaderboard } = await import("@/repository/leaderboard"); + const result = await findDailyProviderCacheHitRateLeaderboard(); + + expect(result).toHaveLength(1); + const entry = result[0]; + // Empty model names must be excluded (only 2 valid models) + expect(entry.modelStats).toHaveLength(2); + for (const ms of entry.modelStats) { + expect(ms.model).toBeTruthy(); + expect(ms.model.trim()).not.toBe(""); + } + // Deterministic order: cacheHitRate desc (0.53 > 0.42) + expect(entry.modelStats[0].cacheHitRate).toBeGreaterThanOrEqual( + entry.modelStats[entry.modelStats.length - 1].cacheHitRate + ); + }); + + it("preserves all existing provider-level fields unchanged", async () => { + chainMocks = [ + createChainMock([ + { + providerId: 1, + providerName: "full-provider", + totalRequests: 50, + totalCost: "2.5", + cacheReadTokens: 10000, + cacheCreationCost: "1.0", + totalInputTokens: 20000, + cacheHitRate: 0.5, + }, + ]), + createChainMock([]), + ]; + + const { findDailyProviderCacheHitRateLeaderboard } = await import("@/repository/leaderboard"); + const result = await findDailyProviderCacheHitRateLeaderboard(); + + expect(result).toHaveLength(1); + const entry = result[0]; + expect(entry).toHaveProperty("providerId", 1); + expect(entry).toHaveProperty("providerName", "full-provider"); + expect(entry).toHaveProperty("totalRequests", 50); + expect(entry).toHaveProperty("cacheReadTokens", 10000); + expect(entry).toHaveProperty("totalCost", 2.5); + expect(entry).toHaveProperty("cacheCreationCost", 1.0); + expect(entry).toHaveProperty("totalInputTokens", 20000); + expect(entry).toHaveProperty("cacheHitRate", 0.5); + expect(entry).toHaveProperty("modelStats"); + }); + + it("groups model stats correctly across multiple providers", async () => { + chainMocks = [ + createChainMock([ + { + providerId: 1, + providerName: "provider-alpha", + totalRequests: 50, + totalCost: "2.5", + cacheReadTokens: 10000, + cacheCreationCost: "1.0", + totalInputTokens: 20000, + cacheHitRate: 0.5, + }, + { + providerId: 2, + providerName: "provider-beta", + totalRequests: 30, + totalCost: "1.0", + cacheReadTokens: 5000, + cacheCreationCost: "0.5", + totalInputTokens: 10000, + cacheHitRate: 0.5, + }, + ]), + createChainMock([ + { + providerId: 1, + model: "model-a", + totalRequests: 30, + cacheReadTokens: 6000, + totalInputTokens: 12000, + cacheHitRate: 0.5, + }, + { + providerId: 1, + model: "model-b", + totalRequests: 20, + cacheReadTokens: 4000, + totalInputTokens: 8000, + cacheHitRate: 0.5, + }, + { + providerId: 2, + model: "model-c", + totalRequests: 30, + cacheReadTokens: 5000, + totalInputTokens: 10000, + cacheHitRate: 0.5, + }, + ]), + ]; + + const { findDailyProviderCacheHitRateLeaderboard } = await import("@/repository/leaderboard"); + const result = await findDailyProviderCacheHitRateLeaderboard(); + + expect(result).toHaveLength(2); + + // Provider 1 should have 2 model stats + const p1 = result.find((r) => r.providerId === 1); + expect(p1).toBeDefined(); + expect(p1!.modelStats).toHaveLength(2); + const p1Models = p1!.modelStats.map((m) => m.model).sort(); + expect(p1Models).toEqual(["model-a", "model-b"]); + + // Provider 2 should have 1 model stat + const p2 = result.find((r) => r.providerId === 2); + expect(p2).toBeDefined(); + expect(p2!.modelStats).toHaveLength(1); + expect(p2!.modelStats[0].model).toBe("model-c"); + }); +});