From 0d10896495448b8dff16a993813a5c445699e319 Mon Sep 17 00:00:00 2001 From: ding113 Date: Wed, 28 Jan 2026 19:28:14 +0800 Subject: [PATCH 1/2] feat(providers): recluster vendors by host:port when website_url empty - Add computeVendorKey helper with host:port support for IP-based providers - When website_url is empty, use host:port as vendor key (different ports = different vendors) - Support IPv6 addresses with [ipv6]:port format - Use protocol default ports (http=80, https=443) when port not specified - Add reclusterProviderVendors action with preview/apply mode - Add ReclusterVendorsDialog UI component - Add i18n support for 5 languages (zh-CN, zh-TW, en, ja, ru) - Existing behavior unchanged when website_url is present (hostname only) Co-Authored-By: Claude Opus 4.5 --- messages/en/settings/index.ts | 2 + messages/en/settings/providers/recluster.json | 16 + messages/ja/settings/index.ts | 2 + messages/ja/settings/providers/recluster.json | 16 + messages/ru/settings/index.ts | 2 + messages/ru/settings/providers/recluster.json | 16 + messages/zh-CN/settings/index.ts | 2 + .../zh-CN/settings/providers/recluster.json | 16 + messages/zh-TW/settings/index.ts | 2 + .../zh-TW/settings/providers/recluster.json | 16 + src/actions/providers.ts | 177 ++++++++++- src/app/[locale]/dashboard/providers/page.tsx | 2 + .../_components/recluster-vendors-dialog.tsx | 259 +++++++++++++++ src/app/[locale]/settings/providers/page.tsx | 2 + src/repository/provider-endpoints.ts | 90 +++++- .../unit/actions/providers-recluster.test.ts | 298 ++++++++++++++++++ .../provider-endpoints-vendor-key.test.ts | 206 ++++++++++++ 17 files changed, 1117 insertions(+), 7 deletions(-) create mode 100644 messages/en/settings/providers/recluster.json create mode 100644 messages/ja/settings/providers/recluster.json create mode 100644 messages/ru/settings/providers/recluster.json create mode 100644 messages/zh-CN/settings/providers/recluster.json create mode 100644 messages/zh-TW/settings/providers/recluster.json create mode 100644 src/app/[locale]/settings/providers/_components/recluster-vendors-dialog.tsx create mode 100644 tests/unit/actions/providers-recluster.test.ts create mode 100644 tests/unit/repository/provider-endpoints-vendor-key.test.ts diff --git a/messages/en/settings/index.ts b/messages/en/settings/index.ts index 47a6b5424..d014d0dba 100644 --- a/messages/en/settings/index.ts +++ b/messages/en/settings/index.ts @@ -18,6 +18,7 @@ import providersFilter from "./providers/filter.json"; import providersGuide from "./providers/guide.json"; import providersInlineEdit from "./providers/inlineEdit.json"; import providersList from "./providers/list.json"; +import providersRecluster from "./providers/recluster.json"; import providersSchedulingDialog from "./providers/schedulingDialog.json"; import providersSearch from "./providers/search.json"; import providersSection from "./providers/section.json"; @@ -81,6 +82,7 @@ const providers = { guide: providersGuide, inlineEdit: providersInlineEdit, list: providersList, + recluster: providersRecluster, schedulingDialog: providersSchedulingDialog, search: providersSearch, section: providersSection, diff --git a/messages/en/settings/providers/recluster.json b/messages/en/settings/providers/recluster.json new file mode 100644 index 000000000..ef137acb7 --- /dev/null +++ b/messages/en/settings/providers/recluster.json @@ -0,0 +1,16 @@ +{ + "button": "Recluster", + "dialogTitle": "Recluster Providers", + "dialogDescription": "Reorganize providers based on updated clustering rules. For providers without a website URL, host:port will be used as the clustering key.", + "providersMoved": "Providers Moved", + "vendorsCreated": "Vendors Created", + "vendorsToDelete": "Vendors to Delete", + "skipped": "Skipped (Invalid URL)", + "providerHeader": "Provider", + "vendorChangeHeader": "Vendor Change", + "noChanges": "No changes needed (already clustered correctly)", + "moreChanges": "{count} more changes...", + "confirm": "Apply Changes", + "success": "Reclustered {count} providers", + "error": "Recluster failed" +} diff --git a/messages/ja/settings/index.ts b/messages/ja/settings/index.ts index 47a6b5424..d014d0dba 100644 --- a/messages/ja/settings/index.ts +++ b/messages/ja/settings/index.ts @@ -18,6 +18,7 @@ import providersFilter from "./providers/filter.json"; import providersGuide from "./providers/guide.json"; import providersInlineEdit from "./providers/inlineEdit.json"; import providersList from "./providers/list.json"; +import providersRecluster from "./providers/recluster.json"; import providersSchedulingDialog from "./providers/schedulingDialog.json"; import providersSearch from "./providers/search.json"; import providersSection from "./providers/section.json"; @@ -81,6 +82,7 @@ const providers = { guide: providersGuide, inlineEdit: providersInlineEdit, list: providersList, + recluster: providersRecluster, schedulingDialog: providersSchedulingDialog, search: providersSearch, section: providersSection, diff --git a/messages/ja/settings/providers/recluster.json b/messages/ja/settings/providers/recluster.json new file mode 100644 index 000000000..429550af8 --- /dev/null +++ b/messages/ja/settings/providers/recluster.json @@ -0,0 +1,16 @@ +{ + "button": "再クラスタ", + "dialogTitle": "プロバイダーの再クラスタリング", + "dialogDescription": "更新されたクラスタリングルールに基づいてプロバイダーを再編成します。ウェブサイトURLが設定されていないプロバイダーには、host:port がクラスタリングキーとして使用されます。", + "providersMoved": "移動したプロバイダー", + "vendorsCreated": "作成されるベンダー", + "vendorsToDelete": "削除されるベンダー", + "skipped": "スキップ (無効なURL)", + "providerHeader": "プロバイダー", + "vendorChangeHeader": "ベンダー変更", + "noChanges": "変更不要 (既に正しくクラスタリング済み)", + "moreChanges": "他 {count} 件の変更...", + "confirm": "変更を適用", + "success": "{count} 件のプロバイダーを再クラスタリングしました", + "error": "再クラスタリングに失敗しました" +} diff --git a/messages/ru/settings/index.ts b/messages/ru/settings/index.ts index 47a6b5424..d014d0dba 100644 --- a/messages/ru/settings/index.ts +++ b/messages/ru/settings/index.ts @@ -18,6 +18,7 @@ import providersFilter from "./providers/filter.json"; import providersGuide from "./providers/guide.json"; import providersInlineEdit from "./providers/inlineEdit.json"; import providersList from "./providers/list.json"; +import providersRecluster from "./providers/recluster.json"; import providersSchedulingDialog from "./providers/schedulingDialog.json"; import providersSearch from "./providers/search.json"; import providersSection from "./providers/section.json"; @@ -81,6 +82,7 @@ const providers = { guide: providersGuide, inlineEdit: providersInlineEdit, list: providersList, + recluster: providersRecluster, schedulingDialog: providersSchedulingDialog, search: providersSearch, section: providersSection, diff --git a/messages/ru/settings/providers/recluster.json b/messages/ru/settings/providers/recluster.json new file mode 100644 index 000000000..56e692266 --- /dev/null +++ b/messages/ru/settings/providers/recluster.json @@ -0,0 +1,16 @@ +{ + "button": "Перегруппировать", + "dialogTitle": "Перегруппировка поставщиков", + "dialogDescription": "Реорганизация поставщиков по обновленным правилам группировки. Для поставщиков без URL сайта в качестве ключа группировки используется host:port.", + "providersMoved": "Перемещено поставщиков", + "vendorsCreated": "Создано групп", + "vendorsToDelete": "Групп к удалению", + "skipped": "Пропущено (неверный URL)", + "providerHeader": "Поставщик", + "vendorChangeHeader": "Изменение группы", + "noChanges": "Изменения не требуются (уже правильно сгруппировано)", + "moreChanges": "Еще {count} изменений...", + "confirm": "Применить изменения", + "success": "Перегруппировано {count} поставщиков", + "error": "Ошибка перегруппировки" +} diff --git a/messages/zh-CN/settings/index.ts b/messages/zh-CN/settings/index.ts index 47a6b5424..d014d0dba 100644 --- a/messages/zh-CN/settings/index.ts +++ b/messages/zh-CN/settings/index.ts @@ -18,6 +18,7 @@ import providersFilter from "./providers/filter.json"; import providersGuide from "./providers/guide.json"; import providersInlineEdit from "./providers/inlineEdit.json"; import providersList from "./providers/list.json"; +import providersRecluster from "./providers/recluster.json"; import providersSchedulingDialog from "./providers/schedulingDialog.json"; import providersSearch from "./providers/search.json"; import providersSection from "./providers/section.json"; @@ -81,6 +82,7 @@ const providers = { guide: providersGuide, inlineEdit: providersInlineEdit, list: providersList, + recluster: providersRecluster, schedulingDialog: providersSchedulingDialog, search: providersSearch, section: providersSection, diff --git a/messages/zh-CN/settings/providers/recluster.json b/messages/zh-CN/settings/providers/recluster.json new file mode 100644 index 000000000..0250bbb7d --- /dev/null +++ b/messages/zh-CN/settings/providers/recluster.json @@ -0,0 +1,16 @@ +{ + "button": "重新分组", + "dialogTitle": "重新分组供应商", + "dialogDescription": "根据更新的分组规则重新组织供应商。对于未设置网站URL的供应商，将使用 host:port 作为分组键。", + "providersMoved": "供应商移动", + "vendorsCreated": "新建分组", + "vendorsToDelete": "待删除分组", + "skipped": "跳过(无效URL)", + "providerHeader": "供应商", + "vendorChangeHeader": "分组变更", + "noChanges": "无需更改（已正确分组）", + "moreChanges": "还有 {count} 条变更...", + "confirm": "应用变更", + "success": "已重新分组 {count} 个供应商", + "error": "重新分组失败" +} diff --git a/messages/zh-TW/settings/index.ts b/messages/zh-TW/settings/index.ts index 47a6b5424..d014d0dba 100644 --- a/messages/zh-TW/settings/index.ts +++ b/messages/zh-TW/settings/index.ts @@ -18,6 +18,7 @@ import providersFilter from "./providers/filter.json"; import providersGuide from "./providers/guide.json"; import providersInlineEdit from "./providers/inlineEdit.json"; import providersList from "./providers/list.json"; +import providersRecluster from "./providers/recluster.json"; import providersSchedulingDialog from "./providers/schedulingDialog.json"; import providersSearch from "./providers/search.json"; import providersSection from "./providers/section.json"; @@ -81,6 +82,7 @@ const providers = { guide: providersGuide, inlineEdit: providersInlineEdit, list: providersList, + recluster: providersRecluster, schedulingDialog: providersSchedulingDialog, search: providersSearch, section: providersSection, diff --git a/messages/zh-TW/settings/providers/recluster.json b/messages/zh-TW/settings/providers/recluster.json new file mode 100644 index 000000000..5aee32ca4 --- /dev/null +++ b/messages/zh-TW/settings/providers/recluster.json @@ -0,0 +1,16 @@ +{ + "button": "重新分組", + "dialogTitle": "重新分組供應商", + "dialogDescription": "根據更新的分組規則重新組織供應商。對於未設定網站URL的供應商，將使用 host:port 作為分組鍵。", + "providersMoved": "供應商移動", + "vendorsCreated": "新建分組", + "vendorsToDelete": "待刪除分組", + "skipped": "跳過(無效URL)", + "providerHeader": "供應商", + "vendorChangeHeader": "分組變更", + "noChanges": "無需更改（已正確分組）", + "moreChanges": "還有 {count} 條變更...", + "confirm": "應用變更", + "success": "已重新分組 {count} 個供應商", + "error": "重新分組失敗" +} diff --git a/src/actions/providers.ts b/src/actions/providers.ts index 69fcc273c..4274ca3d5 100644 --- a/src/actions/providers.ts +++ b/src/actions/providers.ts @@ -1,8 +1,11 @@ "use server"; +import { eq } from "drizzle-orm"; import { GeminiAuth } from "@/app/v1/_lib/gemini/auth"; import { isClientAbortError } from "@/app/v1/_lib/proxy/errors"; import { buildProxyUrl } from "@/app/v1/_lib/url"; +import { db } from "@/drizzle/db"; +import { providers as providersTable } from "@/drizzle/schema"; import { getSession } from "@/lib/auth"; import { publishProviderCacheInvalidation } from "@/lib/cache/provider-cache"; import { @@ -44,7 +47,13 @@ import { updateProvider, updateProviderPrioritiesBatch, } from "@/repository/provider"; -import { tryDeleteProviderVendorIfEmpty } from "@/repository/provider-endpoints"; +import { + backfillProviderEndpointsFromProviders, + computeVendorKey, + findProviderVendorById, + getOrCreateProviderVendorIdFromUrls, + tryDeleteProviderVendorIfEmpty, +} from "@/repository/provider-endpoints"; import type { CacheTtlPreference } from "@/types/cache"; import type { CodexParallelToolCallsPreference, @@ -3548,3 +3557,169 @@ export async function getModelSuggestionsByProviderGroup( return { ok: false, error: "获取模型建议列表失败" }; } } + +// ============================================================================ +// Recluster Provider Vendors +// ============================================================================ + +type ReclusterChange = { + providerId: number; + providerName: string; + oldVendorId: number; + oldVendorDomain: string; + newVendorDomain: string; +}; + +type ReclusterResult = { + preview: { + providersMoved: number; + vendorsCreated: number; + vendorsToDelete: number; + skippedInvalidUrl: number; + }; + changes: ReclusterChange[]; + applied: boolean; +}; + +/** + * Recluster provider vendors based on updated clustering rules. + * When websiteUrl is empty, uses host:port as vendor key instead of just hostname. + * + * @param confirm - false=preview mode (calculate changes only), true=apply mode (execute changes) + */ +export async function reclusterProviderVendors(args: { + confirm: boolean; +}): Promise> { + try { + const session = await getSession(); + if (!session || session.user.role !== "admin") { + return { ok: false, error: "NO_PERMISSION" }; + } + + const allProviders = await findAllProvidersFresh(); + + if (allProviders.length === 0) { + return { + ok: true, + data: { + preview: { + providersMoved: 0, + vendorsCreated: 0, + vendorsToDelete: 0, + skippedInvalidUrl: 0, + }, + changes: [], + applied: args.confirm, + }, + }; + } + + const changes: ReclusterChange[] = []; + const newVendorKeys = new Set(); + const oldVendorIds = new Set(); + let skippedInvalidUrl = 0; + + // Calculate new vendor key for each provider + for (const provider of allProviders) { + const newVendorKey = computeVendorKey({ + providerUrl: provider.url, + websiteUrl: provider.websiteUrl, + }); + + if (!newVendorKey) { + skippedInvalidUrl++; + continue; + } + + // Get current vendor domain + const currentVendor = provider.providerVendorId + ? await findProviderVendorById(provider.providerVendorId) + : null; + const currentDomain = currentVendor?.websiteDomain ?? ""; + + // If key changed, record the change + if (currentDomain !== newVendorKey) { + newVendorKeys.add(newVendorKey); + if (provider.providerVendorId) { + oldVendorIds.add(provider.providerVendorId); + } + changes.push({ + providerId: provider.id, + providerName: provider.name, + oldVendorId: provider.providerVendorId ?? 0, + oldVendorDomain: currentDomain, + newVendorDomain: newVendorKey, + }); + } + } + + const preview = { + providersMoved: changes.length, + vendorsCreated: newVendorKeys.size, + vendorsToDelete: oldVendorIds.size, + skippedInvalidUrl, + }; + + // Preview mode: return without modifying DB + if (!args.confirm) { + return { + ok: true, + data: { + preview, + changes, + applied: false, + }, + }; + } + + // Apply mode: execute changes in transaction + if (changes.length > 0) { + await db.transaction(async (tx) => { + for (const change of changes) { + // Get or create new vendor + const newVendorId = await getOrCreateProviderVendorIdFromUrls({ + providerUrl: allProviders.find((p) => p.id === change.providerId)?.url ?? "", + websiteUrl: allProviders.find((p) => p.id === change.providerId)?.websiteUrl ?? null, + }); + + // Update provider's vendorId + await tx + .update(providersTable) + .set({ providerVendorId: newVendorId, updatedAt: new Date() }) + .where(eq(providersTable.id, change.providerId)); + } + }); + + // Backfill provider_endpoints + await backfillProviderEndpointsFromProviders(); + + // Cleanup empty vendors + for (const oldVendorId of oldVendorIds) { + await tryDeleteProviderVendorIfEmpty(oldVendorId); + } + + // Publish cache invalidation + try { + await publishProviderCacheInvalidation(); + } catch (error) { + logger.warn("reclusterProviderVendors:cache_invalidation_failed", { + changedCount: changes.length, + error: error instanceof Error ? error.message : String(error), + }); + } + } + + return { + ok: true, + data: { + preview, + changes, + applied: true, + }, + }; + } catch (error) { + logger.error("reclusterProviderVendors:error", error); + const message = error instanceof Error ? error.message : "Recluster failed"; + return { ok: false, error: message }; + } +} diff --git a/src/app/[locale]/dashboard/providers/page.tsx b/src/app/[locale]/dashboard/providers/page.tsx index e95f29720..60f2b2777 100644 --- a/src/app/[locale]/dashboard/providers/page.tsx +++ b/src/app/[locale]/dashboard/providers/page.tsx @@ -2,6 +2,7 @@ import { BarChart3 } from "lucide-react"; import { getTranslations } from "next-intl/server"; import { AutoSortPriorityDialog } from "@/app/[locale]/settings/providers/_components/auto-sort-priority-dialog"; import { ProviderManagerLoader } from "@/app/[locale]/settings/providers/_components/provider-manager-loader"; +import { ReclusterVendorsDialog } from "@/app/[locale]/settings/providers/_components/recluster-vendors-dialog"; import { SchedulingRulesDialog } from "@/app/[locale]/settings/providers/_components/scheduling-rules-dialog"; import { Section } from "@/components/section"; import { Button } from "@/components/ui/button"; @@ -48,6 +49,7 @@ export default async function DashboardProvidersPage({ + } diff --git a/src/app/[locale]/settings/providers/_components/recluster-vendors-dialog.tsx b/src/app/[locale]/settings/providers/_components/recluster-vendors-dialog.tsx new file mode 100644 index 000000000..89b4a263c --- /dev/null +++ b/src/app/[locale]/settings/providers/_components/recluster-vendors-dialog.tsx @@ -0,0 +1,259 @@ +"use client"; + +import { useQueryClient } from "@tanstack/react-query"; +import { ArrowRight, FolderGit2, Loader2 } from "lucide-react"; +import { useTranslations } from "next-intl"; +import { useState, useTransition } from "react"; +import { toast } from "sonner"; +import { reclusterProviderVendors } from "@/actions/providers"; +import { Badge } from "@/components/ui/badge"; +import { Button } from "@/components/ui/button"; +import { + Dialog, + DialogContent, + DialogDescription, + DialogFooter, + DialogHeader, + DialogTitle, + DialogTrigger, +} from "@/components/ui/dialog"; +import { + Table, + TableBody, + TableCell, + TableHead, + TableHeader, + TableRow, +} from "@/components/ui/table"; + +type ReclusterChange = { + providerId: number; + providerName: string; + oldVendorId: number; + oldVendorDomain: string; + newVendorDomain: string; +}; + +type ReclusterResult = { + preview: { + providersMoved: number; + vendorsCreated: number; + vendorsToDelete: number; + skippedInvalidUrl: number; + }; + changes: ReclusterChange[]; + applied: boolean; +}; + +const MAX_DISPLAY_CHANGES = 10; + +export function ReclusterVendorsDialog() { + const queryClient = useQueryClient(); + const t = useTranslations("settings.providers.recluster"); + const tCommon = useTranslations("settings.common"); + const tErrors = useTranslations("errors"); + + const [open, setOpen] = useState(false); + const [previewData, setPreviewData] = useState(null); + const [isPending, startTransition] = useTransition(); + const [isApplying, setIsApplying] = useState(false); + + const getActionErrorMessage = (result: { + errorCode?: string; + errorParams?: Record; + error?: string | null; + }): string => { + if (result.errorCode) { + try { + return tErrors(result.errorCode, result.errorParams); + } catch { + return t("error"); + } + } + + if (result.error) { + try { + return tErrors(result.error); + } catch { + return t("error"); + } + } + + return t("error"); + }; + + const handleOpenChange = (isOpen: boolean) => { + setOpen(isOpen); + if (isOpen) { + // Load preview when dialog opens + startTransition(async () => { + try { + const result = await reclusterProviderVendors({ confirm: false }); + if (result.ok) { + setPreviewData(result.data); + } else { + toast.error(getActionErrorMessage(result)); + setOpen(false); + } + } catch (error) { + console.error("reclusterProviderVendors preview failed", error); + toast.error(t("error")); + setOpen(false); + } + }); + } else { + // Clear preview when dialog closes + setPreviewData(null); + } + }; + + const handleApply = async () => { + setIsApplying(true); + try { + const result = await reclusterProviderVendors({ confirm: true }); + if (result.ok) { + toast.success(t("success", { count: result.data.preview.providersMoved })); + queryClient.invalidateQueries({ queryKey: ["providers"] }); + queryClient.invalidateQueries({ queryKey: ["provider-vendors"] }); + queryClient.invalidateQueries({ queryKey: ["provider-endpoints"] }); + setOpen(false); + } else { + toast.error(getActionErrorMessage(result)); + } + } catch (error) { + console.error("reclusterProviderVendors apply failed", error); + toast.error(t("error")); + } finally { + setIsApplying(false); + } + }; + + const hasChanges = previewData && previewData.preview.providersMoved > 0; + const displayedChanges = previewData?.changes.slice(0, MAX_DISPLAY_CHANGES) ?? []; + const remainingCount = (previewData?.changes.length ?? 0) - MAX_DISPLAY_CHANGES; + + return ( + + + + + + + {t("dialogTitle")} + {t("dialogDescription")} + + +
+ {isPending ? ( +
+ +
+ ) : previewData ? ( + <> + {/* Statistics Summary */} +
+ 0} + /> + + + +
+ + {/* No Changes Message */} + {!hasChanges && ( +
+ {t("noChanges")} +
+ )} + + {/* Changes Table */} + {hasChanges && ( +
+ + + + {t("providerHeader")} + {t("vendorChangeHeader")} + + + + {displayedChanges.map((change) => ( + + + {change.providerName} + + +
+ + {change.oldVendorDomain || "-"} + + + + {change.newVendorDomain} + +
+ + + ))} + +
+ {remainingCount > 0 && ( +
+ {t("moreChanges", { count: remainingCount })} +
+ )} +
+ )} + + ) : null} +
+ + + + + + + + ); +} + +function StatCard({ + label, + value, + highlight = false, +}: { + label: string; + value: number; + highlight?: boolean; +}) { + return ( +
+
0 ? "text-primary" : ""}`} + > + {value} +
+
{label}
+
+ ); +} diff --git a/src/app/[locale]/settings/providers/page.tsx b/src/app/[locale]/settings/providers/page.tsx index 3db554604..9a561a0e8 100644 --- a/src/app/[locale]/settings/providers/page.tsx +++ b/src/app/[locale]/settings/providers/page.tsx @@ -7,6 +7,7 @@ import { getSession } from "@/lib/auth"; import { SettingsPageHeader } from "../_components/settings-page-header"; import { AutoSortPriorityDialog } from "./_components/auto-sort-priority-dialog"; import { ProviderManagerLoader } from "./_components/provider-manager-loader"; +import { ReclusterVendorsDialog } from "./_components/recluster-vendors-dialog"; import { SchedulingRulesDialog } from "./_components/scheduling-rules-dialog"; export const dynamic = "force-dynamic"; @@ -31,6 +32,7 @@ export default async function SettingsProvidersPage() { + } diff --git a/src/repository/provider-endpoints.ts b/src/repository/provider-endpoints.ts index fe6144003..f2909b9b2 100644 --- a/src/repository/provider-endpoints.ts +++ b/src/repository/provider-endpoints.ts @@ -54,6 +54,76 @@ function normalizeWebsiteDomainFromUrl(rawUrl: string): string | null { return null; } +/** + * Normalize URL to host:port format for vendor key when websiteUrl is empty. + * - IPv6 addresses are formatted as [ipv6]:port + * - Default ports: http=80, https=443 + * - URLs without scheme are assumed to be https + */ +function normalizeHostWithPort(rawUrl: string): string | null { + const trimmed = rawUrl.trim(); + if (!trimmed) return null; + + // Add https:// if no scheme present + let urlString = trimmed; + if (!/^[a-zA-Z][a-zA-Z\d+.-]*:\/\//.test(trimmed)) { + urlString = `https://${trimmed}`; + } + + try { + const parsed = new URL(urlString); + const hostname = parsed.hostname?.toLowerCase(); + if (!hostname) return null; + + // Strip www. prefix + const normalizedHostname = hostname.startsWith("www.") ? hostname.slice(4) : hostname; + + // Determine port + let port: string; + if (parsed.port) { + port = parsed.port; + } else { + // Use protocol default port + port = parsed.protocol === "http:" ? "80" : "443"; + } + + // IPv6 addresses already have brackets from URL parser (e.g., "[::1]") + // Just append the port directly + return `${normalizedHostname}:${port}`; + } catch (error) { + logger.debug("[ProviderVendor] Failed to parse URL for host:port", { + urlLength: rawUrl.length, + error: error instanceof Error ? error.message : String(error), + }); + return null; + } +} + +/** + * Compute vendor clustering key based on URLs. + * + * Rules: + * - If websiteUrl is non-empty: key = normalized hostname (strip www, lowercase), ignore port + * - If websiteUrl is empty: key = host:port + * - IPv6 format: [ipv6]:port + * - Missing port: use protocol default (http=80, https=443) + * - No scheme: assume https + */ +export function computeVendorKey(input: { + providerUrl: string; + websiteUrl?: string | null; +}): string | null { + const { providerUrl, websiteUrl } = input; + + // Case 1: websiteUrl is non-empty - use hostname only (existing behavior) + if (websiteUrl?.trim()) { + return normalizeWebsiteDomainFromUrl(websiteUrl); + } + + // Case 2: websiteUrl is empty - use host:port as key + return normalizeHostWithPort(providerUrl); +} + // eslint-disable-next-line @typescript-eslint/no-explicit-any function toProviderVendor(row: any): ProviderVendor { return { @@ -188,8 +258,11 @@ export async function getOrCreateProviderVendorIdFromUrls(input: { faviconUrl?: string | null; displayName?: string | null; }): Promise { - const domainSource = input.websiteUrl?.trim() ? input.websiteUrl : input.providerUrl; - const websiteDomain = normalizeWebsiteDomainFromUrl(domainSource); + // Use new computeVendorKey for consistent vendor key calculation + const websiteDomain = computeVendorKey({ + providerUrl: input.providerUrl, + websiteUrl: input.websiteUrl, + }); if (!websiteDomain) { throw new Error("Failed to resolve provider vendor domain"); } @@ -305,10 +378,13 @@ export async function backfillProviderVendorsFromProviders(): Promise<{ for (const row of rows) { stats.processed++; - const domainSource = row.websiteUrl?.trim() || row.url; - const domain = normalizeWebsiteDomainFromUrl(domainSource); + // Use new computeVendorKey for consistent vendor key calculation + const vendorKey = computeVendorKey({ + providerUrl: row.url, + websiteUrl: row.websiteUrl, + }); - if (!domain) { + if (!vendorKey) { logger.warn("[backfillVendors] Invalid URL for provider", { providerId: row.id, url: row.url, @@ -319,7 +395,9 @@ export async function backfillProviderVendorsFromProviders(): Promise<{ } try { - const displayName = await deriveDisplayNameFromDomain(domain); + // For displayName, extract domain part (remove port if present) + const domainForDisplayName = vendorKey.replace(/:\d+$/, "").replace(/^\[|\]$/g, ""); + const displayName = await deriveDisplayNameFromDomain(domainForDisplayName); const vendorId = await getOrCreateProviderVendorIdFromUrls({ providerUrl: row.url, websiteUrl: row.websiteUrl ?? null, diff --git a/tests/unit/actions/providers-recluster.test.ts b/tests/unit/actions/providers-recluster.test.ts new file mode 100644 index 000000000..a9b775d6b --- /dev/null +++ b/tests/unit/actions/providers-recluster.test.ts @@ -0,0 +1,298 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const getSessionMock = vi.fn(); +const findAllProvidersFreshMock = vi.fn(); +const findProviderVendorByIdMock = vi.fn(); +const getOrCreateProviderVendorIdFromUrlsMock = vi.fn(); +const computeVendorKeyMock = vi.fn(); +const backfillProviderEndpointsFromProvidersMock = vi.fn(); +const tryDeleteProviderVendorIfEmptyMock = vi.fn(); +const publishProviderCacheInvalidationMock = vi.fn(); +const dbMock = { + transaction: vi.fn(), + update: vi.fn(), +}; + +vi.mock("@/lib/auth", () => ({ + getSession: getSessionMock, +})); + +vi.mock("@/repository/provider", () => ({ + findAllProviders: vi.fn(async () => []), + findAllProvidersFresh: findAllProvidersFreshMock, + findProviderById: vi.fn(async () => null), +})); + +vi.mock("@/repository/provider-endpoints", () => ({ + computeVendorKey: computeVendorKeyMock, + findProviderVendorById: findProviderVendorByIdMock, + getOrCreateProviderVendorIdFromUrls: getOrCreateProviderVendorIdFromUrlsMock, + backfillProviderEndpointsFromProviders: backfillProviderEndpointsFromProvidersMock, + tryDeleteProviderVendorIfEmpty: tryDeleteProviderVendorIfEmptyMock, +})); + +vi.mock("@/lib/cache/provider-cache", () => ({ + publishProviderCacheInvalidation: publishProviderCacheInvalidationMock, +})); + +vi.mock("@/drizzle/db", () => ({ + db: dbMock, +})); + +vi.mock("@/lib/logger", () => ({ + logger: { + trace: vi.fn(), + debug: vi.fn(), + info: vi.fn(), + warn: vi.fn(), + error: vi.fn(), + }, +})); + +describe("reclusterProviderVendors", () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + describe("permission checks", () => { + it("returns error when not logged in", async () => { + getSessionMock.mockResolvedValue(null); + + const { reclusterProviderVendors } = await import("@/actions/providers"); + const result = await reclusterProviderVendors({ confirm: false }); + + expect(result.ok).toBe(false); + expect(result.error).toBeDefined(); + }); + + it("returns error when user is not admin", async () => { + getSessionMock.mockResolvedValue({ user: { id: 1, role: "user" } }); + + const { reclusterProviderVendors } = await import("@/actions/providers"); + const result = await reclusterProviderVendors({ confirm: false }); + + expect(result.ok).toBe(false); + expect(result.error).toBeDefined(); + }); + + it("allows admin users", async () => { + getSessionMock.mockResolvedValue({ user: { id: 1, role: "admin" } }); + findAllProvidersFreshMock.mockResolvedValue([]); + + const { reclusterProviderVendors } = await import("@/actions/providers"); + const result = await reclusterProviderVendors({ confirm: false }); + + expect(result.ok).toBe(true); + }); + }); + + describe("preview mode (confirm=false)", () => { + it("returns empty changes when no providers", async () => { + getSessionMock.mockResolvedValue({ user: { id: 1, role: "admin" } }); + findAllProvidersFreshMock.mockResolvedValue([]); + + const { reclusterProviderVendors } = await import("@/actions/providers"); + const result = await reclusterProviderVendors({ confirm: false }); + + expect(result.ok).toBe(true); + if (result.ok) { + expect(result.data.applied).toBe(false); + expect(result.data.changes).toEqual([]); + expect(result.data.preview.providersMoved).toBe(0); + } + }); + + it("detects providers that need vendor change", async () => { + getSessionMock.mockResolvedValue({ user: { id: 1, role: "admin" } }); + findAllProvidersFreshMock.mockResolvedValue([ + { + id: 1, + name: "Provider 1", + url: "http://192.168.1.1:8080/v1/messages", + websiteUrl: null, + providerVendorId: 1, + }, + { + id: 2, + name: "Provider 2", + url: "http://192.168.1.1:9090/v1/messages", + websiteUrl: null, + providerVendorId: 1, // Same vendor but different port - should change + }, + ]); + + // Current vendor has domain "192.168.1.1" (old behavior) + findProviderVendorByIdMock.mockResolvedValue({ + id: 1, + websiteDomain: "192.168.1.1", + }); + + // New vendor keys include port + computeVendorKeyMock + .mockReturnValueOnce("192.168.1.1:8080") + .mockReturnValueOnce("192.168.1.1:9090"); + + const { reclusterProviderVendors } = await import("@/actions/providers"); + const result = await reclusterProviderVendors({ confirm: false }); + + expect(result.ok).toBe(true); + if (result.ok) { + expect(result.data.applied).toBe(false); + expect(result.data.preview.providersMoved).toBe(2); + expect(result.data.changes.length).toBe(2); + } + }); + + it("does not modify database in preview mode", async () => { + getSessionMock.mockResolvedValue({ user: { id: 1, role: "admin" } }); + findAllProvidersFreshMock.mockResolvedValue([ + { + id: 1, + name: "Provider 1", + url: "http://192.168.1.1:8080/v1/messages", + websiteUrl: null, + providerVendorId: 1, + }, + ]); + findProviderVendorByIdMock.mockResolvedValue({ + id: 1, + websiteDomain: "192.168.1.1", + }); + computeVendorKeyMock.mockReturnValue("192.168.1.1:8080"); + + const { reclusterProviderVendors } = await import("@/actions/providers"); + await reclusterProviderVendors({ confirm: false }); + + expect(dbMock.transaction).not.toHaveBeenCalled(); + expect(publishProviderCacheInvalidationMock).not.toHaveBeenCalled(); + }); + + it("skips providers with invalid URLs", async () => { + getSessionMock.mockResolvedValue({ user: { id: 1, role: "admin" } }); + findAllProvidersFreshMock.mockResolvedValue([ + { + id: 1, + name: "Invalid Provider", + url: "://invalid", + websiteUrl: null, + providerVendorId: null, + }, + ]); + computeVendorKeyMock.mockReturnValue(null); + + const { reclusterProviderVendors } = await import("@/actions/providers"); + const result = await reclusterProviderVendors({ confirm: false }); + + expect(result.ok).toBe(true); + if (result.ok) { + expect(result.data.preview.skippedInvalidUrl).toBe(1); + expect(result.data.preview.providersMoved).toBe(0); + } + }); + }); + + describe("apply mode (confirm=true)", () => { + it("executes database updates in transaction", async () => { + getSessionMock.mockResolvedValue({ user: { id: 1, role: "admin" } }); + findAllProvidersFreshMock.mockResolvedValue([ + { + id: 1, + name: "Provider 1", + url: "http://192.168.1.1:8080/v1/messages", + websiteUrl: null, + providerVendorId: 1, + }, + ]); + findProviderVendorByIdMock.mockResolvedValue({ + id: 1, + websiteDomain: "192.168.1.1", + }); + computeVendorKeyMock.mockReturnValue("192.168.1.1:8080"); + getOrCreateProviderVendorIdFromUrlsMock.mockResolvedValue(2); + backfillProviderEndpointsFromProvidersMock.mockResolvedValue({}); + tryDeleteProviderVendorIfEmptyMock.mockResolvedValue(true); + dbMock.transaction.mockImplementation(async (fn) => { + return fn({ + update: vi.fn().mockReturnValue({ + set: vi.fn().mockReturnValue({ + where: vi.fn().mockResolvedValue({}), + }), + }), + }); + }); + + const { reclusterProviderVendors } = await import("@/actions/providers"); + const result = await reclusterProviderVendors({ confirm: true }); + + expect(result.ok).toBe(true); + if (result.ok) { + expect(result.data.applied).toBe(true); + } + expect(dbMock.transaction).toHaveBeenCalled(); + }); + + it("publishes cache invalidation after apply", async () => { + getSessionMock.mockResolvedValue({ user: { id: 1, role: "admin" } }); + findAllProvidersFreshMock.mockResolvedValue([ + { + id: 1, + name: "Provider 1", + url: "http://192.168.1.1:8080/v1/messages", + websiteUrl: null, + providerVendorId: 1, + }, + ]); + findProviderVendorByIdMock.mockResolvedValue({ + id: 1, + websiteDomain: "192.168.1.1", + }); + computeVendorKeyMock.mockReturnValue("192.168.1.1:8080"); + getOrCreateProviderVendorIdFromUrlsMock.mockResolvedValue(2); + backfillProviderEndpointsFromProvidersMock.mockResolvedValue({}); + tryDeleteProviderVendorIfEmptyMock.mockResolvedValue(true); + dbMock.transaction.mockImplementation(async (fn) => { + return fn({ + update: vi.fn().mockReturnValue({ + set: vi.fn().mockReturnValue({ + where: vi.fn().mockResolvedValue({}), + }), + }), + }); + }); + + const { reclusterProviderVendors } = await import("@/actions/providers"); + await reclusterProviderVendors({ confirm: true }); + + expect(publishProviderCacheInvalidationMock).toHaveBeenCalled(); + }); + + it("does not apply when no changes needed", async () => { + getSessionMock.mockResolvedValue({ user: { id: 1, role: "admin" } }); + findAllProvidersFreshMock.mockResolvedValue([ + { + id: 1, + name: "Provider 1", + url: "http://192.168.1.1:8080/v1/messages", + websiteUrl: null, + providerVendorId: 1, + }, + ]); + // Vendor already has correct domain + findProviderVendorByIdMock.mockResolvedValue({ + id: 1, + websiteDomain: "192.168.1.1:8080", + }); + computeVendorKeyMock.mockReturnValue("192.168.1.1:8080"); + + const { reclusterProviderVendors } = await import("@/actions/providers"); + const result = await reclusterProviderVendors({ confirm: true }); + + expect(result.ok).toBe(true); + if (result.ok) { + expect(result.data.applied).toBe(true); + expect(result.data.preview.providersMoved).toBe(0); + } + expect(dbMock.transaction).not.toHaveBeenCalled(); + }); + }); +}); diff --git a/tests/unit/repository/provider-endpoints-vendor-key.test.ts b/tests/unit/repository/provider-endpoints-vendor-key.test.ts new file mode 100644 index 000000000..04746355f --- /dev/null +++ b/tests/unit/repository/provider-endpoints-vendor-key.test.ts @@ -0,0 +1,206 @@ +import { describe, expect, test } from "vitest"; +import { computeVendorKey } from "@/repository/provider-endpoints"; + +describe("computeVendorKey", () => { + describe("with websiteUrl (priority over providerUrl)", () => { + test("returns hostname only, ignoring port", () => { + expect( + computeVendorKey({ + providerUrl: "https://api.example.com:8080/v1/messages", + websiteUrl: "https://example.com:3000", + }) + ).toBe("example.com"); + }); + + test("strips www prefix", () => { + expect( + computeVendorKey({ + providerUrl: "https://api.example.com", + websiteUrl: "https://www.example.com", + }) + ).toBe("example.com"); + }); + + test("lowercases hostname", () => { + expect( + computeVendorKey({ + providerUrl: "https://api.Example.COM", + websiteUrl: "https://WWW.EXAMPLE.COM", + }) + ).toBe("example.com"); + }); + + test("handles websiteUrl without protocol", () => { + expect( + computeVendorKey({ + providerUrl: "https://api.example.com", + websiteUrl: "example.com", + }) + ).toBe("example.com"); + }); + }); + + describe("without websiteUrl (fallback to providerUrl with host:port)", () => { + test("returns host:port for IP address", () => { + expect( + computeVendorKey({ + providerUrl: "http://192.168.1.1:8080/v1/messages", + websiteUrl: null, + }) + ).toBe("192.168.1.1:8080"); + }); + + test("different ports create different keys", () => { + const key1 = computeVendorKey({ + providerUrl: "http://192.168.1.1:8080/v1/messages", + websiteUrl: null, + }); + const key2 = computeVendorKey({ + providerUrl: "http://192.168.1.1:9090/v1/messages", + websiteUrl: null, + }); + expect(key1).toBe("192.168.1.1:8080"); + expect(key2).toBe("192.168.1.1:9090"); + expect(key1).not.toBe(key2); + }); + + test("uses default port 443 for https without explicit port", () => { + expect( + computeVendorKey({ + providerUrl: "https://api.example.com/v1/messages", + websiteUrl: null, + }) + ).toBe("api.example.com:443"); + }); + + test("uses default port 80 for http without explicit port", () => { + expect( + computeVendorKey({ + providerUrl: "http://api.example.com/v1/messages", + websiteUrl: null, + }) + ).toBe("api.example.com:80"); + }); + + test("assumes https (port 443) for URL without scheme", () => { + expect( + computeVendorKey({ + providerUrl: "api.example.com/v1/messages", + websiteUrl: null, + }) + ).toBe("api.example.com:443"); + }); + + test("strips www prefix in host:port mode", () => { + expect( + computeVendorKey({ + providerUrl: "https://www.example.com:8080/v1/messages", + websiteUrl: null, + }) + ).toBe("example.com:8080"); + }); + + test("lowercases hostname in host:port mode", () => { + expect( + computeVendorKey({ + providerUrl: "https://API.EXAMPLE.COM:8080/v1/messages", + websiteUrl: null, + }) + ).toBe("api.example.com:8080"); + }); + + test("handles localhost with port", () => { + expect( + computeVendorKey({ + providerUrl: "http://localhost:3000/v1/messages", + websiteUrl: null, + }) + ).toBe("localhost:3000"); + }); + + test("handles localhost without explicit port", () => { + expect( + computeVendorKey({ + providerUrl: "http://localhost/v1/messages", + websiteUrl: null, + }) + ).toBe("localhost:80"); + }); + }); + + describe("IPv6 addresses", () => { + test("formats IPv6 with brackets and port", () => { + expect( + computeVendorKey({ + providerUrl: "http://[::1]:8080/v1/messages", + websiteUrl: null, + }) + ).toBe("[::1]:8080"); + }); + + test("handles IPv6 without explicit port", () => { + expect( + computeVendorKey({ + providerUrl: "https://[::1]/v1/messages", + websiteUrl: null, + }) + ).toBe("[::1]:443"); + }); + + test("handles full IPv6 address", () => { + expect( + computeVendorKey({ + providerUrl: "http://[2001:db8::1]:9000/v1/messages", + websiteUrl: null, + }) + ).toBe("[2001:db8::1]:9000"); + }); + }); + + describe("edge cases", () => { + test("returns null for empty providerUrl", () => { + expect( + computeVendorKey({ + providerUrl: "", + websiteUrl: null, + }) + ).toBeNull(); + }); + + test("returns null for whitespace-only providerUrl", () => { + expect( + computeVendorKey({ + providerUrl: " ", + websiteUrl: null, + }) + ).toBeNull(); + }); + + test("uses providerUrl when websiteUrl is empty string", () => { + expect( + computeVendorKey({ + providerUrl: "http://192.168.1.1:8080/v1/messages", + websiteUrl: "", + }) + ).toBe("192.168.1.1:8080"); + }); + + test("uses providerUrl when websiteUrl is whitespace", () => { + expect( + computeVendorKey({ + providerUrl: "http://192.168.1.1:8080/v1/messages", + websiteUrl: " ", + }) + ).toBe("192.168.1.1:8080"); + }); + + test("returns null for truly invalid URL", () => { + expect( + computeVendorKey({ + providerUrl: "://invalid", + websiteUrl: null, + }) + ).toBeNull(); + }); + }); +}); From e11813845a80fd88fc1e555a4b2b9103d0444c78 Mon Sep 17 00:00:00 2001 From: ding113 Date: Wed, 28 Jan 2026 19:37:24 +0800 Subject: [PATCH 2/2] perf(providers): optimize recluster with batch vendor loading and Map lookup - Batch load all vendor data upfront with Promise.all to avoid N+1 queries - Use Map for O(1) provider lookup instead of O(N) find() in transaction loop - Addresses bugbot review comments from gemini-code-assist and greptile-apps Co-Authored-By: Claude Opus 4.5 --- src/actions/providers.ts | 30 ++++++++++++++++++++++++------ 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/src/actions/providers.ts b/src/actions/providers.ts index 4274ca3d5..09a646aa8 100644 --- a/src/actions/providers.ts +++ b/src/actions/providers.ts @@ -3619,6 +3619,22 @@ export async function reclusterProviderVendors(args: { const oldVendorIds = new Set(); let skippedInvalidUrl = 0; + // Batch load all vendor data upfront to avoid N+1 queries + const uniqueVendorIds = [ + ...new Set( + allProviders + .map((p) => p.providerVendorId) + .filter((id): id is number => id !== null && id !== undefined && id > 0) + ), + ]; + const vendors = await Promise.all(uniqueVendorIds.map((id) => findProviderVendorById(id))); + const vendorMap = new Map( + vendors.filter((v): v is NonNullable => v !== null).map((v) => [v.id, v]) + ); + + // Build provider map for quick lookup in transaction + const providerMap = new Map(allProviders.map((p) => [p.id, p])); + // Calculate new vendor key for each provider for (const provider of allProviders) { const newVendorKey = computeVendorKey({ @@ -3631,10 +3647,8 @@ export async function reclusterProviderVendors(args: { continue; } - // Get current vendor domain - const currentVendor = provider.providerVendorId - ? await findProviderVendorById(provider.providerVendorId) - : null; + // Get current vendor domain from pre-loaded map + const currentVendor = provider.providerVendorId ? vendorMap.get(provider.providerVendorId) : null; const currentDomain = currentVendor?.websiteDomain ?? ""; // If key changed, record the change @@ -3676,10 +3690,14 @@ export async function reclusterProviderVendors(args: { if (changes.length > 0) { await db.transaction(async (tx) => { for (const change of changes) { + // Use pre-built map for O(1) lookup instead of O(N) find() + const provider = providerMap.get(change.providerId); + if (!provider) continue; + // Get or create new vendor const newVendorId = await getOrCreateProviderVendorIdFromUrls({ - providerUrl: allProviders.find((p) => p.id === change.providerId)?.url ?? "", - websiteUrl: allProviders.find((p) => p.id === change.providerId)?.websiteUrl ?? null, + providerUrl: provider.url, + websiteUrl: provider.websiteUrl ?? null, }); // Update provider's vendorId