Skip to content

fix: improve OpenRouter model validation to support newer models #9598

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
roomote wants to merge 1 commit into
base: main
Choose a base branch
from
Open

fix: improve OpenRouter model validation to support newer models #9598

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
125 changes: 118 additions & 7 deletions ...w-ui/src/utils/__tests__/validate.test.ts → ...w-ui/src/utils/__tests__/validate.spec.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,20 @@ import { RouterModels } from "@roo/api"

import { getModelValidationError, validateApiConfigurationExcludingModelErrors } from "../validate"

// Mock i18next
vi.mock("i18next", () => ({
default: {
t: vi.fn((key: string, options?: any) => {
// Return the key as-is for testing (this matches what the actual code expects)
if (options?.modelId) {
return key // For validation.modelAvailability, just return the key
}
return key
}),
language: "en",
},
}))

describe("Model Validation Functions", () => {
const mockRouterModels: RouterModels = {
openrouter: {
Expand Down Expand Up @@ -47,6 +61,25 @@ describe("Model Validation Functions", () => {
chutes: {},
}

// Mock router models with many models (>10) to trigger validation
const mockRouterModelsWithManyModels: RouterModels = {
...mockRouterModels,
openrouter: {
...mockRouterModels.openrouter,
"model-1": { maxTokens: 1000, contextWindow: 10000, supportsImages: false, supportsPromptCache: false },
"model-2": { maxTokens: 1000, contextWindow: 10000, supportsImages: false, supportsPromptCache: false },
"model-3": { maxTokens: 1000, contextWindow: 10000, supportsImages: false, supportsPromptCache: false },
"model-4": { maxTokens: 1000, contextWindow: 10000, supportsImages: false, supportsPromptCache: false },
"model-5": { maxTokens: 1000, contextWindow: 10000, supportsImages: false, supportsPromptCache: false },
"model-6": { maxTokens: 1000, contextWindow: 10000, supportsImages: false, supportsPromptCache: false },
"model-7": { maxTokens: 1000, contextWindow: 10000, supportsImages: false, supportsPromptCache: false },
"model-8": { maxTokens: 1000, contextWindow: 10000, supportsImages: false, supportsPromptCache: false },
"model-9": { maxTokens: 1000, contextWindow: 10000, supportsImages: false, supportsPromptCache: false },
"model-10": { maxTokens: 1000, contextWindow: 10000, supportsImages: false, supportsPromptCache: false },
"model-11": { maxTokens: 1000, contextWindow: 10000, supportsImages: false, supportsPromptCache: false },
},
}

const allowAllOrganization: OrganizationAllowList = {
allowAll: true,
providers: {},
Expand All @@ -73,14 +106,92 @@ describe("Model Validation Functions", () => {
expect(result).toBeUndefined()
})

it("returns error for invalid OpenRouter model", () => {
it("returns error for invalid OpenRouter model when model list has many models", () => {
const config: ProviderSettings = {
apiProvider: "openrouter",
openRouterModelId: "invalid-model",
}

// Should return error when we have many models (>10) in cache
const result = getModelValidationError(config, mockRouterModelsWithManyModels, allowAllOrganization)
expect(result).toBe("settings:validation.modelAvailability")
})

it("allows newer OpenRouter models like gpt-5.1 even when not in cache", () => {
const config: ProviderSettings = {
apiProvider: "openrouter",
openRouterModelId: "openai/gpt-5.1",
}

// Should allow gpt-5.1 even when not in the model list
const result = getModelValidationError(config, mockRouterModelsWithManyModels, allowAllOrganization)
expect(result).toBeUndefined()
})

it("allows newer OpenRouter models like gemini-3-pro-preview even when not in cache", () => {
const config: ProviderSettings = {
apiProvider: "openrouter",
openRouterModelId: "google/gemini-3-pro-preview",
}

// Should allow gemini-3 models even when not in the model list
const result = getModelValidationError(config, mockRouterModelsWithManyModels, allowAllOrganization)
expect(result).toBeUndefined()
})

it("allows newer OpenRouter models like grok-4.1-fast:free even when not in cache", () => {
const config: ProviderSettings = {
apiProvider: "openrouter",
openRouterModelId: "x-ai/grok-4.1-fast:free",
}

// Should allow grok-4 models even when not in the model list
const result = getModelValidationError(config, mockRouterModelsWithManyModels, allowAllOrganization)
expect(result).toBeUndefined()
})

it("allows known pattern models for OpenRouter even when not in cache", () => {
const testCases = [
"openai/gpt-5",
"openai/gpt-5.2",
"google/gemini-4",
"x-ai/grok-5",
"anthropic/claude-4-opus",
"meta-llama/llama-4",
"mistralai/mistral-large-2",
]

testCases.forEach((modelId) => {
const config: ProviderSettings = {
apiProvider: "openrouter",
openRouterModelId: modelId,
}

const result = getModelValidationError(config, mockRouterModelsWithManyModels, allowAllOrganization)
expect(result).toBeUndefined()
})
})

it("still validates unknown pattern models for OpenRouter", () => {
const config: ProviderSettings = {
apiProvider: "openrouter",
openRouterModelId: "unknown-provider/unknown-model",
}

// Should still validate models that don't match known patterns
const result = getModelValidationError(config, mockRouterModelsWithManyModels, allowAllOrganization)
expect(result).toBe("settings:validation.modelAvailability")
})

it("does not validate when model list is small (<=10 models)", () => {
const config: ProviderSettings = {
apiProvider: "openrouter",
openRouterModelId: "any-model-id",
}

// Should not validate when we have few models in cache (initial state)
const result = getModelValidationError(config, mockRouterModels, allowAllOrganization)
expect(result).toBe("validation.modelAvailability")
expect(result).toBeUndefined()
})

it("returns error for model not allowed by organization", () => {
Expand Down Expand Up @@ -130,7 +241,7 @@ describe("Model Validation Functions", () => {
}

const result = getModelValidationError(config, mockRouterModels, allowAllOrganization)
expect(result).toBe("validation.modelId")
expect(result).toBe("settings:validation.modelId")
})

it("handles undefined model IDs gracefully", () => {
Expand All @@ -140,7 +251,7 @@ describe("Model Validation Functions", () => {
}

const result = getModelValidationError(config, mockRouterModels, allowAllOrganization)
expect(result).toBe("validation.modelId")
expect(result).toBe("settings:validation.modelId")
})
})

Expand All @@ -164,7 +275,7 @@ describe("Model Validation Functions", () => {
}

const result = validateApiConfigurationExcludingModelErrors(config, mockRouterModels, allowAllOrganization)
expect(result).toBe("validation.apiKey")
expect(result).toBe("settings:validation.apiKey")
})

it("excludes model-specific errors", () => {
Expand Down Expand Up @@ -196,7 +307,7 @@ describe("Model Validation Functions", () => {
it("returns undefined for valid IO Intelligence model", () => {
const config: ProviderSettings = {
apiProvider: "io-intelligence",
glamaModelId: "valid-model",
ioIntelligenceModelId: "valid-model",
}

const result = getModelValidationError(config, mockRouterModels, allowAllOrganization)
Expand All @@ -206,7 +317,7 @@ describe("Model Validation Functions", () => {
it("returns error for invalid IO Intelligence model", () => {
const config: ProviderSettings = {
apiProvider: "io-intelligence",
glamaModelId: "invalid-model",
ioIntelligenceModelId: "invalid-model",
}

const result = getModelValidationError(config, mockRouterModels, allowAllOrganization)
Expand Down
26 changes: 25 additions & 1 deletion webview-ui/src/utils/validate.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -276,7 +276,31 @@ function validateDynamicProviderModelId(

const models = routerModels?.[provider]

if (models && Object.keys(models).length > 1 && !Object.keys(models).includes(modelId)) {
// Only validate if we have a substantial model list (more than 10 models)
// This prevents false negatives for newer models that haven't been added to the cache yet
// OpenRouter frequently adds new models that are immediately usable but may not appear
// in the /models endpoint right away
if (models && Object.keys(models).length > 10 && !Object.keys(models).includes(modelId)) {
// For OpenRouter specifically, we'll be more lenient with model validation
// to support newer models like gpt-5.1, gemini-3-pro-preview, grok-4.1-fast:free
if (provider === "openrouter") {
// Only show a warning for models that don't match any known pattern
// Allow models that follow common naming patterns even if not in cache
const knownPatterns = [
/^openai\/gpt-\d+(\.\d+)?/, // Matches gpt-5, gpt-5.1, etc.
/^google\/gemini-\d+/, // Matches gemini-3, gemini-3-pro, etc.
/^x-ai\/grok-\d+/, // Matches grok-4, grok-4.1, etc.
Copy link
Contributor

@ellipsis-dev ellipsis-dev bot Nov 26, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The regex for x-ai/grok only matches digits (e.g. /^x-ai\/grok-\d+/). To support versions with decimals (e.g. 4.1), consider updating it to /^x-ai\/grok-\d+(\.\d+)?/ for consistency with the OpenAI pattern.

Suggested change
/^x-ai\/grok-\d+/, // Matches grok-4, grok-4.1, etc.
/^x-ai\/grok-\d+(\.\d+)?/, // Matches grok-4, grok-4.1, etc.

/^anthropic\/claude-/, // Claude models
/^meta-llama\//, // Meta Llama models
/^mistralai\//, // Mistral models
]

// If the model matches a known pattern, allow it
if (knownPatterns.some((pattern) => pattern.test(modelId))) {
return undefined
}
}

return i18next.t("settings:validation.modelAvailability", { modelId })
}

Expand Down
Loading