💄 style: update Groq model list & add GROQ_MODEL_LIST support

Dockerfile

@@ -119,7 +119,7 @@ ENV \
  # Google
  GOOGLE_API_KEY="" GOOGLE_PROXY_URL="" \
  # Groq
- GROQ_API_KEY="" GROQ_PROXY_URL="" \
+ GROQ_API_KEY="" GROQ_MODEL_LIST="" GROQ_PROXY_URL="" \
  # Minimax
  MINIMAX_API_KEY="" \
  # Mistral

Dockerfile.database

@@ -151,7 +151,7 @@ ENV \
  # Google
  GOOGLE_API_KEY="" GOOGLE_PROXY_URL="" \
  # Groq
- GROQ_API_KEY="" GROQ_PROXY_URL="" \
+ GROQ_API_KEY="" GROQ_MODEL_LIST="" GROQ_PROXY_URL="" \
  # Minimax
  MINIMAX_API_KEY="" \
  # Mistral

src/config/llm.ts

@@ -49,6 +49,7 @@ export const getLLMConfig = () => {
 
  ENABLED_GROQ: z.boolean(),
  GROQ_API_KEY: z.string().optional(),
+ GROQ_MODEL_LIST: z.string().optional(),
  GROQ_PROXY_URL: z.string().optional(),
 
  ENABLED_OPENROUTER: z.boolean(),
@@ -153,6 +154,7 @@ export const getLLMConfig = () => {
 
  ENABLED_GROQ: !!process.env.GROQ_API_KEY,
  GROQ_API_KEY: process.env.GROQ_API_KEY,
+ GROQ_MODEL_LIST: process.env.GROQ_MODEL_LIST,
  GROQ_PROXY_URL: process.env.GROQ_PROXY_URL,
 
  ENABLED_ZEROONE: !!process.env.ZEROONE_API_KEY,

src/config/modelProviders/groq.ts

@@ -1,59 +1,59 @@
 import { ModelProviderCard } from '@/types/llm';
 
 // ref https://console.groq.com/docs/models
+// ref https://console.groq.com/docs/tool-use
 const Groq: ModelProviderCard = {
  chatModels: [
+ // TODO: During preview launch, Groq is limiting 3.1 models to max_tokens of 8k.
  {
- displayName: 'LLaMA3.1 405B (Preview)',
+ displayName: 'Llama 3.1 8B (Preview)',
+ enabled: true,
  functionCall: true,
- id: 'llama-3.1-405b-reasoning',
- tokens: 16_000,
+ id: 'llama-3.1-8b-instant',
+ tokens: 8000,
  },
  {
- displayName: 'LLaMA 3.1 70B (Preview)',
+ displayName: 'Llama 3.1 70B (Preview)',
  enabled: true,
  functionCall: true,
  id: 'llama-3.1-70b-versatile',
  tokens: 8000,
  },
+/*
+ // Offline due to overwhelming demand! Stay tuned for updates.
  {
- displayName: 'LLaMA 3.1 8B (Preview)',
- enabled: true,
+ displayName: 'Llama 3.1 405B (Preview)',
  functionCall: true,
- id: 'llama-3.1-8b-instant',
+ id: 'llama-3.1-405b-reasoning',
  tokens: 8000,
  },
+*/
  {
- displayName: 'LLaMA 3 Groq 70b Tool Use (preview)',
+ displayName: 'Llama 3 Groq 8B Tool Use (Preview)',
  enabled: true,
  functionCall: true,
- id: 'llama3-groq-70b-8192-tool-use-preview',
+ id: 'llama3-groq-8b-8192-tool-use-preview',
  tokens: 8192,
  },
  {
- displayName: 'LLaMA 3 Groq 8b Tool Use (preview)',
+ displayName: 'Llama 3 Groq 70B Tool Use (Preview)',
  enabled: true,
  functionCall: true,
- id: 'llama3-groq-8b-8192-tool-use-preview',
+ id: 'llama3-groq-70b-8192-tool-use-preview',
+ tokens: 8192,
  },
  {
- displayName: 'LLaMA3 70B',
+ displayName: 'Meta Llama 3 8B',
  enabled: true,
  functionCall: true,
- id: 'llama3-70b-8192',
+ id: 'llama3-8b-8192',
  tokens: 8192,
  },
  {
- displayName: 'Mixtral-8x7b',
+ displayName: 'Meta Llama 3 70B',
  enabled: true,
  functionCall: true,
- id: 'mixtral-8x7b-32768',
- tokens: 32_768,
- },
- {
- displayName: 'Gemma 7B',
- functionCall: true,
- id: 'gemma-7b-it',
+ id: 'llama3-70b-8192',
  tokens: 8192,
  },
  {
@@ -64,16 +64,17 @@ const Groq: ModelProviderCard = {
  tokens: 8192,
  },
  {
- displayName: 'LLaMA3 8B',
- enabled: true,
+ displayName: 'Gemma 7B',
  functionCall: true,
- id: 'llama3-8b-8192',
+ id: 'gemma-7b-it',
  tokens: 8192,
  },
  {
- displayName: 'LLaMA2-70b-chat',
- id: 'llama2-70b-4096',
- tokens: 4096,
+ displayName: 'Mixtral 8x7B',
+ enabled: true,
+ functionCall: true,
+ id: 'mixtral-8x7b-32768',
+ tokens: 32_768,
  },
  ],
  checkModel: 'gemma2-9b-it',

src/server/globalConfig/index.ts

@@ -4,6 +4,7 @@ import { fileEnv } from '@/config/file';
 import { langfuseEnv } from '@/config/langfuse';
 import { getLLMConfig } from '@/config/llm';
 import {
+ GroqProviderCard,
  NovitaProviderCard,
  OllamaProviderCard,
  OpenAIProviderCard,
@@ -34,7 +35,10 @@ export const getServerGlobalConfig = () => {
 
  ENABLED_AWS_BEDROCK,
  ENABLED_GOOGLE,
+
  ENABLED_GROQ,
+ GROQ_MODEL_LIST,
+
  ENABLED_DEEPSEEK,
  ENABLED_PERPLEXITY,
  ENABLED_ANTHROPIC,
@@ -99,7 +103,14 @@ export const getServerGlobalConfig = () => {
  bedrock: { enabled: ENABLED_AWS_BEDROCK },
  deepseek: { enabled: ENABLED_DEEPSEEK },
  google: { enabled: ENABLED_GOOGLE },
- groq: { enabled: ENABLED_GROQ },
+ groq: {
+ enabled: ENABLED_GROQ,
+ enabledModels: extractEnabledModels(GROQ_MODEL_LIST),
+ serverModelCards: transformToChatModelCards({
+ defaultChatModels: GroqProviderCard.chatModels,
+ modelString: GROQ_MODEL_LIST,
+ }),
+ },
  minimax: { enabled: ENABLED_MINIMAX },
  mistral: { enabled: ENABLED_MISTRAL },
  moonshot: { enabled: ENABLED_MOONSHOT },