Merge pull request #548 from ilsubyeega/provider-sambanova

provider: SambaNova FastAPI implementation.

src/globals.ts

@@ -67,6 +67,7 @@ export const GITHUB: string = 'github';
 export const DEEPBRICKS: string = 'deepbricks';
 export const SILICONFLOW: string = 'siliconflow';
 export const CEREBRAS: string = 'cerebras';
+export const SAMBANOVA: string = 'sambanova';
 
 export const VALID_PROVIDERS = [
   ANTHROPIC,
@@ -108,6 +109,7 @@ export const VALID_PROVIDERS = [
   SILICONFLOW,
   HUGGING_FACE,
   CEREBRAS,
+  SAMBANOVA,
 ];
 
 export const CONTENT_TYPES = {

src/providers/index.ts

@@ -40,6 +40,7 @@ import DeepbricksConfig from './deepbricks';
 import SiliconFlowConfig from './siliconflow';
 import HuggingfaceConfig from './huggingface';
 import { cerebrasProviderAPIConfig } from './cerebras';
+import SambaNovaConfig from './sambanova';
 
 const Providers: { [key: string]: ProviderConfigs } = {
   openai: OpenAIConfig,
@@ -81,6 +82,7 @@ const Providers: { [key: string]: ProviderConfigs } = {
   deepbricks: DeepbricksConfig,
   siliconflow: SiliconFlowConfig,
   cerebras: cerebrasProviderAPIConfig,
+  sambanova: SambaNovaConfig,
 };
 
 export default Providers;

src/providers/sambanova/api.ts

@@ -0,0 +1,19 @@
+import { ProviderAPIConfig } from '../types';
+
+const SambaNovaAPIConfig: ProviderAPIConfig = {
+  getBaseURL: ({ providerOptions }) =>
+    providerOptions.customHost || 'https://fast-api.snova.ai',
+  headers: ({ providerOptions }) => {
+    return { Authorization: `Bearer ${providerOptions.apiKey}` };
+  },
+  getEndpoint: ({ fn }) => {
+    switch (fn) {
+      case 'chatComplete':
+        return '/v1/chat/completions';
+      default:
+        return '';
+    }
+  },
+};
+
+export default SambaNovaAPIConfig;

src/providers/sambanova/chatComplete.ts

@@ -0,0 +1,84 @@
+import { SAMBANOVA } from '../../globals';
+
+export interface SambaNovaStreamChunk {
+  id: string;
+  object: string;
+  created: number;
+  model: string;
+  system_fingerprint: string;
+  choices: {
+    delta: {
+      content?: string;
+    };
+    index: number;
+    finish_reason: string | null;
+    logprobs: object | null;
+  }[];
+  usage?: {
+    is_last_response: boolean;
+    total_tokens: number;
+    prompt_tokens: number;
+    completion_tokens: number;
+    time_to_first_token: number;
+    end_time: number;
+    start_time: number;
+    total_latency: number;
+    total_tokens_per_sec: number;
+    completion_tokens_per_sec: number;
+    completion_tokens_after_first_per_sec: number;
+    completion_tokens_after_first_per_sec_first_ten: number;
+  };
+}
+
+export const SambaNovaChatCompleteStreamChunkTransform: (
+  response: string
+) => string = (responseChunk) => {
+  let chunk = responseChunk.trim();
+  chunk = chunk.replace(/^data: /, '');
+  chunk = chunk.trim();
+  if (chunk === '[DONE]') {
+    return `data: ${chunk}\n\n`;
+  }
+
+  const parsedChunk: SambaNovaStreamChunk = JSON.parse(chunk);
+  if (parsedChunk.usage) {
+    return `data: ${JSON.stringify({
+      id: parsedChunk.id,
+      object: parsedChunk.object,
+      created: parsedChunk.created,
+      model: parsedChunk.model,
+      provider: SAMBANOVA,
+      choices: [
+        {
+          index: 0,
+          delta: {},
+          logprobs: null,
+          finish_reason: 'stop',
+        },
+      ],
+      usage: {
+        prompt_tokens: parsedChunk.usage.prompt_tokens || 0,
+        completion_tokens: parsedChunk.usage.completion_tokens || 0,
+        total_tokens: parsedChunk.usage.total_tokens || 0,
+      },
+    })}\n\n`;
+  }
+  return `data: ${JSON.stringify({
+    id: parsedChunk.id,
+    object: parsedChunk.object,
+    created: parsedChunk.created,
+    model: parsedChunk.model,
+    provider: SAMBANOVA,
+    choices: [
+      {
+        index: parsedChunk.choices[0].index || 0,
+        delta: {
+          role: 'assistant',
+          content: parsedChunk.choices[0].delta.content,
+        },
+        logprobs: parsedChunk.choices[0].logprobs,
+        finish_reason: parsedChunk.choices[0].finish_reason || null,
+      },
+    ],
+  })}\n\n`;
+};

src/providers/sambanova/index.ts

@@ -0,0 +1,54 @@
+import { SAMBANOVA } from '../../globals';
+import { chatCompleteParams, responseTransformers } from '../open-ai-base';
+import { ProviderConfigs } from '../types';
+import SambaNovaAPIConfig from './api';
+import { SambaNovaChatCompleteStreamChunkTransform } from './chatComplete';
+
+const SambaNovaConfig: ProviderConfigs = {
+  chatComplete: chatCompleteParams(
+    [
+      'functions',
+      'function_call',
+      'presence_penalty',
+      'frequency_penalty',
+      'logit_bias',
+      'user',
+      'seed',
+      'tools',
+      'tool_choice',
+      'response_format',
+      'logprobs',
+    ],
+    {
+      model: 'Meta-Llama-3.1-8B-Instruct',
+    }
+  ),
+  api: SambaNovaAPIConfig,
+  responseTransforms: {
+    ...responseTransformers(SAMBANOVA, {
+      chatComplete: (response, isError) => {
+        if (isError || 'choices' in response === false) return response;
+
+        return {
+          ...response,
+          provider: SAMBANOVA,
+          choices: response.choices.map((choice) => ({
+            ...choice,
+            message: {
+              role: 'assistant',
+              ...(choice.message as any),
+            },
+          })),
+          usage: {
+            prompt_tokens: response.usage?.prompt_tokens || 0,
+            completion_tokens: response.usage?.completion_tokens || 0,
+            total_tokens: response.usage?.total_tokens || 0,
+          },
+        };
+      },
+    }),
+    'stream-chatComplete': SambaNovaChatCompleteStreamChunkTransform,
+  },
+};
+
+export default SambaNovaConfig;

src/utils.ts

@@ -5,6 +5,7 @@ import {
   GOOGLE_VERTEX_AI,
   PERPLEXITY_AI,
   DEEPINFRA,
+  SAMBANOVA,
 } from './globals';
 import { Params } from './types/requestBody';
 
@@ -42,6 +43,10 @@ export const getStreamModeSplitPattern = (
     splitPattern = '\r\n\r\n';
   }
 
+  if (proxyProvider === SAMBANOVA) {
+    splitPattern = '\n';
+  }
+
   return splitPattern;
 };
 export type SplitPatternType = '\n\n' | '\r\n\r\n' | '\n' | '\r\n';