provider: SambaNova FastAPI implementation.

src/globals.ts

@@ -67,6 +67,7 @@ export const GITHUB: string = 'github';
 export const DEEPBRICKS: string = 'deepbricks';
 export const SILICONFLOW: string = 'siliconflow';
 export const CEREBRAS: string = 'cerebras';
+export const SAMBANOVA: string = 'sambanova';
 
 export const VALID_PROVIDERS = [
  ANTHROPIC,
@@ -108,6 +109,7 @@ export const VALID_PROVIDERS = [
  SILICONFLOW,
  HUGGING_FACE,
  CEREBRAS,
+ SAMBANOVA,
 ];
 
 export const CONTENT_TYPES = {

src/providers/index.ts

@@ -40,6 +40,7 @@ import DeepbricksConfig from './deepbricks';
 import SiliconFlowConfig from './siliconflow';
 import HuggingfaceConfig from './huggingface';
 import { cerebrasProviderAPIConfig } from './cerebras';
+import SambaNovaConfig from './sambanova';
 
 const Providers: { [key: string]: ProviderConfigs } = {
  openai: OpenAIConfig,
@@ -81,6 +82,7 @@ const Providers: { [key: string]: ProviderConfigs } = {
  deepbricks: DeepbricksConfig,
  siliconflow: SiliconFlowConfig,
  cerebras: cerebrasProviderAPIConfig,
+ sambanova: SambaNovaConfig,
 };
 
 export default Providers;

src/providers/sambanova/api.ts

@@ -0,0 +1,19 @@
+import { ProviderAPIConfig } from '../types';
+
+const SambaNovaAPIConfig: ProviderAPIConfig = {
+ getBaseURL: ({ providerOptions }) =>
+ providerOptions.urlToFetch || 'https://fast-api.snova.ai/v1',
+ headers: ({ providerOptions }) => {
+ return { Authorization: `Basic ${providerOptions.apiKey}` };
+ },
+ getEndpoint: ({ fn }) => {
+ switch (fn) {
+ case 'chatComplete':
+ return '/chat/completions';
+ default:
+ return '';
+ }
+ },
+};
+
+export default SambaNovaAPIConfig;

src/providers/sambanova/chatComplete.ts

@@ -0,0 +1,114 @@
+import { SAMBANOVA } from '../../globals';
+import {
+ ChatCompletionResponse,
+ ErrorResponse,
+ ProviderConfig,
+} from '../types';
+
+export const SamvaNovaChatCompleteConfig: ProviderConfig = {
+ model: {
+ param: 'model',
+ required: true,
+ default: 'Meta-Llama-3.1-8B-Instruct',
+ },
+ messages: {
+ param: 'messages',
+ default: '',
+ },
+ max_tokens: {
+ param: 'max_tokens',
+ default: 100,
+ min: 0,
+ },
+ // Currently FastAPI supports stream responses only.
+ stream: {
+ param: 'stream',
+ default: true,
+ },
+ stream_options: {
+ param: 'stream_options',
+ },
+ stop: {
+ param: 'stop',
+ },
+};
+
+export interface SamvaNovaChatCompleteResponse extends ChatCompletionResponse {}
+
+export interface SamvaNovaErrorResponse extends ErrorResponse {}
+
+export interface SamvaNovaStreamChunk {
+ id: string;
+ object: string;
+ created: number;
+ model: string;
+ system_fingerprint: string;
+ choices: {
+ delta: {
+ content?: string;
+ };
+ index: number;
+ finish_reason: string | null;
+ logprobs: object | null;
+ }[];
+ usage: {
+ is_last_response: boolean;
+ total_tokens: number;
+ prompt_tokens: number;
+ completion_tokens: number;
+ time_to_first_token: number;
+ end_time: number;
+ start_time: number;
+ total_latency: number;
+ total_tokens_per_sec: number;
+ completion_tokens_per_sec: number;
+ completion_tokens_after_first_per_sec: number;
+ completion_tokens_after_first_per_sec_first_ten: number;
+ } | null;
+}
+
+export const SamvaNovaChatCompleteStreamChunkTransform: (
+ response: string
+) => string = (responseChunk) => {
+ let chunk = responseChunk.trim();
+ chunk = chunk.replace(/^data: /, '');
+ chunk = chunk.trim();
+ if (chunk === '[DONE]') {
+ return `data: ${chunk}\n\n`;
+ }
+
+ const parsedChunk: SamvaNovaStreamChunk = JSON.parse(chunk);
+ if (parsedChunk.usage) {
+ return `data: ${JSON.stringify({
+ id: parsedChunk.id,
+ object: parsedChunk.object,
+ created: parsedChunk.created,
+ model: parsedChunk.model,
+ provider: SAMBANOVA,
+ choices: [],
+ usage: {
+ prompt_tokens: parsedChunk.usage.prompt_tokens || 0,
+ completion_tokens: parsedChunk.usage.completion_tokens || 0,
+ total_tokens: parsedChunk.usage.total_tokens || 0,
+ },
+ })}\n\n`;
+ }
+ return `data: ${JSON.stringify({
+ id: parsedChunk.id,
+ object: parsedChunk.object,
+ created: parsedChunk.created,
+ model: parsedChunk.model,
+ provider: SAMBANOVA,
+ choices: [
+ {
+ index: parsedChunk.choices[0].index || 0,
+ delta: {
+ role: 'assistant',
+ content: parsedChunk.choices[0].delta.content,
+ },
+ logprobs: null,
+ finish_reason: parsedChunk.choices[0].finish_reason || null,
+ },
+ ],
+ })}\n\n`;
+};

src/providers/sambanova/index.ts

@@ -0,0 +1,16 @@
+import { ProviderConfigs } from '../types';
+import SambaNovaAPIConfig from './api';
+import {
+ SamvaNovaChatCompleteConfig,
+ SamvaNovaChatCompleteStreamChunkTransform,
+} from './chatComplete';
+
+const SambaNovaConfig: ProviderConfigs = {
+ chatComplete: SamvaNovaChatCompleteConfig,
+ api: SambaNovaAPIConfig,
+ responseTransforms: {
+ 'stream-chatComplete': SamvaNovaChatCompleteStreamChunkTransform,
+ },
+};
+
+export default SambaNovaConfig;