Feat/self hosted models

examples/_config.yaml

@@ -17,6 +17,11 @@ providers:
         max_tokens: 200000
         input_token_cost: 0.000003
         output_token_cost: 0.000015
+      claude-3-sonnet:
+        mode: chat
+        max_tokens: 200000
+        input_token_cost: 0.000003
+        output_token_cost: 0.000015
       claude-3-haiku-20240307:
         mode: chat
         max_tokens: 200000
@@ -66,18 +71,134 @@ providers:
         min: 0
         max: 500
         step: 1
-  ollama:
-    id: ollama
-    name: Ollama
+  bedrock:
+    id: bedrock
+    name: Bedrock ConverseAPI
     chat: true
     embed: true
     keys:
+      - BEDROCK_SECRET_KEY
+      - BEDROCK_ACCESS_KEY
+      - BEDROCK_REGION
     models:
-      llama2:
+      anthropic.claude-3-sonnet-20240229-v1:0:
+        mode: chat
+        max_tokens: 200000
+        input_token_cost: 0.000003
+        output_token_cost: 0.000015
+      anthropic.claude-3-5-sonnet-20240620-v1:0:
+        mode: chat
+        max_tokens: 200000
+        input_token_cost: 0.000003
+        output_token_cost: 0.000015
+      anthropic.claude-3-5-sonnet-20241022-v2:0:
+        mode: chat
+        max_tokens: 200000
+        input_token_cost: 0.000003
+        output_token_cost: 0.000015
+      anthropic.claude-3-haiku-20240307-v1:0:
+        mode: chat
+        max_tokens: 200000
+        input_token_cost: 0.00000025
+        output_token_cost: 0.00000125
+      anthropic.claude-3-5-haiku-20241022-v1:0:
+        mode: chat
+        max_tokens: 200000
+        input_token_cost: 0.000001
+        output_token_cost: 0.000005
+      anthropic.claude-3-opus-20240229-v1:0:
+        mode: chat
+        max_tokens: 200000
+        input_token_cost: 0.000015
+        output_token_cost: 0.000075
+      anthropic.claude-instant-v1:
         mode: chat
-        max_tokens: 0
+        max_tokens: 100000
+        input_token_cost: 0.0000008
+        output_token_cost: 0.000024
+      anthropic.claude-v2:
+        mode: chat
+        max_tokens: 100000
+        input_token_cost: 0.000008
+        output_token_cost: 0.000024
+      anthropic.claude-v2:1:
+        mode: chat
+        max_tokens: 100000
+        input_token_cost: 0.000008
+        output_token_cost: 0.000024
+      us.amazon.nova-pro-v1:0:
+        mode: chat
+        max_tokens: 300000
+        input_token_cost: 0.0000008
+        output_token_cost: 0.0000016
+      us.amazon.nova-lite-v1:0:
+        mode: chat
+        max_tokens: 300000
+        input_token_cost: 0.00000006
+        output_token_cost: 0.00000012
+      us.amazon.nova-micro-v1:0:
+        mode: chat
+        max_tokens: 128000
+        input_token_cost: 0.000000035
+        output_token_cost: 0.00000007
+
+    parameters:
+      temperature:
+        name: "Temperature"
+        type: float
+        default: 1
+        min: 0
+        max: 1
+        step: 0.01
+      max_tokens:
+        name: "Maximum tokens"
+        type: float
+        default: 256
+        min: 1
+        max: 4096
+        step: 0.01
+      top_p:
+        name: "Top P"
+        type: float
+        default: 1
+        min: 0
+        max: 1
+        step: 0.01
+      top_k:
+        name: "Top K"
+        type: float
+        default: 5
+        min: 0
+        max: 500
+        step: 1
+  self-hosted:
+    id: self-hosted
+    name: Self Hosted
+    chat: true
+    embed: true
+    keys:
+    models:
+      deepseek-r1:1.5b:
+        mode: chat
+        max_tokens: 200000
         input_token_cost: 0
         output_token_cost: 0
+
+      deepseek-r1-tool-calling:
+        mode: chat
+        max_tokens: 128000
+        input_token_cost: 0
+        output_token_cost: 0
+      llama3.2:
+        mode: chat
+        max_tokens: 200000
+        input_token_cost: 0
+        output_token_cost: 0
+      Llama-3-3-70B-Instruct-llmstudio:
+        mode: chat
+        max_tokens: 200000
+        input_token_cost: 0.00000071
+        output_token_cost: 0.00000071
     parameters:
       temperature:
         name: "Temperature"
@@ -115,6 +236,24 @@ providers:
     keys:
       - OPENAI_API_KEY
     models:
+      o1-preview:
+        mode: chat
+        max_completion_tokens: 128000
+        input_token_cost: 0.000015
+        cached_token_cost: 0.0000075
+        output_token_cost: 0.000060
+      o1-mini:
+        mode: chat
+        max_completion_tokens: 128000
+        input_token_cost: 0.0000011
+        cached_token_cost: 0.00000055
+        output_token_cost: 0.0000044
+      o3-mini:
+        mode: chat
+        max_completion_tokens: 200000
+        input_token_cost: 0.0000011
+        cached_token_cost: 0.00000055
+        output_token_cost: 0.0000044
       o1-preview:
         mode: chat
         max_completion_tokens: 128000
@@ -204,6 +343,18 @@ providers:
       - AZURE_API_ENDPOINT
       - AZURE_API_VERSION
     models:
+      o1-preview:
+        mode: chat
+        max_completion_tokens: 128000
+        input_token_cost: 0.0000165
+        cached_token_cost: 0.00000825
+        output_token_cost: 0.000066
+      o1-mini:
+        mode: chat
+        max_completion_tokens: 128000
+        input_token_cost: 0.0000033
+        cached_token_cost: 0.00000165
+        output_token_cost: 0.0000132
       gpt-4o-mini:
         mode: chat
         max_tokens: 128000
@@ -212,8 +363,9 @@ providers:
       gpt-4o:
         mode: chat
         max_tokens: 128000
-        input_token_cost: 0.000005
-        output_token_cost: 0.000015
+        input_token_cost: 0.0000025
+        cached_token_cost: 0.00000125
+        output_token_cost: 0.00001
       gpt-4-turbo:
         mode: chat
         max_tokens: 128000

examples/core.py

@@ -5,10 +5,12 @@
 from pprint import pprint
 import os
 import asyncio
+import asyncio
 from dotenv import load_dotenv
 load_dotenv()
 
-def run_provider(provider, model, api_key=None, **kwargs):
+def run_provider(provider, model, api_key=None=None, **kwargs):
+    print(f"\n\n###RUNNING for <{provider}>, <{model}> ###")
     print(f"\n\n###RUNNING for <{provider}>, <{model}> ###")
     llm = LLMCore(provider=provider, api_key=api_key, **kwargs)
 
@@ -58,7 +60,7 @@ def run_provider(provider, model, api_key=None, **kwargs):
 
     print("\nAsync Stream")
     async def async_stream():
-        chat_request = build_chat_request(model, chat_input="Hello, my name is Tom Json", is_stream=True)
+        chat_request = build_chat_request(model, chat_input="Hello, my name is Tom", is_stream=True)
 
         response_async = await llm.achat(**chat_request)
         async for p in response_async:
@@ -74,15 +76,15 @@ async def async_stream():
 
 
     print("\nSync Non-Stream")
-    chat_request = build_chat_request(model, chat_input="Hello, my name is Alice Json", is_stream=False)
+    chat_request = build_chat_request(model, chat_input="Hello, my name is Alice", is_stream=False)
 
     response_sync = llm.chat(**chat_request)
     pprint(response_sync)
     latencies["sync (ms)"]= response_sync.metrics["latency_s"]*1000
 
 
     print("\nSync Stream")
-    chat_request = build_chat_request(model, chat_input="Hello, my name is Mary Json", is_stream=True)
+    chat_request = build_chat_request(model, chat_input="Hello, my name is Mary", is_stream=True)
 
     response_sync_stream = llm.chat(**chat_request)
     for p in response_sync_stream:
@@ -126,7 +128,6 @@ def build_chat_request(model: str, chat_input: str, is_stream: bool, max_tokens:
             "parameters": {
                 "temperature": 0,
                 "max_tokens": max_tokens,
-                "response_format": {"type": "json_object"},
                 "functions": None,
             }
         }
@@ -138,29 +139,75 @@ def multiple_provider_runs(provider:str, model:str, num_runs:int, api_key:str, *
         latencies = run_provider(provider=provider, model=model, api_key=api_key, **kwargs)
         pprint(latencies)
 
-
+def run_chat_all_providers():    
+    # OpenAI
+    multiple_provider_runs(provider="openai", model="gpt-4o-mini", api_key=os.environ["OPENAI_API_KEY"], num_runs=1)
+    multiple_provider_runs(provider="openai", model="o3-mini", api_key=os.environ["OPENAI_API_KEY"], num_runs=1)
+    #multiple_provider_runs(provider="openai", model="o1-preview", api_key=os.environ["OPENAI_API_KEY"], num_runs=1)
+
 
-# OpenAI
-multiple_provider_runs(provider="openai", model="gpt-4o-mini", api_key=os.environ["OPENAI_API_KEY"], num_runs=1)
-multiple_provider_runs(provider="openai", model="o3-mini", api_key=os.environ["OPENAI_API_KEY"], num_runs=1)
-#multiple_provider_runs(provider="openai", model="o1-preview", api_key=os.environ["OPENAI_API_KEY"], num_runs=1)
+    # Azure
+    multiple_provider_runs(provider="azure", model="gpt-4o-mini", num_runs=1, api_key=os.environ["AZURE_API_KEY"], api_version=os.environ["AZURE_API_VERSION"], api_endpoint=os.environ["AZURE_API_ENDPOINT"])
+    #multiple_provider_runs(provider="azure", model="gpt-4o", num_runs=1, api_key=os.environ["AZURE_API_KEY"], api_version=os.environ["AZURE_API_VERSION"], api_endpoint=os.environ["AZURE_API_ENDPOINT"])
+    #multiple_provider_runs(provider="azure", model="o1-mini", num_runs=1, api_key=os.environ["AZURE_API_KEY"], api_version=os.environ["AZURE_API_VERSION"], api_endpoint=os.environ["AZURE_API_ENDPOINT"])
+    #multiple_provider_runs(provider="azure", model="o1-preview", num_runs=1, api_key=os.environ["AZURE_API_KEY"], api_version=os.environ["AZURE_API_VERSION"], api_endpoint=os.environ["AZURE_API_ENDPOINT"])
 
 
-# Azure
-multiple_provider_runs(provider="azure", model="gpt-4o-mini", num_runs=1, api_key=os.environ["AZURE_API_KEY"], api_version=os.environ["AZURE_API_VERSION"], api_endpoint=os.environ["AZURE_API_ENDPOINT"])
-#multiple_provider_runs(provider="azure", model="gpt-4o", num_runs=1, api_key=os.environ["AZURE_API_KEY"], api_version=os.environ["AZURE_API_VERSION"], api_endpoint=os.environ["AZURE_API_ENDPOINT"])
-#multiple_provider_runs(provider="azure", model="o1-mini", num_runs=1, api_key=os.environ["AZURE_API_KEY"], api_version=os.environ["AZURE_API_VERSION"], api_endpoint=os.environ["AZURE_API_ENDPOINT"])
-#multiple_provider_runs(provider="azure", model="o1-preview", num_runs=1, api_key=os.environ["AZURE_API_KEY"], api_version=os.environ["AZURE_API_VERSION"], api_endpoint=os.environ["AZURE_API_ENDPOINT"])
+    #multiple_provider_runs(provider="anthropic", model="claude-3-opus-20240229", num_runs=1, api_key=os.environ["ANTHROPIC_API_KEY"])
 
+    #multiple_provider_runs(provider="azure", model="o1-preview", num_runs=1, api_key=os.environ["AZURE_API_KEY"], api_version=os.environ["AZURE_API_VERSION"], api_endpoint=os.environ["AZURE_API_ENDPOINT"])
+    #multiple_provider_runs(provider="azure", model="o1-mini", num_runs=1, api_key=os.environ["AZURE_API_KEY"], api_version=os.environ["AZURE_API_VERSION"], api_endpoint=os.environ["AZURE_API_ENDPOINT"])
 
-#multiple_provider_runs(provider="anthropic", model="claude-3-opus-20240229", num_runs=1, api_key=os.environ["ANTHROPIC_API_KEY"])
 
-#multiple_provider_runs(provider="azure", model="o1-preview", num_runs=1, api_key=os.environ["AZURE_API_KEY"], api_version=os.environ["AZURE_API_VERSION"], api_endpoint=os.environ["AZURE_API_ENDPOINT"])
-#multiple_provider_runs(provider="azure", model="o1-mini", num_runs=1, api_key=os.environ["AZURE_API_KEY"], api_version=os.environ["AZURE_API_VERSION"], api_endpoint=os.environ["AZURE_API_ENDPOINT"])
+    multiple_provider_runs(provider="vertexai", model="gemini-1.5-flash", num_runs=1, api_key=os.environ["GOOGLE_API_KEY"])
 
+    # Bedrock
+    multiple_provider_runs(provider="bedrock", model="us.amazon.nova-lite-v1:0", num_runs=1, api_key=None, region=os.environ["BEDROCK_REGION"], secret_key=os.environ["BEDROCK_SECRET_KEY"], access_key=os.environ["BEDROCK_ACCESS_KEY"])
+    #multiple_provider_runs(provider="bedrock", model="anthropic.claude-3-5-sonnet-20241022-v2:0", num_runs=1, api_key=None, region=os.environ["BEDROCK_REGION"], secret_key=os.environ["BEDROCK_SECRET_KEY"], access_key=os.environ["BEDROCK_ACCESS_KEY"])
 
-multiple_provider_runs(provider="vertexai", model="gemini-1.5-flash", num_runs=1, api_key=os.environ["GOOGLE_API_KEY"])
+run_chat_all_providers()
 
-# Bedrock
-multiple_provider_runs(provider="bedrock", model="us.amazon.nova-lite-v1:0", num_runs=1, api_key=None, region=os.environ["BEDROCK_REGION"], secret_key=os.environ["BEDROCK_SECRET_KEY"], access_key=os.environ["BEDROCK_ACCESS_KEY"])
-#multiple_provider_runs(provider="bedrock", model="anthropic.claude-3-5-sonnet-20241022-v2:0", num_runs=1, api_key=None, region=os.environ["BEDROCK_REGION"], secret_key=os.environ["BEDROCK_SECRET_KEY"], access_key=os.environ["BEDROCK_ACCESS_KEY"])
+
+import base64
+
+def messages(img_path):
+    """
+    Creates a message payload with both text and image.
+    Adapts format based on the provider.
+    """
+    with open(img_path, "rb") as f:
+        image_bytes = f.read()
+
+    base64_image = base64.b64encode(image_bytes).decode("utf-8")
+    return [
+        {
+            "role": "user",
+            "content": [
+                {"type": "text", "text": "What's in this image?"},
+                {
+                    "type": "image_url",
+                    "image_url": {"url": f"data:image/jpeg;base64,{base64_image}"},
+                },
+                {
+                    "type": "image_url",
+                    "image_url": {"url": "https://awsmp-logos.s3.amazonaws.com/seller-zx4pk43qpmxoa/53d235806f343cec94aac3c577d81c13.png"},
+                },
+            ],
+        }
+    ]
+
+def run_send_imgs():
+    provider="bedrock"
+    model="us.amazon.nova-lite-v1:0"
+    chat_input=messages(img_path="./libs/llmstudio/tests/integration_tests/test_data/llmstudio-logo.jpeg")
+    chat_request = build_chat_request(model=model, chat_input=chat_input, is_stream=False)
+    llm = LLMCore(provider=provider, api_key=os.environ["OPENAI_API_KEY"], region=os.environ["BEDROCK_REGION"], secret_key=os.environ["BEDROCK_SECRET_KEY"], access_key=os.environ["BEDROCK_ACCESS_KEY"])
+    response_sync = llm.chat(**chat_request)
+    #print(response_sync)
+    response_sync.clean_print()
+
+    #for p in response_sync:
+    #    if p.metrics:
+    #        p.clean_print()
+
+run_send_imgs()