Merge pull request #129 from fixie-ai/juberti/baseten2

Update baseten to v0.4
fixie-ai · Oct 7, 2024 · 3d4c2f5 · 3d4c2f5
2 parents 6bf7237 + 1ababbc
commit 3d4c2f5
Show file tree

Hide file tree

Showing 2 changed files with 6 additions and 3 deletions.
diff --git a/llm_benchmark_suite.py b/llm_benchmark_suite.py
@@ -492,9 +492,9 @@ def _audio_models():
         _UltravoxLlm("fixie-ai/ultravox-v0.4", "ultravox-v0.4-8b"),
         _UltravoxLlm("fixie-ai/ultravox-70B", "ultravox-v0.4-70b"),
         _Llm(
-            "fixie-ai/ultravox-v0.2",
-            "baseten.co/ultravox-v0.2",
-            base_url="https://bridge.baseten.co/5wovovzq/v1/direct",
+            "ultravox",
+            "baseten.co/ultravox-v0.4",
+            base_url="https://bridge.baseten.co/v1/direct",
             api_key=os.getenv("BASETEN_API_KEY"),
         ),
     ]

diff --git a/llm_request.py b/llm_request.py
@@ -319,6 +319,9 @@ async def openai_chat(ctx: ApiContext, path: str = "/chat/completions") -> ApiRe
     # See https://github.com/Azure/azure-rest-api-specs/issues/25062
     if not any(p in ctx.name for p in ["azure", "databricks", "fireworks", "mistral"]):
         kwargs["stream_options"] = {"include_usage": True}
+    # Hack to identify our baseten deployment, which isn't contained in the URL.
+    if ctx.name.startswith("baseten"):
+        kwargs["baseten"] = {"model_id": "rwn2v41w"}
     data = make_openai_chat_body(ctx, **kwargs)
     return await post(ctx, url, headers, data, openai_chunk_gen)