Skip to content

Commit

Permalink
(refactor) litellm.Router client initialization utils (#6394)
Browse files Browse the repository at this point in the history
* refactor InitalizeOpenAISDKClient

* use helper func for _should_create_openai_sdk_client_for_model

* use static methods for set client on litellm router

* reduce LOC in _get_client_initialization_params

* fix _should_create_openai_sdk_client_for_model

* code quality fix

* test test_should_create_openai_sdk_client_for_model

* test test_get_client_initialization_params_openai

* fix mypy linting errors

* fix OpenAISDKClientInitializationParams

* test_get_client_initialization_params_all_env_vars

* test_get_client_initialization_params_azure_ai_studio_mistral

* test_get_client_initialization_params_default_values

* fix _get_client_initialization_params
  • Loading branch information
ishaan-jaff authored Oct 23, 2024
1 parent 3991d75 commit b70147f
Show file tree
Hide file tree
Showing 3 changed files with 836 additions and 430 deletions.
23 changes: 14 additions & 9 deletions litellm/router.py
Original file line number Diff line number Diff line change
Expand Up @@ -63,10 +63,7 @@
_get_router_metadata_variable_name,
replace_model_in_jsonl,
)
from litellm.router_utils.client_initalization_utils import (
set_client,
should_initialize_sync_client,
)
from litellm.router_utils.client_initalization_utils import InitalizeOpenAISDKClient
from litellm.router_utils.cooldown_cache import CooldownCache
from litellm.router_utils.cooldown_callbacks import router_cooldown_event_callback
from litellm.router_utils.cooldown_handlers import (
Expand Down Expand Up @@ -3951,7 +3948,7 @@ def _add_deployment(self, deployment: Deployment) -> Deployment:
raise Exception(f"Unsupported provider - {custom_llm_provider}")

# init OpenAI, Azure clients
set_client(
InitalizeOpenAISDKClient.set_client(
litellm_router_instance=self, model=deployment.to_json(exclude_none=True)
)

Expand Down Expand Up @@ -4661,7 +4658,9 @@ def _get_client(self, deployment, kwargs, client_type=None):
"""
Re-initialize the client
"""
set_client(litellm_router_instance=self, model=deployment)
InitalizeOpenAISDKClient.set_client(
litellm_router_instance=self, model=deployment
)
client = self.cache.get_cache(key=cache_key, local_only=True)
return client
else:
Expand All @@ -4671,7 +4670,9 @@ def _get_client(self, deployment, kwargs, client_type=None):
"""
Re-initialize the client
"""
set_client(litellm_router_instance=self, model=deployment)
InitalizeOpenAISDKClient.set_client(
litellm_router_instance=self, model=deployment
)
client = self.cache.get_cache(key=cache_key, local_only=True)
return client
else:
Expand All @@ -4682,7 +4683,9 @@ def _get_client(self, deployment, kwargs, client_type=None):
"""
Re-initialize the client
"""
set_client(litellm_router_instance=self, model=deployment)
InitalizeOpenAISDKClient.set_client(
litellm_router_instance=self, model=deployment
)
client = self.cache.get_cache(key=cache_key)
return client
else:
Expand All @@ -4692,7 +4695,9 @@ def _get_client(self, deployment, kwargs, client_type=None):
"""
Re-initialize the client
"""
set_client(litellm_router_instance=self, model=deployment)
InitalizeOpenAISDKClient.set_client(
litellm_router_instance=self, model=deployment
)
client = self.cache.get_cache(key=cache_key)
return client

Expand Down
Loading

0 comments on commit b70147f

Please sign in to comment.