diff --git a/.gitignore b/.gitignore index 3009cea1cd39..47fccd9c5d7a 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ .env litellm_uuid.txt __pycache__/ +*.pyc bun.lockb **/.DS_Store .aider* @@ -10,3 +11,4 @@ secrets.toml .gitignore litellm/proxy/litellm_secrets.toml litellm/proxy/api_log.json +.idea/ \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 8c8140ffc3b5..be162d4511fe 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,15 +1,13 @@ FROM python:3.10 -RUN pip install poetry - -WORKDIR /app - -COPY . . - +COPY . /app +WORKDIR /app +RUN mkdir -p /root/.config/litellm/ && cp /app/secrets_template.toml /root/.config/litellm/litellm.secrets.toml RUN pip install -r requirements.txt WORKDIR /app/litellm/proxy - -RUN python proxy_cli.py --config -f /app/secrets_template.toml - -RUN python proxy_cli.py \ No newline at end of file +EXPOSE 8000 +ENTRYPOINT [ "python3", "proxy_cli.py" ] +# TODO - Set up a GitHub Action to automatically create the Docker image, +# and then we can quickly deploy the litellm proxy in the following way +# `docker run -p 8000:8000 -v ./secrets_template.toml:/root/.config/litellm/litellm.secrets.toml ghcr.io/BerriAI/litellm:v0.8.4` \ No newline at end of file diff --git a/litellm/__pycache__/__init__.cpython-311.pyc b/litellm/__pycache__/__init__.cpython-311.pyc deleted file mode 100644 index 2c81620388cc..000000000000 Binary files a/litellm/__pycache__/__init__.cpython-311.pyc and /dev/null differ diff --git a/litellm/__pycache__/main.cpython-311.pyc b/litellm/__pycache__/main.cpython-311.pyc deleted file mode 100644 index f990b64c1e8c..000000000000 Binary files a/litellm/__pycache__/main.cpython-311.pyc and /dev/null differ diff --git a/litellm/__pycache__/timeout.cpython-311.pyc b/litellm/__pycache__/timeout.cpython-311.pyc deleted file mode 100644 index 60c6b65bdfb5..000000000000 Binary files a/litellm/__pycache__/timeout.cpython-311.pyc and /dev/null differ diff --git a/litellm/__pycache__/utils.cpython-311.pyc b/litellm/__pycache__/utils.cpython-311.pyc deleted file mode 100644 index 7a371d8144f9..000000000000 Binary files a/litellm/__pycache__/utils.cpython-311.pyc and /dev/null differ diff --git a/litellm/integrations/__pycache__/__init__.cpython-311.pyc b/litellm/integrations/__pycache__/__init__.cpython-311.pyc deleted file mode 100644 index 5bd4bfeb856c..000000000000 Binary files a/litellm/integrations/__pycache__/__init__.cpython-311.pyc and /dev/null differ diff --git a/litellm/integrations/__pycache__/aispend.cpython-311.pyc b/litellm/integrations/__pycache__/aispend.cpython-311.pyc deleted file mode 100644 index 111b4eba1be8..000000000000 Binary files a/litellm/integrations/__pycache__/aispend.cpython-311.pyc and /dev/null differ diff --git a/litellm/integrations/__pycache__/berrispend.cpython-311.pyc b/litellm/integrations/__pycache__/berrispend.cpython-311.pyc deleted file mode 100644 index 616f227eb0f0..000000000000 Binary files a/litellm/integrations/__pycache__/berrispend.cpython-311.pyc and /dev/null differ diff --git a/litellm/integrations/__pycache__/helicone.cpython-311.pyc b/litellm/integrations/__pycache__/helicone.cpython-311.pyc deleted file mode 100644 index 972c339ed39a..000000000000 Binary files a/litellm/integrations/__pycache__/helicone.cpython-311.pyc and /dev/null differ diff --git a/litellm/integrations/__pycache__/supabase.cpython-311.pyc b/litellm/integrations/__pycache__/supabase.cpython-311.pyc deleted file mode 100644 index ab916e5227aa..000000000000 Binary files a/litellm/integrations/__pycache__/supabase.cpython-311.pyc and /dev/null differ diff --git a/litellm/proxy/proxy_cli.py b/litellm/proxy/proxy_cli.py index bd8ef8cc73b1..6c0afa6956fb 100644 --- a/litellm/proxy/proxy_cli.py +++ b/litellm/proxy/proxy_cli.py @@ -7,7 +7,6 @@ import operator config_filename = "litellm.secrets.toml" -pkg_config_filename = "template.secrets.toml" # Using appdirs to determine user-specific config path config_dir = appdirs.user_config_dir("litellm") user_config_path = os.path.join(config_dir, config_filename) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 6d67ccdcc3ce..8a89f90e36b5 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -1,12 +1,12 @@ import sys, os, platform, time, copy import threading import shutil, random, traceback + messages = [] sys.path.insert( 0, os.path.abspath("../..") ) # Adds the parent directory to the system path - for litellm local dev - try: import uvicorn import fastapi @@ -23,13 +23,14 @@ import tomli as tomllib import appdirs import tomli_w - + try: from .llm import litellm_completion -except ImportError as e: +except ImportError as e: from llm import litellm_completion import random + list_of_messages = [ "'The thing I wish you improved is...'", "'A feature I really want is...'", @@ -38,35 +39,36 @@ "'I don't like how this works...'", "'It would help me if you could add...'", "'This feature doesn't meet my needs because...'", - "'I get frustrated when the product...'", + "'I get frustrated when the product...'", ] + def generate_feedback_box(): - box_width = 60 - - # Select a random message - message = random.choice(list_of_messages) - - print() - print('\033[1;37m' + '#' + '-'*box_width + '#\033[0m') - print('\033[1;37m' + '#' + ' '*box_width + '#\033[0m') - print('\033[1;37m' + '# {:^59} #\033[0m'.format(message)) - print('\033[1;37m' + '# {:^59} #\033[0m'.format('https://github.com/BerriAI/litellm/issues/new')) - print('\033[1;37m' + '#' + ' '*box_width + '#\033[0m') - print('\033[1;37m' + '#' + '-'*box_width + '#\033[0m') - print() - print(' Thank you for using LiteLLM! - Krrish & Ishaan') - print() - print() + box_width = 60 -generate_feedback_box() + # Select a random message + message = random.choice(list_of_messages) + + print() + print('\033[1;37m' + '#' + '-' * box_width + '#\033[0m') + print('\033[1;37m' + '#' + ' ' * box_width + '#\033[0m') + print('\033[1;37m' + '# {:^59} #\033[0m'.format(message)) + print('\033[1;37m' + '# {:^59} #\033[0m'.format('https://github.com/BerriAI/litellm/issues/new')) + print('\033[1;37m' + '#' + ' ' * box_width + '#\033[0m') + print('\033[1;37m' + '#' + '-' * box_width + '#\033[0m') + print() + print(' Thank you for using LiteLLM! - Krrish & Ishaan') + print() + print() +generate_feedback_box() + print() print("\033[1;31mGive Feedback / Get Help: https://github.com/BerriAI/litellm/issues/new\033[0m") print() print("\033[1;34mDocs: https://docs.litellm.ai/docs/proxy_server\033[0m") -print() +print() import litellm from fastapi import FastAPI, Request @@ -102,11 +104,14 @@ def generate_feedback_box(): config_dir = appdirs.user_config_dir("litellm") user_config_path = os.path.join(config_dir, config_filename) log_file = 'api_log.json' + + #### HELPER FUNCTIONS #### def print_verbose(print_statement): - global user_debug - if user_debug: - print(print_statement) + global user_debug + if user_debug: + print(print_statement) + def find_avatar_url(role): role = role.replace(" ", "%20") @@ -114,18 +119,21 @@ def find_avatar_url(role): avatar_url = f"/static/{avatar_filename}" return avatar_url -def usage_telemetry(feature: str): # helps us know if people are using this feature. Set `litellm --telemetry False` to your cli call to turn this off - if user_telemetry: + +def usage_telemetry( + feature: str): # helps us know if people are using this feature. Set `litellm --telemetry False` to your cli call to turn this off + if user_telemetry: data = { - "feature": feature # "local_proxy_server" + "feature": feature # "local_proxy_server" } threading.Thread(target=litellm.utils.litellm_telemetry, args=(data,), daemon=True).start() + def add_keys_to_config(key, value): # Check if file exists if os.path.exists(user_config_path): # Load existing file - with open(user_config_path, "rb") as f: + with open(user_config_path, "rb") as f: config = tomllib.load(f) else: # File doesn't exist, create empty config @@ -138,21 +146,22 @@ def add_keys_to_config(key, value): with open(user_config_path, 'wb') as f: tomli_w.dump(config, f) -def save_params_to_config(data: dict): + +def save_params_to_config(data: dict): # Check if file exists if os.path.exists(user_config_path): # Load existing file - with open(user_config_path, "rb") as f: + with open(user_config_path, "rb") as f: config = tomllib.load(f) else: # File doesn't exist, create empty config config = {} - + config.setdefault('general', {}) ## general config general_settings = data["general"] - + for key, value in general_settings.items(): config["general"][key] = value @@ -170,107 +179,102 @@ def save_params_to_config(data: dict): with open(user_config_path, 'wb') as f: tomli_w.dump(config, f) -def print_cost_logs(): - with open('costs.json', 'r') as f: - # print this in green - print("\033[1;32m") - print(f.read()) - print("\033[0m") - return def load_config(): - try: - global user_config, user_api_base, user_max_tokens, user_temperature, user_model - # As the .env file is typically much simpler in structure, we use load_dotenv here directly - with open(user_config_path, "rb") as f: - user_config = tomllib.load(f) - - ## load keys - if "keys" in user_config: - for key in user_config["keys"]: - os.environ[key] = user_config["keys"][key] # litellm can read keys from the environment - ## settings - if "general" in user_config: - litellm.add_function_to_prompt = user_config["general"].get("add_function_to_prompt", True) # by default add function to prompt if unsupported by provider - litellm.drop_params = user_config["general"].get("drop_params", True) # by default drop params if unsupported by provider - litellm.model_fallbacks = user_config["general"].get("fallbacks", None) # fallback models in case initial completion call fails - default_model = user_config["general"].get("default_model", None) # route all requests to this model. - - if user_model is None: # `litellm --model `` > default_model. - user_model = default_model - - ## load model config - to set this run `litellm --config` - model_config = None - if "model" in user_config: - if user_model in user_config["model"]: - model_config = user_config["model"][user_model] - - print_verbose(f"user_config: {user_config}") - print_verbose(f"model_config: {model_config}") - print_verbose(f"user_model: {user_model}") - if model_config is None: - return - - user_max_tokens = model_config.get("max_tokens", None) - user_temperature = model_config.get("temperature", None) - user_api_base = model_config.get("api_base", None) - - ## custom prompt template - if "prompt_template" in model_config: - model_prompt_template = model_config["prompt_template"] - if len(model_prompt_template.keys()) > 0: # if user has initialized this at all - litellm.register_prompt_template( - model=user_model, - initial_prompt_value=model_prompt_template.get("MODEL_PRE_PROMPT", ""), - roles={ - "system": { - "pre_message": model_prompt_template.get("MODEL_SYSTEM_MESSAGE_START_TOKEN", ""), - "post_message": model_prompt_template.get("MODEL_SYSTEM_MESSAGE_END_TOKEN", ""), - }, - "user": { - "pre_message": model_prompt_template.get("MODEL_USER_MESSAGE_START_TOKEN", ""), - "post_message": model_prompt_template.get("MODEL_USER_MESSAGE_END_TOKEN", ""), - }, - "assistant": { - "pre_message": model_prompt_template.get("MODEL_ASSISTANT_MESSAGE_START_TOKEN", ""), - "post_message": model_prompt_template.get("MODEL_ASSISTANT_MESSAGE_END_TOKEN", ""), - } - }, - final_prompt_value=model_prompt_template.get("MODEL_POST_PROMPT", ""), - ) - except Exception as e: - pass + global user_config, user_api_base, user_max_tokens, user_temperature, user_model + # As the .env file is typically much simpler in structure, we use load_dotenv here directly + with open(user_config_path, "rb") as f: + user_config = tomllib.load(f) + + ## load keys + if "keys" in user_config: + for key in user_config["keys"]: + os.environ[key] = user_config["keys"][key] # litellm can read keys from the environment + ## settings + if "general" in user_config: + litellm.add_function_to_prompt = user_config["general"].get("add_function_to_prompt", + True) # by default add function to prompt if unsupported by provider + litellm.drop_params = user_config["general"].get("drop_params", + True) # by default drop params if unsupported by provider + litellm.model_fallbacks = user_config["general"].get("fallbacks", + None) # fallback models in case initial completion call fails + default_model = user_config["general"].get("default_model", None) # route all requests to this model. + + if user_model is None: # `litellm --model `` > default_model. + user_model = default_model + + ## load model config - to set this run `litellm --config` + model_config = None + if "model" in user_config: + if user_model in user_config["model"]: + model_config = user_config["model"][user_model] + + print_verbose(f"user_config: {user_config}") + print_verbose(f"model_config: {model_config}") + print_verbose(f"user_model: {user_model}") + if model_config is None: + return + + user_max_tokens = model_config.get("max_tokens", None) + user_temperature = model_config.get("temperature", None) + user_api_base = model_config.get("api_base", None) + + ## custom prompt template + if "prompt_template" in model_config: + model_prompt_template = model_config["prompt_template"] + if len(model_prompt_template.keys()) > 0: # if user has initialized this at all + litellm.register_prompt_template( + model=user_model, + initial_prompt_value=model_prompt_template.get("MODEL_PRE_PROMPT", ""), + roles={ + "system": { + "pre_message": model_prompt_template.get("MODEL_SYSTEM_MESSAGE_START_TOKEN", ""), + "post_message": model_prompt_template.get("MODEL_SYSTEM_MESSAGE_END_TOKEN", ""), + }, + "user": { + "pre_message": model_prompt_template.get("MODEL_USER_MESSAGE_START_TOKEN", ""), + "post_message": model_prompt_template.get("MODEL_USER_MESSAGE_END_TOKEN", ""), + }, + "assistant": { + "pre_message": model_prompt_template.get("MODEL_ASSISTANT_MESSAGE_START_TOKEN", ""), + "post_message": model_prompt_template.get("MODEL_ASSISTANT_MESSAGE_END_TOKEN", ""), + } + }, + final_prompt_value=model_prompt_template.get("MODEL_POST_PROMPT", ""), + ) + -def initialize(model, alias, api_base, debug, temperature, max_tokens, max_budget, telemetry, drop_params, add_function_to_prompt, headers, save): +def initialize(model, alias, api_base, debug, temperature, max_tokens, max_budget, telemetry, drop_params, + add_function_to_prompt, headers, save): global user_model, user_api_base, user_debug, user_max_tokens, user_temperature, user_telemetry, user_headers user_model = model user_debug = debug load_config() - dynamic_config = {"general": {}, user_model: {}} - if headers: # model-specific param + dynamic_config = {"general": {}, user_model: {}} + if headers: # model-specific param user_headers = headers dynamic_config[user_model]["headers"] = headers - if api_base: # model-specific param + if api_base: # model-specific param user_api_base = api_base dynamic_config[user_model]["api_base"] = api_base - if max_tokens: # model-specific param + if max_tokens: # model-specific param user_max_tokens = max_tokens dynamic_config[user_model]["max_tokens"] = max_tokens - if temperature: # model-specific param + if temperature: # model-specific param user_temperature = temperature dynamic_config[user_model]["temperature"] = temperature - if alias: # model-specific param + if alias: # model-specific param dynamic_config[user_model]["alias"] = alias - if drop_params == True: # litellm-specific param + if drop_params == True: # litellm-specific param litellm.drop_params = True dynamic_config["general"]["drop_params"] = True - if add_function_to_prompt == True: # litellm-specific param + if add_function_to_prompt == True: # litellm-specific param litellm.add_function_to_prompt = True dynamic_config["general"]["add_function_to_prompt"] = True - if max_budget: # litellm-specific param + if max_budget: # litellm-specific param litellm.max_budget = max_budget dynamic_config["general"]["max_budget"] = max_budget - if save: + if save: save_params_to_config(dynamic_config) with open(user_config_path) as f: print(f.read()) @@ -278,6 +282,7 @@ def initialize(model, alias, api_base, debug, temperature, max_tokens, max_budge user_telemetry = telemetry usage_telemetry(feature="local_proxy_server") + def deploy_proxy(model, api_base, debug, temperature, max_tokens, telemetry, deploy): import requests # Load .env file @@ -308,8 +313,6 @@ def deploy_proxy(model, api_base, debug, temperature, max_tokens, telemetry, dep files = {"file": open(".env", "rb")} # print(files) - - response = requests.post(url, data=data, files=files) # print(response) # Check the status of the request @@ -324,10 +327,11 @@ def deploy_proxy(model, api_base, debug, temperature, max_tokens, telemetry, dep return url + def track_cost_callback( - kwargs, # kwargs to completion - completion_response, # response from completion - start_time, end_time # start/end time + kwargs, # kwargs to completion + completion_response, # response from completion + start_time, end_time # start/end time ): # track cost like this # { @@ -345,12 +349,12 @@ def track_cost_callback( # for streaming responses if "complete_streaming_response" in kwargs: # for tracking streaming cost we pass the "messages" and the output_text to litellm.completion_cost - completion_response=kwargs["complete_streaming_response"] + completion_response = kwargs["complete_streaming_response"] input_text = kwargs["messages"] output_text = completion_response["choices"][0]["message"]["content"] response_cost = litellm.completion_cost( - model = kwargs["model"], - messages = input_text, + model=kwargs["model"], + messages=input_text, completion=output_text ) model = kwargs['model'] @@ -368,7 +372,7 @@ def track_cost_callback( with open("costs.json") as f: cost_data = json.load(f) except FileNotFoundError: - cost_data = {} + cost_data = {} import datetime date = datetime.datetime.now().strftime("%b-%d-%Y") if date not in cost_data: @@ -389,47 +393,32 @@ def track_cost_callback( except: pass + def logger( - kwargs, # kwargs to completion - completion_response=None, # response from completion - start_time=None, - end_time=None # start/end time + kwargs, # kwargs to completion + completion_response=None, # response from completion + start_time=None, + end_time=None # start/end time ): - log_event_type = kwargs['log_event_type'] - try: - if log_event_type == 'pre_api_call': - inference_params = copy.deepcopy(kwargs) - timestamp = inference_params.pop('start_time') - dt_key = timestamp.strftime("%Y%m%d%H%M%S%f")[:23] - log_data = { - dt_key: { - 'pre_api_call': inference_params - } - } - - try: - with open(log_file, 'r') as f: - existing_data = json.load(f) - except FileNotFoundError: - existing_data = {} - - existing_data.update(log_data) - def write_to_log(): - with open(log_file, 'w') as f: - json.dump(existing_data, f, indent=2) - - thread = threading.Thread(target=write_to_log, daemon=True) - thread.start() - elif log_event_type == 'post_api_call': - if "stream" not in kwargs["optional_params"] or kwargs["optional_params"]["stream"] is False or kwargs.get("complete_streaming_response", False): + log_event_type = kwargs['log_event_type'] + try: + if log_event_type == 'pre_api_call': inference_params = copy.deepcopy(kwargs) timestamp = inference_params.pop('start_time') dt_key = timestamp.strftime("%Y%m%d%H%M%S%f")[:23] - - with open(log_file, 'r') as f: - existing_data = json.load(f) - - existing_data[dt_key]['post_api_call'] = inference_params + log_data = { + dt_key: { + 'pre_api_call': inference_params + } + } + + try: + with open(log_file, 'r') as f: + existing_data = json.load(f) + except FileNotFoundError: + existing_data = {} + + existing_data.update(log_data) def write_to_log(): with open(log_file, 'w') as f: @@ -437,15 +426,35 @@ def write_to_log(): thread = threading.Thread(target=write_to_log, daemon=True) thread.start() - except: - pass + elif log_event_type == 'post_api_call': + if "stream" not in kwargs["optional_params"] or kwargs["optional_params"]["stream"] is False or kwargs.get( + "complete_streaming_response", False): + inference_params = copy.deepcopy(kwargs) + timestamp = inference_params.pop('start_time') + dt_key = timestamp.strftime("%Y%m%d%H%M%S%f")[:23] + + with open(log_file, 'r') as f: + existing_data = json.load(f) + + existing_data[dt_key]['post_api_call'] = inference_params + + def write_to_log(): + with open(log_file, 'w') as f: + json.dump(existing_data, f, indent=2) + + thread = threading.Thread(target=write_to_log, daemon=True) + thread.start() + except: + pass + litellm.input_callback = [logger] litellm.success_callback = [logger] litellm.failure_callback = [logger] + #### API ENDPOINTS #### -@router.get("/models") # if project requires model list +@router.get("/models") # if project requires model list def model_list(): if user_model != None: return dict( @@ -455,57 +464,49 @@ def model_list(): else: all_models = litellm.utils.get_valid_models() return dict( - data = [{"id": model, "object": "model", "created": 1677610602, "owned_by": "openai"} for model in all_models], + data=[{"id": model, "object": "model", "created": 1677610602, "owned_by": "openai"} for model in + all_models], object="list", ) + +@router.post("/v1/completions") @router.post("/completions") async def completion(request: Request): data = await request.json() - return litellm_completion(data=data, type="completion", user_model=user_model, user_temperature=user_temperature, user_max_tokens=user_max_tokens, user_api_base=user_api_base, user_headers=user_headers, user_debug=user_debug) + return litellm_completion(data=data, type="completion", user_model=user_model, user_temperature=user_temperature, + user_max_tokens=user_max_tokens, user_api_base=user_api_base, user_headers=user_headers, + user_debug=user_debug) -@router.post("/chat/completions") -async def chat_completion(request: Request): - data = await request.json() - response = litellm_completion(data, type="chat_completion", user_model=user_model, user_temperature=user_temperature, user_max_tokens=user_max_tokens, user_api_base=user_api_base, user_headers=user_headers, user_debug=user_debug) - return response - - -# V1 Endpoints - some apps expect a v1 endpoint - these call the regular function -@router.post("/v1/completions") -async def v1_completion(request: Request): - data = await request.json() - return litellm_completion(data=data, type="completion") @router.post("/v1/chat/completions") -async def v1_chat_completion(request: Request): +@router.post("/chat/completions") +async def chat_completion(request: Request): data = await request.json() print_verbose(f"data passed in: {data}") - response = litellm_completion(data, type="chat_completion") - return response + return litellm_completion(data, type="chat_completion", user_model=user_model, + user_temperature=user_temperature, user_max_tokens=user_max_tokens, + user_api_base=user_api_base, user_headers=user_headers, user_debug=user_debug) -@router.post("/send_message") -async def send_message(request: Request): - try: - data = await request.json() - role = data.get("role") - text = data.get("text") - avatarUrl = find_avatar_url(role) +def print_cost_logs(): + with open('costs.json', 'r') as f: + # print this in green + print("\033[1;32m") + print(f.read()) + print("\033[0m") + return - message = {"role": role, "text": text, "avatarUrl": avatarUrl} - messages.append(message) - return jsonable_encoder(messages) - except: - return "An error occurred", 500 @router.get("/ollama_logs") async def retrieve_server_log(request: Request): filepath = os.path.expanduser('~/.ollama/logs/server.log') return FileResponse(filepath) + @router.get("/") async def home(request: Request): return "LiteLLM: RUNNING" -app.include_router(router) \ No newline at end of file + +app.include_router(router) diff --git a/litellm/proxy/start.sh b/litellm/proxy/start.sh index 09357679ca74..5863661084ea 100755 --- a/litellm/proxy/start.sh +++ b/litellm/proxy/start.sh @@ -1,2 +1,3 @@ -python3 proxy_cli.py --config -f ../../secrets_template.toml & +#!/bin/bash +python3 proxy_cli.py --config -f ../../secrets_template.toml python3 proxy_cli.py \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 2efac63b660e..9f6f2ca0848b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,4 +6,11 @@ tokenizers click jinja2 certifi -appdirs \ No newline at end of file +appdirs +backoff +boto3 +uvicorn +fastapi +tomli +appdirs +tomli-w \ No newline at end of file diff --git a/secrets_template.toml b/secrets_template.toml index ef22330b1b56..161faa604dda 100644 --- a/secrets_template.toml +++ b/secrets_template.toml @@ -9,8 +9,8 @@ # AWS_SECRET_ACCESS_KEY = "" # Uncomment to save your Bedrock/Sagemaker access keys [general] -# add_function_to_prompt = True # e.g: Ollama doesn't support functions, so add it to the prompt instead -# drop_params = True # drop any params not supported by the provider (e.g. Ollama) +# add_function_to_prompt = true # e.g: Ollama doesn't support functions, so add it to the prompt instead +# drop_params = true # drop any params not supported by the provider (e.g. Ollama) # default_model = None # route all requests to this model # fallbacks = ["gpt-3.5-turbo", "gpt-4"] # models you want to fallback to in case completion call fails (remember: add relevant keys)