From db4f825a94a0d49fca115086b3f1a8f35435b3ce Mon Sep 17 00:00:00 2001 From: Ryan Marten Date: Mon, 16 Dec 2024 19:10:02 -0800 Subject: [PATCH 1/3] Update map_finish_reason --- litellm/litellm_core_utils/core_helpers.py | 100 +++++++++++++-------- 1 file changed, 63 insertions(+), 37 deletions(-) diff --git a/litellm/litellm_core_utils/core_helpers.py b/litellm/litellm_core_utils/core_helpers.py index 816dff81ee92..e50d96084fee 100644 --- a/litellm/litellm_core_utils/core_helpers.py +++ b/litellm/litellm_core_utils/core_helpers.py @@ -14,44 +14,70 @@ else: Span = Any +FinishReason = Literal["stop", "length", "tool_calls", "content_filter"] -def map_finish_reason( - finish_reason: str, -): # openai supports 5 stop sequences - 'stop', 'length', 'function_call', 'content_filter', 'null' - # anthropic mapping - if finish_reason == "stop_sequence": - return "stop" - # cohere mapping - https://docs.cohere.com/reference/generate - elif finish_reason == "COMPLETE": - return "stop" - elif finish_reason == "MAX_TOKENS": # cohere + vertex ai - return "length" - elif finish_reason == "ERROR_TOXIC": - return "content_filter" - elif ( - finish_reason == "ERROR" - ): # openai currently doesn't support an 'error' finish reason - return "stop" - # huggingface mapping https://huggingface.github.io/text-generation-inference/#/Text%20Generation%20Inference/generate_stream - elif finish_reason == "eos_token" or finish_reason == "stop_sequence": - return "stop" - elif ( - finish_reason == "FINISH_REASON_UNSPECIFIED" or finish_reason == "STOP" - ): # vertex ai - got from running `print(dir(response_obj.candidates[0].finish_reason))`: ['FINISH_REASON_UNSPECIFIED', 'MAX_TOKENS', 'OTHER', 'RECITATION', 'SAFETY', 'STOP',] - return "stop" - elif finish_reason == "SAFETY" or finish_reason == "RECITATION": # vertex ai - return "content_filter" - elif finish_reason == "STOP": # vertex ai - return "stop" - elif finish_reason == "end_turn" or finish_reason == "stop_sequence": # anthropic - return "stop" - elif finish_reason == "max_tokens": # anthropic - return "length" - elif finish_reason == "tool_use": # anthropic - return "tool_calls" - elif finish_reason == "content_filtered": - return "content_filter" - return finish_reason +def map_finish_reason(finish_reason: str) -> FinishReason: + """ + Maps finish reasons from various AI providers to a standardized format. + + This function normalizes finish reason strings from different AI providers + (OpenAI, Vertex AI, HuggingFace, Cohere, Anthropic) to a consistent set + of values. + + Args: + finish_reason (str): The finish reason string from the AI provider + + Returns: + FinishReason: One of the following standardized finish reasons: + - "stop": Normal completion (includes EOS token, complete, etc.) + - "length": Maximum token limit reached + - "tool_calls": Stopped due to tool/function calls + - "content_filter": Stopped due to content filtering/safety + + Provider-specific mappings: + - OpenAI: 'stop', 'length', 'tool_calls', 'content_filter', 'function_call' + - Vertex AI: 'FINISH_REASON_UNSPECIFIED', 'MAX_TOKENS', 'STOP', 'SAFETY', 'RECITATION' + - HuggingFace: 'stop_sequence', 'eos_token', 'max_tokens' + - Cohere: 'COMPLETE', 'ERROR_TOXIC', 'ERROR', 'MAX_TOKENS' + - Anthropic: 'stop_sequence', 'max_tokens', 'end_turn' + + Provider-speicific mappings source: + - openai.types.chat.chat_completion.Choice.model_fields['finish_reason'] + - google.generativeai.protos.Candidate.FinishReason.__members__.keys() + - cohere.types.ChatFinishReason + - anthropic.types.Message.model_fields['stop_reason'] + """ + # Mapping of provider-specific finish reasons to standardized values + finish_reason_mapping = { + # Normal completion reasons + "stop": "stop", + "COMPLETE": "stop", + "ERROR": "stop", + "eos_token": "stop", + "stop_sequence": "stop", + "FINISH_REASON_UNSPECIFIED": "stop", + "STOP": "stop", + "end_turn": "stop", + + # Length-related reasons + "length": "length", + "MAX_TOKENS": "length", + "max_tokens": "length", + + # Tool/function call reasons + "tool_calls": "tool_calls", + "tool_use": "tool_calls", + "function_call": "tool_calls", + + # Content filtering/safety reasons + "content_filter": "content_filter", + "ERROR_TOXIC": "content_filter", + "SAFETY": "content_filter", + "RECITATION": "content_filter", + "content_filtered": "content_filter" + } + + return finish_reason_mapping.get(finish_reason, finish_reason) def remove_index_from_tool_calls(messages, tool_calls): From a6e5dd39db1e6420b2afe7d2df3154a7d8adf7b4 Mon Sep 17 00:00:00 2001 From: Ryan Marten Date: Mon, 16 Dec 2024 19:22:35 -0800 Subject: [PATCH 2/3] update docs and mapping --- litellm/litellm_core_utils/core_helpers.py | 47 ++++++++++------------ 1 file changed, 22 insertions(+), 25 deletions(-) diff --git a/litellm/litellm_core_utils/core_helpers.py b/litellm/litellm_core_utils/core_helpers.py index e50d96084fee..052ed72c404d 100644 --- a/litellm/litellm_core_utils/core_helpers.py +++ b/litellm/litellm_core_utils/core_helpers.py @@ -14,34 +14,20 @@ else: Span = Any -FinishReason = Literal["stop", "length", "tool_calls", "content_filter"] -def map_finish_reason(finish_reason: str) -> FinishReason: +def map_finish_reason(finish_reason: str) -> str: """ - Maps finish reasons from various AI providers to a standardized format. - - This function normalizes finish reason strings from different AI providers - (OpenAI, Vertex AI, HuggingFace, Cohere, Anthropic) to a consistent set - of values. - - Args: - finish_reason (str): The finish reason string from the AI provider - - Returns: - FinishReason: One of the following standardized finish reasons: - - "stop": Normal completion (includes EOS token, complete, etc.) - - "length": Maximum token limit reached - - "tool_calls": Stopped due to tool/function calls - - "content_filter": Stopped due to content filtering/safety + Maps finish reasons from various AI providers (OpenAI, Vertex AI, HuggingFace, Cohere, Anthropic) to a consistent set + of values based on OpenAI's finish reason values. - Provider-specific mappings: + Provider-specific values: - OpenAI: 'stop', 'length', 'tool_calls', 'content_filter', 'function_call' - - Vertex AI: 'FINISH_REASON_UNSPECIFIED', 'MAX_TOKENS', 'STOP', 'SAFETY', 'RECITATION' + - Vertex AI: 'FINISH_REASON_UNSPECIFIED', 'STOP', 'MAX_TOKENS', 'SAFETY', 'RECITATION', 'LANGUAGE', 'OTHER', 'BLOCKLIST', 'PROHIBITED_CONTENT', 'SPII', 'MALFORMED_FUNCTION_CALL' - HuggingFace: 'stop_sequence', 'eos_token', 'max_tokens' - - Cohere: 'COMPLETE', 'ERROR_TOXIC', 'ERROR', 'MAX_TOKENS' - - Anthropic: 'stop_sequence', 'max_tokens', 'end_turn' + - Cohere: 'COMPLETE', 'STOP_SEQUENCE', 'MAX_TOKENS', 'TOOL_CALL', 'ERROR' + - Anthropic: 'end_turn', 'max_tokens', 'stop_sequence', 'tool_use' - Provider-speicific mappings source: + Provider-speicific values source: - openai.types.chat.chat_completion.Choice.model_fields['finish_reason'] - google.generativeai.protos.Candidate.FinishReason.__members__.keys() - cohere.types.ChatFinishReason @@ -55,10 +41,17 @@ def map_finish_reason(finish_reason: str) -> FinishReason: "ERROR": "stop", "eos_token": "stop", "stop_sequence": "stop", - "FINISH_REASON_UNSPECIFIED": "stop", + "STOP_SEQUENCE": "stop", "STOP": "stop", "end_turn": "stop", - + "OTHER": "stop", + "FINISH_REASON_UNSPECIFIED": "stop", + + # Error completions reasons + # Mapping to stop since our set of finish reasons doesn't include error + "ERROR": "stop", + "MALFORMED_FUNCTION_CALL": "stop", + # Length-related reasons "length": "length", "MAX_TOKENS": "length", @@ -68,13 +61,17 @@ def map_finish_reason(finish_reason: str) -> FinishReason: "tool_calls": "tool_calls", "tool_use": "tool_calls", "function_call": "tool_calls", + "TOOL_CALL": "tool_calls", # Content filtering/safety reasons "content_filter": "content_filter", "ERROR_TOXIC": "content_filter", "SAFETY": "content_filter", "RECITATION": "content_filter", - "content_filtered": "content_filter" + "content_filtered": "content_filter", + "BLOCKLIST": "content_filter", + "PROHIBITED_CONTENT": "content_filter", + "SPII": "content_filter", } return finish_reason_mapping.get(finish_reason, finish_reason) From 581c00215890d64048112d14f32676a88dcc64b4 Mon Sep 17 00:00:00 2001 From: Ryan Marten Date: Wed, 25 Dec 2024 10:38:44 -0500 Subject: [PATCH 3/3] add test for map_finish_reason --- litellm/litellm_core_utils/core_helpers.py | 30 ++++++++++------------ tests/local_testing/test_utils.py | 9 +++++++ 2 files changed, 22 insertions(+), 17 deletions(-) diff --git a/litellm/litellm_core_utils/core_helpers.py b/litellm/litellm_core_utils/core_helpers.py index 052ed72c404d..8c87db829303 100644 --- a/litellm/litellm_core_utils/core_helpers.py +++ b/litellm/litellm_core_utils/core_helpers.py @@ -37,44 +37,40 @@ def map_finish_reason(finish_reason: str) -> str: finish_reason_mapping = { # Normal completion reasons "stop": "stop", - "COMPLETE": "stop", - "ERROR": "stop", + "complete": "stop", "eos_token": "stop", "stop_sequence": "stop", - "STOP_SEQUENCE": "stop", - "STOP": "stop", "end_turn": "stop", - "OTHER": "stop", - "FINISH_REASON_UNSPECIFIED": "stop", + "other": "stop", + "finish_reason_unspecified": "stop", # Error completions reasons # Mapping to stop since our set of finish reasons doesn't include error - "ERROR": "stop", - "MALFORMED_FUNCTION_CALL": "stop", + "error": "stop", + "malformed_function_call": "stop", # Length-related reasons "length": "length", - "MAX_TOKENS": "length", "max_tokens": "length", # Tool/function call reasons "tool_calls": "tool_calls", "tool_use": "tool_calls", "function_call": "tool_calls", - "TOOL_CALL": "tool_calls", + "tool_call": "tool_calls", # Content filtering/safety reasons "content_filter": "content_filter", - "ERROR_TOXIC": "content_filter", - "SAFETY": "content_filter", - "RECITATION": "content_filter", + "error_toxic": "content_filter", + "safety": "content_filter", + "recitation": "content_filter", "content_filtered": "content_filter", - "BLOCKLIST": "content_filter", - "PROHIBITED_CONTENT": "content_filter", - "SPII": "content_filter", + "blocklist": "content_filter", + "prohibited_content": "content_filter", + "spii": "content_filter", } - return finish_reason_mapping.get(finish_reason, finish_reason) + return finish_reason_mapping.get(finish_reason.lower(), finish_reason) def remove_index_from_tool_calls(messages, tool_calls): diff --git a/tests/local_testing/test_utils.py b/tests/local_testing/test_utils.py index 7d922e19b632..a82c49c98825 100644 --- a/tests/local_testing/test_utils.py +++ b/tests/local_testing/test_utils.py @@ -1238,3 +1238,12 @@ def test_token_counter_with_image_url_with_detail_high(): ) print("tokens", _tokens) assert _tokens == DEFAULT_IMAGE_TOKEN_COUNT + 7 + +def test_map_finish_reason(): + from litellm.litellm_core_utils.core_helpers import map_finish_reason + + assert map_finish_reason("STOP") == "stop" + assert map_finish_reason("RECITATION") == "content_filter" + assert map_finish_reason("tool_use") == "tool_calls" + assert map_finish_reason("reason_not_in_mapping") == "reason_not_in_mapping" + assert map_finish_reason("MALFORMED_FUNCTION_CALL") == "stop" \ No newline at end of file