allow tool call from in the middle of text

rubra-ai · Jul 13, 2024 · fde81db · fde81db
1 parent f27ae3d
commit fde81db
Showing 1 changed file with 8 additions and 6 deletions.
diff --git a/vllm/entrypoints/openai/serving_chat.py b/vllm/entrypoints/openai/serving_chat.py
@@ -383,20 +383,20 @@ async def chat_completion_stream_generator(
                     first_iteration = False
 
                 is_function_call = False
-                checked_function_call = False
+                last_str = ""
                 for output in res.outputs:
                     i = output.index
 
                     if finish_reason_sent[i]:
                         continue
 
-                    if (not checked_function_call and len(output.text)>= 15):
-                        if "starttoolcall" in output.text:
+                    if (not is_function_call):
+                        delta_text = output.text[len(previous_texts[i]):] 
+                        if "starttool" in (last_str + delta_text):
                             is_function_call = True
-                        checked_function_call = True
-
-                    if (checked_function_call and not is_function_call) or output.finish_reason is not None:
 
+                    if (last_str != "" and not is_function_call) or output.finish_reason is not None:
+
                         delta_token_ids = output.token_ids[previous_num_tokens[i]:]
                         out_logprobs = output.logprobs[
                             previous_num_tokens[i]:] if output.logprobs else None
@@ -482,6 +482,8 @@ async def chat_completion_stream_generator(
                             data = chunk.model_dump_json(exclude_unset=False)
                             yield f"data: {data}\n\n"
                             finish_reason_sent[i] = True
+                    else:
+                        last_str = delta_text
 
             if (request.stream_options
                     and request.stream_options.include_usage):