From e366b570e26eb914e30cb97ce83a321b0dc7de38 Mon Sep 17 00:00:00 2001 From: Andrew Xia Date: Fri, 12 Sep 2025 16:31:15 -0700 Subject: [PATCH 1/3] item id, content id Signed-off-by: Andrew Xia --- vllm/entrypoints/openai/serving_responses.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/vllm/entrypoints/openai/serving_responses.py b/vllm/entrypoints/openai/serving_responses.py index 401ba6c53331..5273a7d31a80 100644 --- a/vllm/entrypoints/openai/serving_responses.py +++ b/vllm/entrypoints/openai/serving_responses.py @@ -1245,9 +1245,9 @@ async def _process_harmony_streaming_events( created_time: int, _send_event: Callable[[BaseModel], str], ) -> AsyncGenerator[str, None]: - current_content_index = 0 # FIXME: this number is never changed + current_content_index = -1 current_output_index = 0 - current_item_id = "" # FIXME: this number is never changed + current_item_id: str = "" sent_output_item_added = False async for ctx in result_generator: @@ -1337,6 +1337,8 @@ async def _process_harmony_streaming_events( and ctx.parser.current_recipient is None): if not sent_output_item_added: sent_output_item_added = True + # wut + current_item_id = f"msg_{random_uuid()}" yield _send_event( openai_responses_types. ResponseOutputItemAddedEvent( @@ -1352,6 +1354,7 @@ async def _process_harmony_streaming_events( status="in_progress", ), )) + current_content_index += 1 yield _send_event( openai_responses_types. ResponseContentPartAddedEvent( @@ -1382,6 +1385,7 @@ async def _process_harmony_streaming_events( and ctx.parser.current_recipient is None): if not sent_output_item_added: sent_output_item_added = True + current_item_id = f"msg_{random_uuid()}" yield _send_event( openai_responses_types. ResponseOutputItemAddedEvent( @@ -1396,6 +1400,7 @@ async def _process_harmony_streaming_events( status="in_progress", ), )) + current_content_index += 1 yield _send_event( openai_responses_types. ResponseContentPartAddedEvent( @@ -1428,6 +1433,7 @@ async def _process_harmony_streaming_events( ) and ctx.parser.current_recipient == "python": if not sent_output_item_added: sent_output_item_added = True + current_item_id = f"tool_{random_uuid()}" yield _send_event( openai_responses_types. ResponseOutputItemAddedEvent( @@ -1498,6 +1504,7 @@ async def _process_harmony_streaming_events( raise ValueError( f"Unknown function name: {function_name}") + current_item_id = f"tool_{random_uuid()}", yield _send_event( openai_responses_types.ResponseOutputItemAddedEvent( type="response.output_item.added", From f2bf2433270b871722003400d7f4b12e37db8919 Mon Sep 17 00:00:00 2001 From: Andrew Xia Date: Fri, 12 Sep 2025 16:53:59 -0700 Subject: [PATCH 2/3] unit test Signed-off-by: Andrew Xia --- .../openai/test_response_api_with_harmony.py | 23 +++++++++++++++++++ vllm/entrypoints/openai/serving_responses.py | 2 +- 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/tests/entrypoints/openai/test_response_api_with_harmony.py b/tests/entrypoints/openai/test_response_api_with_harmony.py index 0d5836fab5a7..fcda077dfe9a 100644 --- a/tests/entrypoints/openai/test_response_api_with_harmony.py +++ b/tests/entrypoints/openai/test_response_api_with_harmony.py @@ -304,6 +304,9 @@ async def test_streaming(client: OpenAI, model_name: str, background: bool): background=background, ) + current_item_id = "" + current_content_index = -1 + events = [] current_event_mode = None resp_id = None @@ -315,6 +318,26 @@ async def test_streaming(client: OpenAI, model_name: str, background: bool): current_event_mode = event.type print(f"\n[{event.type}] ", end="", flush=True) + # verify current_item_id is correct + if event.type == "response.output_item.added": + assert event.item.id != current_item_id + current_item_id = event.item.id + elif event.type in [ + "response.output_text.delta", + "response.reasoning_text.delta" + ]: + assert event.item_id == current_item_id + + # verify content_index_id is correct + if event.type == "response.content_part.added": + assert event.content_index != current_content_index + current_content_index = event.content_index + elif event.type in [ + "response.output_text.delta", + "response.reasoning_text.delta" + ]: + assert event.content_index == current_content_index + if "text.delta" in event.type: print(event.delta, end="", flush=True) elif "reasoning_text.delta" in event.type: diff --git a/vllm/entrypoints/openai/serving_responses.py b/vllm/entrypoints/openai/serving_responses.py index 5273a7d31a80..fbb49fc05d6b 100644 --- a/vllm/entrypoints/openai/serving_responses.py +++ b/vllm/entrypoints/openai/serving_responses.py @@ -1504,7 +1504,7 @@ async def _process_harmony_streaming_events( raise ValueError( f"Unknown function name: {function_name}") - current_item_id = f"tool_{random_uuid()}", + current_item_id = f"tool_{random_uuid()}" yield _send_event( openai_responses_types.ResponseOutputItemAddedEvent( type="response.output_item.added", From 1bd029f42151135a40ec25dac73d67c15672d8cd Mon Sep 17 00:00:00 2001 From: Andrew Xia Date: Mon, 15 Sep 2025 13:29:46 -0700 Subject: [PATCH 3/3] fix merge conflict Signed-off-by: Andrew Xia --- vllm/entrypoints/openai/serving_responses.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/vllm/entrypoints/openai/serving_responses.py b/vllm/entrypoints/openai/serving_responses.py index 6bbc45666816..7be5e54208bd 100644 --- a/vllm/entrypoints/openai/serving_responses.py +++ b/vllm/entrypoints/openai/serving_responses.py @@ -1353,6 +1353,7 @@ async def _process_harmony_streaming_events( and ctx.parser.current_recipient is None): if not sent_output_item_added: sent_output_item_added = True + current_item_id = f"msg_{random_uuid()}" yield _increment_sequence_number_and_return( openai_responses_types. ResponseOutputItemAddedEvent( @@ -1414,6 +1415,7 @@ async def _process_harmony_streaming_events( status="in_progress", ), )) + current_content_index += 1 yield _increment_sequence_number_and_return( openai_responses_types. ResponseContentPartAddedEvent( @@ -1446,6 +1448,7 @@ async def _process_harmony_streaming_events( ) and ctx.parser.current_recipient == "python": if not sent_output_item_added: sent_output_item_added = True + current_item_id = f"tool_{random_uuid()}" yield _increment_sequence_number_and_return( openai_responses_types. ResponseOutputItemAddedEvent(