File tree Expand file tree Collapse file tree 1 file changed +24
-0
lines changed Expand file tree Collapse file tree 1 file changed +24
-0
lines changed Original file line number Diff line number Diff line change @@ -105,6 +105,20 @@ def deepseek_r1_qwen_tokenizer():
105105 "reasoning_content" : None ,
106106 "content" : None ,
107107}
108+ NEW_LINE = {
109+ "output" : "\n <think>This is a reasoning section</think>\n This is the rest" ,
110+ "reasoning_content" : "This is a reasoning section" ,
111+ "content" : "\n This is the rest" ,
112+ }
113+ # Streaming cannot handle new lines at the beginning of the output
114+ # because we need to support <think>...</think> and </think>...
115+ # We cannot know if the text before <think> is reasoning content
116+ # or not.
117+ NEW_LINE_STREAMING = {
118+ "output" : "\n <think>This is a reasoning section</think>\n This is the rest" ,
119+ "reasoning_content" : "\n This is a reasoning section" ,
120+ "content" : "\n This is the rest" ,
121+ }
108122
109123TEST_CASES = [
110124 pytest .param (
@@ -217,6 +231,16 @@ def deepseek_r1_qwen_tokenizer():
217231 EMPTY_STREAMING ,
218232 id = "empty_streaming" ,
219233 ),
234+ pytest .param (
235+ False ,
236+ NEW_LINE ,
237+ id = "new_line" ,
238+ ),
239+ pytest .param (
240+ True ,
241+ NEW_LINE_STREAMING ,
242+ id = "new_line_streaming" ,
243+ ),
220244]
221245
222246
You can’t perform that action at this time.
0 commit comments