fix bug

PaddlePaddle · Jan 8, 2025 · c249b98 · c249b98
1 parent c7e1d58
commit c249b98
Show file tree

Hide file tree

Showing 2 changed files with 2 additions and 3 deletions.
diff --git a/llm/server/server/http_server/api.py b/llm/server/server/http_server/api.py
@@ -31,6 +31,7 @@ class Req(BaseModel):
     req_id: str = Field(default_factory=lambda: str(uuid.uuid4()))
     input_ids: Optional[List[int]] = None
     text: Optional[str] = None
+    stop_sequences: Optional[List] = None
     messages: Optional[List] = None
     max_dec_len: Optional[int] = None
     seq_len: Optional[int] = None

diff --git a/llm/server/server/triton_server.py b/llm/server/server/triton_server.py
@@ -102,9 +102,7 @@ def _cache_special_tokens(self, batch_result):
         for i in range(len(batch_result)):
             is_end = batch_result[i].get("is_end", 0)
             token_ids = batch_result[i]["token_ids"]
-            return_all_tokens = batch_result[i].get("return_all_tokens", False)
-            cache_special_token = False if is_end == 1 else True
-            if is_end != 1 and (cache_special_token or return_all_tokens or self.cfg.disable_streaming):
+            if is_end != 1:
                 if batch_result[i]["req_id"] not in self.token_buffer:
                     self.token_buffer[batch_result[i]["req_id"]] = list()
                     self.score_buffer[batch_result[i]["req_id"]] = list()