bump fastdeploy_llm to v1.0.0

PaddlePaddle · Dec 26, 2023 · fc61eb4 · fc61eb4
1 parent 4749a60
commit fc61eb4
Show file tree

Hide file tree

Showing 3 changed files with 7 additions and 4 deletions.
diff --git a/llm/fastdeploy_llm/serving/serving_model.py b/llm/fastdeploy_llm/serving/serving_model.py
@@ -81,7 +81,7 @@ def runner(self):
                         task = self.requests_queue.get() # only block when get first data
                     else:
                         task = self.requests_queue.get(timeout=0.01)  # wait only 10ms for batch
-                        get_tasks.append(task)
+                    get_tasks.append(task)
                 except Exception as e:
                     break
 

diff --git a/llm/fastdeploy_llm/serving/triton_model_nonstream.py b/llm/fastdeploy_llm/serving/triton_model_nonstream.py
@@ -21,7 +21,8 @@
 import functools
 from collections import defaultdict
 from fastdeploy_llm.serving.serving_model import ServingModel
-from fastdeploy_llm.utils.logging_util import logger
+from fastdeploy_llm.utils.logging_util import logger, warning_logger
+from fastdeploy_llm.utils.logging_util import error_format, ErrorCode,  ErrorType
 from fastdeploy_llm.task import Task, BatchTask
 import fastdeploy_llm as fdlm
 import queue
@@ -51,7 +52,8 @@ def stream_call_back(call_back_task, token_tuple, index, is_last_token,
         out_tensor = pb_utils.Tensor(
             "OUT", np.array(
                 [json.dumps(out)], dtype=np.object_))
-        response_dict[call_back_task.task_id] = out_tensor
+        response = pb_utils.InferenceResponse([out_tensor])
+        response_dict[call_back_task.task_id] = response
         response_finished_queue.put(call_back_task.task_id)
 
         logger.info("Model output for req_id: {}  results_all: {} tokens_all: {}".format(call_back_task.task_id, all_strs, all_token_ids)) 
@@ -233,6 +235,7 @@ def execute(self, requests):
                 responses[index] = response_dict[task_id]
                 del inflight_valid_tasks[task_id]
                 del response_dict[task_id]
+                del self.response_handler[task_id]
             except:
                 for task_id, index in inflight_valid_tasks.items():
                     error_type = ErrorType.Query

diff --git a/llm/setup.py b/llm/setup.py
@@ -16,7 +16,7 @@
 
 setuptools.setup(
     name="fastdeploy-llm",
-    version="0.0.9",
+    version="1.0.0",
     author="fastdeploy",
     author_email="fastdeploy@baidu.com",
     description="FastDeploy for Large Language Model",