fix bugs in aborting

benchmarks/benchmark_serving.py

@@ -496,15 +496,6 @@ async def handle_requests():
             request_func_input = await asyncio.get_event_loop().run_in_executor(None, request_queue.get)
             if request_func_input is None:
                 break
-
-#         # if backend == "vllm":
-#         #     result = asyncio.run(
-#         #         request_func(args.scheduler_policy, request_func_input=request_func_input, pbar=pbar)
-#         #     )
-#         # else:
-#         #     result = asyncio.run(
-#         #         request_func(request_func_input=request_func_input, pbar=pbar)
-#         #     )
 
             if backend == "vllm":
                 tasks.append(
@@ -524,11 +515,11 @@ async def handle_requests():
 
         # Gather the results of all tasks
         outputs = await asyncio.gather(*tasks)
-        # result_queue.put(outputs)
         await asyncio.get_event_loop().run_in_executor(None, result_queue.put, outputs)
-
-
-    asyncio.run(handle_requests())
+    loop = asyncio.new_event_loop()
+    asyncio.set_event_loop(loop)
+    loop.run_until_complete(handle_requests())
+    # asyncio.run(handle_requests())
 
 async def generate_requests(input_requests, request_rate, request_duration, model_id, api_url, best_of, use_beam_search):
     async for request in get_request_duration(input_requests, request_rate, request_duration):
@@ -647,21 +638,25 @@ async def benchmark(
             use_beam_search=use_beam_search,
         )
         request_queue.put(request_func_input)
+        # await asyncio.get_event_loop().run_in_executor(None, request_queue.put, request_func_input)
     # request_generator.join()
 
     # tasks = []
     total_outputs: List[RequestFuncOutput] = []
 
     for _ in range(num_workers):
+        # await asyncio.get_event_loop().run_in_executor(None, request_queue.put, None)
         request_queue.put(None)
 
     outputs: List[RequestFuncOutput] = []
     # with lock:
 
     while True:
         try:
+            # result = await asyncio.get_event_loop().run_in_executor(None, result_queue.get, timeout=2)
             result = result_queue.get(timeout=2)
-            outputs.append(result)
+            for res in result:
+                outputs.append(res)
             # break
         except Exception as error:
             break

rebuild_vllm.sh

@@ -1,8 +1,8 @@
 # rm -r build
-pip uninstall -y vllm
-taskset -c 14-19 python3 setup.py bdist_wheel --dist-dir=dist
-pip install dist/vllm-0.4.3+cu124-cp310-cp310-linux_x86_64.whl
-pip uninstall -y vllm-flash-attn
-clear
+#pip uninstall -y vllm
+#taskset -c 14-19 python3 setup.py bdist_wheel --dist-dir=dist
+#pip install dist/vllm-0.4.3+cu124-cp310-cp310-linux_x86_64.whl
+#pip uninstall -y vllm-flash-attn
+#clear
 cd benchmarks
 bash 1_serving_benchmark.sh