From de6d42a4e186a51f3058c66622de04b7aaab7492 Mon Sep 17 00:00:00 2001 From: "Yang, Bo" Date: Thu, 2 May 2024 18:35:18 -0700 Subject: [PATCH] [BugFix] Prevent the task of `_force_log` from being garbage collected (#4567) --- vllm/entrypoints/openai/api_server.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/vllm/entrypoints/openai/api_server.py b/vllm/entrypoints/openai/api_server.py index 54a7f27351cab..fb6a8fd52144d 100644 --- a/vllm/entrypoints/openai/api_server.py +++ b/vllm/entrypoints/openai/api_server.py @@ -5,6 +5,7 @@ import re from contextlib import asynccontextmanager from http import HTTPStatus +from typing import Any, Set import fastapi import uvicorn @@ -37,6 +38,8 @@ async_llm_engine: AsyncLLMEngine logger = init_logger(__name__) +_running_tasks: Set[asyncio.Task[Any]] = set() + @asynccontextmanager async def lifespan(app: fastapi.FastAPI): @@ -47,7 +50,9 @@ async def _force_log(): await engine.do_log_stats() if not engine_args.disable_log_stats: - asyncio.create_task(_force_log()) + task = asyncio.create_task(_force_log()) + _running_tasks.add(task) + task.add_done_callback(_running_tasks.remove) grpc_server = await start_grpc_server(async_llm_engine, args)