add minimal_generation to async_llm

Signed-off-by: rongfu.leng <rongfu.leng@daocloud.io>

Author: lengrongfu

vllm/engine/protocol.py

@@ -59,6 +59,11 @@ def generate(
         """Generate outputs for a request."""
         ...
 
+    @abstractmethod
+    def minimal_generation(self) -> str:
+        """Generate outputs for a minimal spec prompt"""
+        ...
+
     async def beam_search(
         self,
         prompt: PromptType,

vllm/entrypoints/openai/api_server.py

@@ -23,8 +23,7 @@
 import pydantic
 import regex as re
 import uvloop
-from fastapi import (APIRouter, Depends, FastAPI, Form, HTTPException, Query,
-                     Request)
+from fastapi import APIRouter, Depends, FastAPI, Form, HTTPException, Request
 from fastapi.exceptions import RequestValidationError
 from fastapi.middleware.cors import CORSMiddleware
 from fastapi.responses import JSONResponse, Response, StreamingResponse
@@ -99,13 +98,13 @@
                                     log_non_default_args, with_cancellation)
 from vllm.logger import init_logger
 from vllm.reasoning import ReasoningParserManager
-from vllm.sampling_params import SamplingParams
 from vllm.transformers_utils.config import (
     maybe_register_config_serialize_by_value)
 from vllm.transformers_utils.tokenizer import MistralTokenizer
 from vllm.usage.usage_lib import UsageContext
 from vllm.utils import (Device, FlexibleArgumentParser, decorate_logs,
-                        is_valid_ipv6_address, random_uuid, set_ulimit)
+                        get_open_zmq_ipc_path,is_valid_ipv6_address, random_uuid, 
+                        set_ulimit)
 from vllm.v1.engine.exceptions import EngineDeadError
 from vllm.v1.metrics.prometheus import get_prometheus_registry
 from vllm.version import __version__ as VLLM_VERSION
@@ -344,18 +343,13 @@ def engine_client(request: Request) -> EngineClient:
 
 
 @router.get("/health", response_class=Response)
-async def health(
-    raw_request: Request, generate: Optional[bool] = Query(False)) -> Response:
+async def health(raw_request: Request) -> Response:
     """Health check."""
     try:
         await engine_client(raw_request).check_health()
-        if generate:
-            prompt = "Hi"
-            sampling_params = SamplingParams(temperature=0, max_tokens=2)
-            request_id = random_uuid()
-            async for _ in engine_client(raw_request).generate(prompt, sampling_params,
-                                           request_id):
-                pass
+        generate_str = raw_request.query_params.get("generate")
+        if generate_str == "true":
+            await engine_client(raw_request).minimal_generation()
         return Response(status_code=200)
     except EngineDeadError:
         return Response(status_code=503)

vllm/v1/engine/async_llm.py

@@ -33,7 +33,7 @@
                                                init_tokenizer_from_configs)
 from vllm.usage.usage_lib import UsageContext
 from vllm.utils import (Device, as_list, cancel_task_threadsafe, cdiv,
-                        deprecate_kwargs)
+                        deprecate_kwargs, random_uuid)
 from vllm.v1.engine import EngineCoreRequest
 from vllm.v1.engine.core_client import EngineCoreClient
 from vllm.v1.engine.exceptions import EngineDeadError, EngineGenerateError
@@ -724,6 +724,18 @@ async def scale_elastic_ep(self,
                 custom_stat_loggers=None,
             )
 
+    async def minimal_generation(self) -> str:
+        prompt = "Hi"
+        sampling_params = SamplingParams(temperature=0, max_tokens=2)
+        request_id = random_uuid()
+        result_text = ""
+        async for output in self.generate(prompt, sampling_params, request_id):
+            for completion in output.outputs:
+                result_text = completion.text
+            if output.finished:
+                break
+        return result_text
+
     @property
     def is_running(self) -> bool:
         # Is None before the loop is started.