(vllm) fix import error

Author: Bryce1010

vllm/entrypoints/llm.py

@@ -10,7 +10,7 @@
 import torch.nn as nn
 from tqdm import tqdm
 from typing_extensions import TypeVar, deprecated
-
+import torch
 from vllm import envs
 from vllm.beam_search import (BeamSearchInstance, BeamSearchOutput,
                               BeamSearchSequence, get_beam_search_score)
@@ -286,6 +286,8 @@ def generate(
         prompt_adapter_request: Optional[PromptAdapterRequest] = None,
         guided_options_request: Optional[Union[LLMGuidedOptions,
                                                GuidedDecodingRequest]] = None,
+        prompt_embeds: Optional[torch.Tensor] = None,
+        priority: Optional[list[int]] = None,
     ) -> list[RequestOutput]:
         ...
 
@@ -302,6 +304,8 @@ def generate(
         prompt_adapter_request: Optional[PromptAdapterRequest] = None,
         guided_options_request: Optional[Union[LLMGuidedOptions,
                                                GuidedDecodingRequest]] = None,
+        prompt_embeds: Optional[torch.Tensor] = None,
+        priority: Optional[list[int]] = None,
     ) -> list[RequestOutput]:
         ...
 
@@ -318,6 +322,8 @@ def generate(
         prompt_adapter_request: Optional[PromptAdapterRequest] = None,
         guided_options_request: Optional[Union[LLMGuidedOptions,
                                                GuidedDecodingRequest]] = None,
+        prompt_embeds: Optional[torch.Tensor] = None,
+        priority: Optional[list[int]] = None,
     ) -> list[RequestOutput]:
         ...
 
@@ -335,6 +341,8 @@ def generate(
         prompt_adapter_request: Optional[PromptAdapterRequest] = None,
         guided_options_request: Optional[Union[LLMGuidedOptions,
                                                GuidedDecodingRequest]] = None,
+        prompt_embeds: Optional[torch.Tensor] = None,
+        priority: Optional[list[int]] = None,
     ) -> list[RequestOutput]:
         ...
 
@@ -352,6 +360,8 @@ def generate(
         prompt_adapter_request: Optional[PromptAdapterRequest] = None,
         guided_options_request: Optional[Union[LLMGuidedOptions,
                                                GuidedDecodingRequest]] = None,
+        prompt_embeds: Optional[torch.Tensor] = None,
+        priority: Optional[list[int]] = None,
     ) -> list[RequestOutput]:
         ...
 
@@ -367,6 +377,8 @@ def generate(
         prompt_adapter_request: Optional[PromptAdapterRequest] = None,
         guided_options_request: Optional[Union[LLMGuidedOptions,
                                                GuidedDecodingRequest]] = None,
+        prompt_embeds: Optional[torch.Tensor] = None,
+        priority: Optional[list[int]] = None,
     ) -> list[RequestOutput]:
         ...
 
@@ -381,7 +393,7 @@ def generate(
                        Optional[Union[str, list[str]]]] = None,
         sampling_params: Optional[Union[SamplingParams,
                                         Sequence[SamplingParams]]] = None,
-        prompt_token_ids: Optional[Union[List[int], List[List[int]]]] = None,
+        prompt_token_ids: Optional[Union[list[int], list[list[int]]]] = None,
         prompt_embeds: Optional[torch.Tensor] = None,
         use_tqdm: bool = True,
         lora_request: Optional[Union[list[LoRARequest], LoRARequest]] = None,
@@ -405,10 +417,15 @@ def generate(
                 When it is a single value, it is applied to every prompt.
                 When it is a list, the list must have the same length as the
                 prompts and it is paired one by one with the prompt.
+            prompt_token_ids: DEPRECATED. Token IDs for the prompts. If provided,
+                the `prompts` will be ignored.
+            prompt_embeds: Optional tensor of prompt embeddings to use instead of
+                text prompts.
             use_tqdm: Whether to use tqdm to display the progress bar.
             lora_request: LoRA request to use for generation, if any.
             prompt_adapter_request: Prompt Adapter request to use for
                 generation, if any.
+            guided_options_request: Options for guided decoding, if any.
             priority: The priority of the requests, if any.
                 Only applicable when priority scheduling policy is enabled.
 
@@ -442,13 +459,13 @@ def generate(
             parsed_prompts = self._convert_v1_inputs(
                 prompts=cast(Optional[Union[str, list[str]]], prompts),
                 prompt_token_ids=prompt_token_ids,
+                prompt_embeds=prompt_embeds,
             )
         else:
             parsed_prompts = cast(Union[PromptType, Sequence[PromptType]],
                                   prompts)
-
-        if prompt_embeds is not None:
-            parsed_prompts.prompt_embeds = prompt_embeds
+            if prompt_embeds is not None and hasattr(parsed_prompts, "prompt_embeds"):
+                parsed_prompts.prompt_embeds = prompt_embeds
 
         if isinstance(guided_options_request, dict):
             if len(guided_options_request) > 1:
@@ -1229,8 +1246,8 @@ def wake_up(self):
     # LEGACY
     def _convert_v1_inputs(
         self,
-        prompts: Optional[Union[str, List[str]]],
-        prompt_token_ids: Optional[Union[List[int], List[List[int]]]],
+        prompts: Optional[Union[str, list[str]]],
+        prompt_token_ids: Optional[Union[list[int], list[list[int]]]],
         prompt_embeds: Optional[torch.Tensor] = None,
     ):
         # skip_tokenizer_init is now checked in engine
@@ -1269,6 +1286,13 @@ def _convert_v1_inputs(
 
             parsed_prompts.append(item)
 
+        # Handle prompt_embeds if provided
+        if prompt_embeds is not None:
+            # Assuming prompt_embeds is a tensor that can be assigned to the first prompt
+            # This might need adjustment based on how prompt_embeds is actually used
+            if len(parsed_prompts) > 0 and hasattr(parsed_prompts[0], "prompt_embeds"):
+                parsed_prompts[0].prompt_embeds = prompt_embeds
+
         return parsed_prompts
 
     def _validate_and_add_requests(
@@ -1403,3 +1427,4 @@ def _run_engine(
         # This is necessary because some requests may be finished earlier than
         # its previous requests.
         return sorted(outputs, key=lambda x: int(x.request_id))
+

vllm/model_executor/models/qwen2.py

@@ -461,9 +461,7 @@ def forward(
         inputs_embeds: Optional[torch.Tensor] = None,
     ) -> Union[torch.Tensor, IntermediateTensors]:
 
-        hidden_states = self.model(input_ids, positions, kv_caches,
-                                   attn_metadata, intermediate_tensors,
-                                   inputs_embeds, self.lm_head.bias)
+        hidden_states = self.model(input_ids, positions,intermediate_tensors, inputs_embeds, self.lm_head.bias)
         return hidden_states
 
     def compute_logits(