[BugFix] Fix seeded random sampling with encoder-decoder models (vllm…

…-project#8870) Co-authored-by: Roger Wang <ywang@roblox.com>
siddvenk · Oct 16, 2024 · 71d6071 · 71d6071
1 parent 7193774
commit 71d6071
Showing 1 changed file with 3 additions and 1 deletion.
diff --git a/vllm/worker/enc_dec_model_runner.py b/vllm/worker/enc_dec_model_runner.py
@@ -268,11 +268,13 @@ def prepare_model_input(
             encoder_input_positions=encoder_input_positions_tensor,
         )
 
+        generators = self.get_generators(finished_requests_ids)
         sampling_metadata = SamplingMetadata.prepare(seq_group_metadata_list,
                                                      model_input.seq_lens,
                                                      model_input.query_lens,
                                                      self.device,
-                                                     self.pin_memory)
+                                                     self.pin_memory,
+                                                     generators=generators)
         is_prompt = (seq_group_metadata_list[0].is_prompt
                      if seq_group_metadata_list else None)
         return dataclasses.replace(model_input,