Skip to content

Commit

Permalink
[Bugfix] Fix missing seq_start_loc in xformers prefill metadata (vllm…
Browse files Browse the repository at this point in the history
…-project#12464)

Signed-off-by: Isotr0py <2037008807@qq.com>
  • Loading branch information
Isotr0py authored and rasmith committed Jan 30, 2025
1 parent 6753b86 commit b409e64
Showing 1 changed file with 3 additions and 0 deletions.
3 changes: 3 additions & 0 deletions vllm/attention/backends/xformers.py
Original file line number Diff line number Diff line change
Expand Up @@ -199,6 +199,8 @@ def prefill_metadata(self) -> Optional["XFormersMetadata"]:
# Compute some attn_metadata fields which default to None
query_start_loc = (None if self.query_start_loc is None else
self.query_start_loc[:self.num_prefills + 1])
seq_start_loc = (None if self.seq_start_loc is None else
self.seq_start_loc[:self.num_prefills + 1])
slot_mapping = (None if self.slot_mapping is None else
self.slot_mapping[:self.num_prefill_tokens])
seq_lens = (None if self.seq_lens is None else
Expand All @@ -225,6 +227,7 @@ def prefill_metadata(self) -> Optional["XFormersMetadata"]:
max_prefill_seq_len=self.max_prefill_seq_len,
max_decode_seq_len=0,
query_start_loc=query_start_loc,
seq_start_loc=seq_start_loc,
context_lens_tensor=context_lens_tensor,
block_tables=block_tables,
use_cuda_graph=False,
Expand Down

0 comments on commit b409e64

Please sign in to comment.