Skip to content

Commit 73563ca

Browse files
WoosukKwonshreyankg
authored andcommitted
[V1][Spec Decode] Use better defaults for N-gram (vllm-project#15358)
Signed-off-by: Woosuk Kwon <woosuk.kwon@berkeley.edu>
1 parent dfb2e0d commit 73563ca

File tree

1 file changed

+22
-10
lines changed

1 file changed

+22
-10
lines changed

vllm/config.py

Lines changed: 22 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -2014,18 +2014,30 @@ def __post_init__(self):
20142014
if self.method in ("ngram", "[ngram]"):
20152015
# Unified to "ngram" internally
20162016
self.method = "ngram"
2017-
if self.prompt_lookup_min is None:
2018-
self.prompt_lookup_min = 1
2019-
if self.prompt_lookup_max is None or self.prompt_lookup_max < 1:
2020-
raise ValueError("prompt_lookup_max="
2021-
f"{self.prompt_lookup_max} must be > 0")
2017+
# Set default values if not provided
2018+
if (self.prompt_lookup_min is None
2019+
and self.prompt_lookup_max is None):
2020+
# TODO(woosuk): Tune these values. They are arbitrarily chosen.
2021+
self.prompt_lookup_min = 5
2022+
self.prompt_lookup_max = 5
2023+
elif self.prompt_lookup_min is None:
2024+
assert self.prompt_lookup_max is not None
2025+
self.prompt_lookup_min = self.prompt_lookup_max
2026+
elif self.prompt_lookup_max is None:
2027+
assert self.prompt_lookup_min is not None
2028+
self.prompt_lookup_max = self.prompt_lookup_min
2029+
2030+
# Validate values
20222031
if self.prompt_lookup_min < 1:
2023-
raise ValueError("prompt_lookup_min="
2024-
f"{self.prompt_lookup_min} must be > 0")
2032+
raise ValueError(
2033+
f"prompt_lookup_min={self.prompt_lookup_min} must be > 0")
2034+
if self.prompt_lookup_max < 1:
2035+
raise ValueError(
2036+
f"prompt_lookup_max={self.prompt_lookup_max} must be > 0")
20252037
if self.prompt_lookup_min > self.prompt_lookup_max:
2026-
raise ValueError(f"prompt_lookup_min={self.prompt_lookup_min} "
2027-
"cannot be larger than prompt_lookup_max="
2028-
f"{self.prompt_lookup_max}")
2038+
raise ValueError(
2039+
f"prompt_lookup_min={self.prompt_lookup_min} must "
2040+
f"be <= prompt_lookup_max={self.prompt_lookup_max}")
20292041

20302042
# TODO: current we still need extract vocab_size from target model
20312043
# config, in future, we may try refactor it out, and set

0 commit comments

Comments
 (0)