Skip to content

Commit bc8ed3c

Browse files
authored
[V1][Spec Decode] Use better defaults for N-gram (#15358)
Signed-off-by: Woosuk Kwon <woosuk.kwon@berkeley.edu>
1 parent b9bd76c commit bc8ed3c

File tree

1 file changed

+22
-10
lines changed

1 file changed

+22
-10
lines changed

vllm/config.py

Lines changed: 22 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -2012,18 +2012,30 @@ def __post_init__(self):
20122012
if self.method in ("ngram", "[ngram]"):
20132013
# Unified to "ngram" internally
20142014
self.method = "ngram"
2015-
if self.prompt_lookup_min is None:
2016-
self.prompt_lookup_min = 1
2017-
if self.prompt_lookup_max is None or self.prompt_lookup_max < 1:
2018-
raise ValueError("prompt_lookup_max="
2019-
f"{self.prompt_lookup_max} must be > 0")
2015+
# Set default values if not provided
2016+
if (self.prompt_lookup_min is None
2017+
and self.prompt_lookup_max is None):
2018+
# TODO(woosuk): Tune these values. They are arbitrarily chosen.
2019+
self.prompt_lookup_min = 5
2020+
self.prompt_lookup_max = 5
2021+
elif self.prompt_lookup_min is None:
2022+
assert self.prompt_lookup_max is not None
2023+
self.prompt_lookup_min = self.prompt_lookup_max
2024+
elif self.prompt_lookup_max is None:
2025+
assert self.prompt_lookup_min is not None
2026+
self.prompt_lookup_max = self.prompt_lookup_min
2027+
2028+
# Validate values
20202029
if self.prompt_lookup_min < 1:
2021-
raise ValueError("prompt_lookup_min="
2022-
f"{self.prompt_lookup_min} must be > 0")
2030+
raise ValueError(
2031+
f"prompt_lookup_min={self.prompt_lookup_min} must be > 0")
2032+
if self.prompt_lookup_max < 1:
2033+
raise ValueError(
2034+
f"prompt_lookup_max={self.prompt_lookup_max} must be > 0")
20232035
if self.prompt_lookup_min > self.prompt_lookup_max:
2024-
raise ValueError(f"prompt_lookup_min={self.prompt_lookup_min} "
2025-
"cannot be larger than prompt_lookup_max="
2026-
f"{self.prompt_lookup_max}")
2036+
raise ValueError(
2037+
f"prompt_lookup_min={self.prompt_lookup_min} must "
2038+
f"be <= prompt_lookup_max={self.prompt_lookup_max}")
20272039

20282040
# TODO: current we still need extract vocab_size from target model
20292041
# config, in future, we may try refactor it out, and set

0 commit comments

Comments
 (0)