Rebase to main

Signed-off-by: southfreebird <yvorott@gmail.com>

Author: southfreebird

tests/entrypoints/llm/test_guided_generate.py

@@ -30,8 +30,10 @@ def get_llm_kwargs(mode: str):
         if mode == "autoregressive":
             return {}
         return {
-            "speculative_model": "Qwen/Qwen2.5-0.5B-Instruct",
-            "num_speculative_tokens": 3,
+            "speculative_config": {
+                "model": "Qwen/Qwen2.5-0.5B-Instruct",
+                "num_speculative_tokens": 3,
+            },
         }
 
     test_llm_kwargs = get_llm_kwargs(request.param)

vllm/model_executor/guided_decoding/xgrammar_decoding.py

@@ -176,6 +176,7 @@ def from_guided_params(
         )
         num_lookahead_slots = (speculative_config.num_lookahead_slots
                                if speculative_config else None)
+        print("num_lookahead_slots", num_lookahead_slots)
 
         if guided_params.json:
             if not isinstance(guided_params.json, str):
@@ -274,6 +275,7 @@ def from_guided_params(
                 tokenizer_hash=tokenizer_hash,
                 max_threads=max_threads,
                 tokenizer_data=tokenizer_data,
+                num_lookahead_slots=num_lookahead_slots,
             )
         else:
             raise ValueError(
@@ -372,6 +374,7 @@ def __call__(self, input_ids: list[int],
         if len(self.matchers) == 0:
             max_rollback_tokens = (self.config.num_lookahead_slots
                                    if self.config.num_lookahead_slots else 0)
+            print("max_rollback_tokens", max_rollback_tokens)
             self.matchers = [
                 xgr.GrammarMatcher(self.ctx,
                                    max_rollback_tokens=max_rollback_tokens)