wip

Signed-off-by: Julien Denize <julien.denize@mistral.ai>

Author: juliendenize

tests/models/language/generation/test_mistral.py

@@ -208,7 +208,7 @@ def test_mistral_format(
     with vllm_runner(
         model,
         dtype=dtype,
-        tokenizer_mode="auto",
+        tokenizer_mode="hf",
         load_format="safetensors",
         config_format="hf",
     ) as hf_format_model:

tests/models/quantization/test_bitsandbytes.py

@@ -155,6 +155,7 @@ def test_4bit_bnb_moe_model(
         quantization="bitsandbytes",
         enforce_eager=False,
         default_torch_num_threads=1,
+        tokenizer_mode="hf",
     ) as llm:
         vllm_outputs = llm.generate_greedy_logprobs(
             example_prompts, max_tokens=32, num_logprobs=5
@@ -204,6 +205,7 @@ def test_4bit_bnb_embedding_model(
         gpu_memory_utilization=0.5,
         quantization="bitsandbytes",
         default_torch_num_threads=1,
+        tokenizer_mode="hf",
     ) as vllm_model:
         vllm_outputs = vllm_model.embed(example_prompts)
 
@@ -256,6 +258,7 @@ def validate_generated_texts(
         tensor_parallel_size=vllm_tp_size,
         enforce_eager=False,
         default_torch_num_threads=1,
+        tokenizer_mode="hf",
     ) as llm:
         vllm_outputs = llm.generate_greedy(prompts, max_tokens)
         vllm_logs = log_generated_texts(prompts, vllm_outputs, "VllmRunner")

vllm/config/model.py

@@ -128,7 +128,8 @@ class ModelConfig:
     name or path will be used."""
     tokenizer_mode: TokenizerMode = "auto"
     """Tokenizer mode:\n
-    - "auto" will use the fast tokenizer if available.\n
+    - "auto" will use "hf" tokenizer if Mistral's tokenizer is not available.\n
+    - "hf" will use the fast tokenizer if available.\n
     - "slow" will always use the slow tokenizer.\n
     - "mistral" will always use the tokenizer from `mistral_common`.\n
     - "custom" will use --tokenizer to select the preregistered tokenizer."""

vllm/model_executor/model_loader/default_loader.py

@@ -108,11 +108,11 @@ def _prepare_weights(
                     )
                 )
                 > 0
-                else "auto"
+                else "hf"
             )
 
         # Some quantized models use .pt files for storing the weights.
-        if load_format in ["auto", "hf"]:
+        if load_format in "hf":
             allow_patterns = ["*.safetensors", "*.bin"]
         elif load_format == "safetensors" or load_format == "fastsafetensors":
             use_safetensors = True