fix test not to try importing Sigliptokenizerfast

NielsRogge · Oct 2, 2024 · 2133809 · 2133809
1 parent 80d2f46
commit 2133809
Showing 1 changed file with 6 additions and 1 deletion.
diff --git a/src/transformers/convert_slow_tokenizers_checkpoints_to_fast.py b/src/transformers/convert_slow_tokenizers_checkpoints_to_fast.py
@@ -30,7 +30,12 @@
 
 TOKENIZER_CLASSES = {
  # Phi3 uses Llama tokenizer
- name: getattr(transformers, "LlamaTokenizerFast" if name == "Phi3Tokenizer" else name + "Fast")
+ name: getattr(
+ transformers,
+ "LlamaTokenizerFast"
+ if name == "Phi3Tokenizer"
+ else ("PreTrainedTokenizerFast" if name == "SiglipTokenizerFast" else name + "Fast"),
+ )
  for name in SLOW_TO_FAST_CONVERTERS
 }