Map mistral tokenizer to llama

elixir-nx · Oct 21, 2023 · 3c2e563 · 3c2e563
1 parent 17846e0
commit 3c2e563
Showing 1 changed file with 1 addition and 0 deletions.
diff --git a/lib/bumblebee.ex b/lib/bumblebee.ex
@@ -217,6 +217,7 @@ defmodule Bumblebee do
     "gpt2" => Bumblebee.Text.Gpt2Tokenizer,
     "layoutlm" => Bumblebee.Text.LayoutLmTokenizer,
     "llama" => Bumblebee.Text.LlamaTokenizer,
+    "mistral" => Bumblebee.Text.LlamaTokenizer,
     "mbart" => Bumblebee.Text.MbartTokenizer,
     "roberta" => Bumblebee.Text.RobertaTokenizer,
     "t5" => Bumblebee.Text.T5Tokenizer,