update AutoTokenizer

shibing624 · May 31, 2024 · a3a402d · a3a402d
1 parent d0b487e
commit a3a402d
Show file tree

Hide file tree

Showing 11 changed files with 7 additions and 18 deletions.
diff --git a/dpo_training.py b/dpo_training.py
@@ -18,7 +18,6 @@
     BloomForCausalLM,
     AutoModelForCausalLM,
     AutoModel,
-    LlamaTokenizer,
     LlamaForCausalLM,
     BloomTokenizerFast,
     AutoTokenizer,
@@ -37,7 +36,7 @@
 MODEL_CLASSES = {
     "bloom": (AutoConfig, BloomForCausalLM, BloomTokenizerFast),
     "chatglm": (AutoConfig, AutoModel, AutoTokenizer),
-    "llama": (AutoConfig, LlamaForCausalLM, LlamaTokenizer),
+    "llama": (AutoConfig, LlamaForCausalLM, AutoTokenizer),
     "baichuan": (AutoConfig, AutoModelForCausalLM, AutoTokenizer),
     "auto": (AutoConfig, AutoModelForCausalLM, AutoTokenizer),
 }

diff --git a/fastapi_server_demo.py b/fastapi_server_demo.py
@@ -31,7 +31,6 @@
     AutoTokenizer,
     BloomForCausalLM,
     BloomTokenizerFast,
-    LlamaTokenizer,
     LlamaForCausalLM,
     TextIteratorStreamer,
     GenerationConfig,
@@ -42,7 +41,7 @@
 MODEL_CLASSES = {
     "bloom": (BloomForCausalLM, BloomTokenizerFast),
     "chatglm": (AutoModel, AutoTokenizer),
-    "llama": (LlamaForCausalLM, LlamaTokenizer),
+    "llama": (LlamaForCausalLM, AutoTokenizer),
     "baichuan": (AutoModelForCausalLM, AutoTokenizer),
     "auto": (AutoModelForCausalLM, AutoTokenizer),
 }

diff --git a/gradio_demo.py b/gradio_demo.py
@@ -17,7 +17,6 @@
     AutoModelForCausalLM,
     BloomForCausalLM,
     BloomTokenizerFast,
-    LlamaTokenizer,
     LlamaForCausalLM,
     GenerationConfig,
     TextIteratorStreamer,
@@ -28,7 +27,7 @@
 MODEL_CLASSES = {
     "bloom": (BloomForCausalLM, BloomTokenizerFast),
     "chatglm": (AutoModel, AutoTokenizer),
-    "llama": (LlamaForCausalLM, LlamaTokenizer),
+    "llama": (LlamaForCausalLM, AutoTokenizer),
     "baichuan": (AutoModelForCausalLM, AutoTokenizer),
     "auto": (AutoModelForCausalLM, AutoTokenizer),
 }

diff --git a/inference.py b/inference.py
@@ -17,7 +17,6 @@
     AutoTokenizer,
     BloomForCausalLM,
     BloomTokenizerFast,
-    LlamaTokenizer,
     LlamaForCausalLM,
     TextIteratorStreamer,
     GenerationConfig,
@@ -29,7 +28,7 @@
 MODEL_CLASSES = {
     "bloom": (BloomForCausalLM, BloomTokenizerFast),
     "chatglm": (AutoModel, AutoTokenizer),
-    "llama": (LlamaForCausalLM, LlamaTokenizer),
+    "llama": (LlamaForCausalLM, AutoTokenizer),
     "baichuan": (AutoModelForCausalLM, AutoTokenizer),
     "auto": (AutoModelForCausalLM, AutoTokenizer),
 }

diff --git a/inference_multigpu_demo.py b/inference_multigpu_demo.py
@@ -23,7 +23,6 @@
     AutoTokenizer,
     BloomForCausalLM,
     BloomTokenizerFast,
-    LlamaTokenizer,
     LlamaForCausalLM,
     GenerationConfig,
     BitsAndBytesConfig,
@@ -34,7 +33,7 @@
 MODEL_CLASSES = {
     "bloom": (BloomForCausalLM, BloomTokenizerFast),
     "chatglm": (AutoModel, AutoTokenizer),
-    "llama": (LlamaForCausalLM, LlamaTokenizer),
+    "llama": (LlamaForCausalLM, AutoTokenizer),
     "baichuan": (AutoModelForCausalLM, AutoTokenizer),
     "auto": (AutoModelForCausalLM, AutoTokenizer),
 }

diff --git a/merge_peft_adapter.py b/merge_peft_adapter.py
@@ -22,15 +22,14 @@
     BloomForCausalLM,
     BloomTokenizerFast,
     AutoModelForCausalLM,
-    LlamaTokenizer,
     LlamaForCausalLM,
     AutoModelForSequenceClassification,
 )
 
 MODEL_CLASSES = {
     "bloom": (BloomForCausalLM, BloomTokenizerFast),
     "chatglm": (AutoModel, AutoTokenizer),
-    "llama": (LlamaForCausalLM, LlamaTokenizer),
+    "llama": (LlamaForCausalLM, AutoTokenizer),
     "baichuan": (AutoModelForCausalLM, AutoTokenizer),
     "auto": (AutoModelForCausalLM, AutoTokenizer),
 }

diff --git a/orpo_training.py b/orpo_training.py
@@ -17,7 +17,6 @@
     BloomForCausalLM,
     AutoModelForCausalLM,
     AutoModel,
-    LlamaTokenizer,
     LlamaForCausalLM,
     BloomTokenizerFast,
     AutoTokenizer,

diff --git a/ppo_training.py b/ppo_training.py
@@ -20,7 +20,6 @@
     BloomForCausalLM,
     AutoModelForCausalLM,
     AutoModel,
-    LlamaTokenizer,
     LlamaForCausalLM,
     BloomTokenizerFast,
     AutoTokenizer,

diff --git a/pretraining.py b/pretraining.py
@@ -35,7 +35,6 @@
     BloomForCausalLM,
     AutoModelForCausalLM,
     AutoModel,
-    LlamaTokenizer,
     LlamaForCausalLM,
     BloomTokenizerFast,
     AutoTokenizer,

diff --git a/reward_modeling.py b/reward_modeling.py
@@ -21,7 +21,6 @@
     PreTrainedTokenizerBase,
     BloomForSequenceClassification,
     LlamaForSequenceClassification,
-    LlamaTokenizer,
     BloomTokenizerFast,
     AlbertForSequenceClassification,
     BertForSequenceClassification,
@@ -44,7 +43,7 @@
     "roberta": (AutoConfig, RobertaForSequenceClassification, RobertaTokenizer),
     "albert": (AutoConfig, AlbertForSequenceClassification, AutoTokenizer),
     "bloom": (AutoConfig, BloomForSequenceClassification, BloomTokenizerFast),
-    "llama": (AutoConfig, LlamaForSequenceClassification, LlamaTokenizer),
+    "llama": (AutoConfig, LlamaForSequenceClassification, AutoTokenizer),
     "auto": (AutoConfig, AutoModelForSequenceClassification, AutoTokenizer),
 }
 

diff --git a/supervised_finetuning.py b/supervised_finetuning.py
@@ -35,7 +35,6 @@
     BloomForCausalLM,
     AutoModel,
     AutoModelForCausalLM,
-    LlamaTokenizer,
     LlamaForCausalLM,
     BloomTokenizerFast,
     AutoTokenizer,