From 259867ba434c865b087f4d419c522b6b9fa8367b Mon Sep 17 00:00:00 2001 From: ydshieh Date: Thu, 21 Apr 2022 10:22:07 +0200 Subject: [PATCH 1/2] add missing entries in some mappings --- .../models/auto/feature_extraction_auto.py | 4 ++++ .../models/auto/processing_auto.py | 6 ++++++ .../models/auto/tokenization_auto.py | 19 +++++++++++++++++++ 3 files changed, 29 insertions(+) diff --git a/src/transformers/models/auto/feature_extraction_auto.py b/src/transformers/models/auto/feature_extraction_auto.py index dad7e165e8d721..552e3b7ae418f5 100644 --- a/src/transformers/models/auto/feature_extraction_auto.py +++ b/src/transformers/models/auto/feature_extraction_auto.py @@ -57,6 +57,10 @@ ("regnet", "ConvNextFeatureExtractor"), ("poolformer", "PoolFormerFeatureExtractor"), ("maskformer", "MaskFormerFeatureExtractor"), + ("data2vec-audio", "Wav2Vec2FeatureExtractor"), + ("data2vec-vision", "BeitFeatureExtractor"), + ("dpt", "DPTFeatureExtractor"), + ("glpn", "GLPNFeatureExtractor"), ] ) diff --git a/src/transformers/models/auto/processing_auto.py b/src/transformers/models/auto/processing_auto.py index b51ef9ef312e10..ca49db286a21f9 100644 --- a/src/transformers/models/auto/processing_auto.py +++ b/src/transformers/models/auto/processing_auto.py @@ -46,6 +46,12 @@ ("wav2vec2", "Wav2Vec2Processor"), ("wav2vec2_with_lm", "Wav2Vec2ProcessorWithLM"), ("vision-text-dual-encoder", "VisionTextDualEncoderProcessor"), + ("unispeech", "Wav2Vec2Processor"), + ("unispeech-sat", "Wav2Vec2Processor"), + ("sew", "Wav2Vec2Processor"), + ("sew-d", "Wav2Vec2Processor"), + ("vilt", "ViltProcessor"), + ("wavlm", "Wav2Vec2Processor"), ] ) diff --git a/src/transformers/models/auto/tokenization_auto.py b/src/transformers/models/auto/tokenization_auto.py index fb275a2037f2d8..8541a4eeeb9cfd 100644 --- a/src/transformers/models/auto/tokenization_auto.py +++ b/src/transformers/models/auto/tokenization_auto.py @@ -241,6 +241,25 @@ "XGLMTokenizerFast" if is_tokenizers_available() else None, ), ), + ("visual_bert", ("BertTokenizer", "BertTokenizerFast" if is_tokenizers_available() else None)), + ("gptj", ("GPT2Tokenizer", "GPT2TokenizerFast" if is_tokenizers_available() else None)), + ("megatron-bert", ("BertTokenizer", "BertTokenizerFast" if is_tokenizers_available() else None)), + ( + "nystromformer", + ( + "AlbertTokenizer" if is_sentencepiece_available() else None, + "AlbertTokenizerFast" if is_tokenizers_available() else None, + ), + ), + ("xlm-roberta-xl", ("RobertaTokenizer", "RobertaTokenizerFast" if is_tokenizers_available() else None)), + ( + "yoso", + ( + "AlbertTokenizer" if is_sentencepiece_available() else None, + "AlbertTokenizerFast" if is_tokenizers_available() else None, + ), + ), + ("data2vec-text", ("RobertaTokenizer", "RobertaTokenizerFast" if is_tokenizers_available() else None)), ] ) From 21ecef4bb53497dddcc6e13819d3ea7ad47ac5b2 Mon Sep 17 00:00:00 2001 From: ydshieh Date: Fri, 22 Apr 2022 10:11:37 +0200 Subject: [PATCH 2/2] rebase --- src/transformers/models/auto/tokenization_auto.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/transformers/models/auto/tokenization_auto.py b/src/transformers/models/auto/tokenization_auto.py index 8541a4eeeb9cfd..0c0059c7c65751 100644 --- a/src/transformers/models/auto/tokenization_auto.py +++ b/src/transformers/models/auto/tokenization_auto.py @@ -242,7 +242,6 @@ ), ), ("visual_bert", ("BertTokenizer", "BertTokenizerFast" if is_tokenizers_available() else None)), - ("gptj", ("GPT2Tokenizer", "GPT2TokenizerFast" if is_tokenizers_available() else None)), ("megatron-bert", ("BertTokenizer", "BertTokenizerFast" if is_tokenizers_available() else None)), ( "nystromformer",