Add RoCBert support for Bettertransformer (#542)

* Add RoCBert to overview doc Signed-off-by: Shogo Hida <shogo.hida@gmail.com> * Add RoCBertLayerBetterTransformer Signed-off-by: Shogo Hida <shogo.hida@gmail.com> * Add RoCBertLayerBetterTransformer to init Signed-off-by: Shogo Hida <shogo.hida@gmail.com> * Add tiny-random-RoCBertModel Signed-off-by: Shogo Hida <shogo.hida@gmail.com> * Run make style Signed-off-by: Shogo Hida <shogo.hida@gmail.com> * Delete RoCBertLayerBetterTransformer Signed-off-by: Shogo Hida <shogo.hida@gmail.com> * Add rocbert to init Signed-off-by: Shogo Hida <shogo.hida@gmail.com> * Fix test Signed-off-by: Shogo Hida <shogo.hida@gmail.com> * Fix test Signed-off-by: Shogo Hida <shogo.hida@gmail.com> * Change rocbert to roc_bert Signed-off-by: Shogo Hida <shogo.hida@gmail.com> * Remove tiny-random-RoCBertModel Signed-off-by: Shogo Hida <shogo.hida@gmail.com> Signed-off-by: Shogo Hida <shogo.hida@gmail.com>
huggingface · Jan 12, 2023 · b412390 · b412390
1 parent 4d764e1
commit b412390
Show file tree

Hide file tree

Showing 3 changed files with 10 additions and 0 deletions.
diff --git a/docs/source/bettertransformer/overview.mdx b/docs/source/bettertransformer/overview.mdx
@@ -43,6 +43,7 @@ The list of supported model below:
 - [M2M100](https://arxiv.org/abs/2010.11125)
 - [RemBERT](https://arxiv.org/abs/2010.12821)
 - [RoBERTa](https://arxiv.org/abs/1907.11692)
+- [RoCBert](https://aclanthology.org/2022.acl-long.65.pdf)
 - [Splinter](https://arxiv.org/abs/2101.00438)
 - [Tapas](https://arxiv.org/abs/2211.06550)
 - [ViLT](https://arxiv.org/abs/2102.03334)

diff --git a/optimum/bettertransformer/models/__init__.py b/optimum/bettertransformer/models/__init__.py
@@ -49,6 +49,7 @@ class BetterTransformerManager:
  "mbart": ("MBartEncoderLayer", MBartEncoderLayerBetterTransformer),
  "rembert": ("RemBertLayer", BertLayerBetterTransformer),
  "roberta": ("RobertaLayer", BertLayerBetterTransformer),
+ "roc_bert": ("RoCBertLayer", BertLayerBetterTransformer),
  "splinter": ("SplinterLayer", BertLayerBetterTransformer),
  "tapas": ("TapasLayer", BertLayerBetterTransformer),
  "vilt": ("ViltLayer", ViltLayerBetterTransformer),

diff --git a/tests/bettertransformer/test_bettertransformer_encoder.py b/tests/bettertransformer/test_bettertransformer_encoder.py
@@ -255,6 +255,14 @@ def test_accelerate_compatibility_single_gpu_without_keeping(self):
  self.check_accelerate_compatibility_cpu_gpu(keep_original_model=False, max_memory=max_memory)
 
 
+class BetterTransformersRoCBertTest(BetterTransformersEncoderTest):
+ all_models_to_test = ["hf-internal-testing/tiny-random-RoCBertModel"]
+
+ # unrelated issue with torch.amp.autocast with rocbert (expected scalar type BFloat16 but found Float)
+ def test_raise_autocast(self):
+ pass
+
+
 class BetterTransformersEncoderDecoderTest(BetterTransformersTestMixin, unittest.TestCase):
  r"""
  Full testing suite of the `BetterTransformers` integration into Hugging Face