Update tiny models and pipeline tests (huggingface#23446)

* fix --------- Co-authored-by: ydshieh <ydshieh@users.noreply.github.com>
gojiteji · Jun 5, 2023 · 53f61d3 · 53f61d3
1 parent 7776ce6
commit 53f61d3
Show file tree

Hide file tree

Showing 9 changed files with 187 additions and 12 deletions.
diff --git a/tests/models/biogpt/test_modeling_biogpt.py b/tests/models/biogpt/test_modeling_biogpt.py
@@ -289,9 +289,10 @@ class BioGptModelTest(ModelTesterMixin, GenerationTesterMixin, PipelineTesterMix
     pipeline_model_mapping = (
         {
             "feature-extraction": BioGptModel,
+            "text-classification": BioGptForSequenceClassification,
             "text-generation": BioGptForCausalLM,
             "token-classification": BioGptForTokenClassification,
-            "text-classification": BioGptForSequenceClassification,
+            "zero-shot": BioGptForSequenceClassification,
         }
         if is_torch_available()
         else {}

diff --git a/tests/models/cpmant/test_modeling_cpmant.py b/tests/models/cpmant/test_modeling_cpmant.py
@@ -21,6 +21,7 @@
 from ...generation.test_utils import torch_device
 from ...test_configuration_common import ConfigTester
 from ...test_modeling_common import ModelTesterMixin, ids_tensor
+from ...test_pipeline_mixin import PipelineTesterMixin
 
 
 if is_torch_available():
@@ -133,8 +134,11 @@ def prepare_config_and_inputs_for_common(self):
 
 
 @require_torch
-class CpmAntModelTest(ModelTesterMixin, unittest.TestCase):
+class CpmAntModelTest(ModelTesterMixin, PipelineTesterMixin, unittest.TestCase):
     all_model_classes = (CpmAntModel, CpmAntForCausalLM) if is_torch_available() else ()
+    pipeline_model_mapping = (
+        {"feature-extraction": CpmAntModel, "text-generation": CpmAntForCausalLM} if is_torch_available() else {}
+    )
 
     test_pruning = False
     test_missing_keys = False

diff --git a/tests/models/focalnet/test_modeling_focalnet.py b/tests/models/focalnet/test_modeling_focalnet.py
@@ -25,6 +25,7 @@
 from ...test_backbone_common import BackboneTesterMixin
 from ...test_configuration_common import ConfigTester
 from ...test_modeling_common import ModelTesterMixin, _config_zero_init, floats_tensor, ids_tensor
+from ...test_pipeline_mixin import PipelineTesterMixin
 
 
 if is_torch_available():
@@ -226,7 +227,7 @@ def prepare_config_and_inputs_for_common(self):
 
 
 @require_torch
-class FocalNetModelTest(ModelTesterMixin, unittest.TestCase):
+class FocalNetModelTest(ModelTesterMixin, PipelineTesterMixin, unittest.TestCase):
     all_model_classes = (
         (
             FocalNetModel,
@@ -237,6 +238,11 @@ class FocalNetModelTest(ModelTesterMixin, unittest.TestCase):
         if is_torch_available()
         else ()
     )
+    pipeline_model_mapping = (
+        {"feature-extraction": FocalNetModel, "image-classification": FocalNetForImageClassification}
+        if is_torch_available()
+        else {}
+    )
     fx_compatible = False
 
     test_pruning = False

diff --git a/tests/models/gpt_neo/test_modeling_gpt_neo.py b/tests/models/gpt_neo/test_modeling_gpt_neo.py
@@ -413,8 +413,8 @@ class GPTNeoModelTest(ModelTesterMixin, GenerationTesterMixin, PipelineTesterMix
             "feature-extraction": GPTNeoModel,
             "question-answering": GPTNeoForQuestionAnswering,
             "text-classification": GPTNeoForSequenceClassification,
-            "token-classification": GPTNeoForTokenClassification,
             "text-generation": GPTNeoForCausalLM,
+            "token-classification": GPTNeoForTokenClassification,
             "zero-shot": GPTNeoForSequenceClassification,
         }
         if is_torch_available()

diff --git a/tests/models/gpt_neox/test_modeling_gpt_neox.py b/tests/models/gpt_neox/test_modeling_gpt_neox.py
@@ -239,8 +239,8 @@ class GPTNeoXModelTest(ModelTesterMixin, GenerationTesterMixin, PipelineTesterMi
             "feature-extraction": GPTNeoXModel,
             "question-answering": GPTNeoXForQuestionAnswering,
             "text-classification": GPTNeoXForSequenceClassification,
-            "token-classification": GPTNeoXForTokenClassification,
             "text-generation": GPTNeoXForCausalLM,
+            "token-classification": GPTNeoXForTokenClassification,
             "zero-shot": GPTNeoXForSequenceClassification,
         }
         if is_torch_available()

diff --git a/tests/models/rwkv/test_modeling_rwkv.py b/tests/models/rwkv/test_modeling_rwkv.py
@@ -262,12 +262,7 @@ def prepare_config_and_inputs_for_common(self):
 class RwkvModelTest(ModelTesterMixin, GenerationTesterMixin, PipelineTesterMixin, unittest.TestCase):
     all_model_classes = (RwkvModel, RwkvForCausalLM) if is_torch_available() else ()
     pipeline_model_mapping = (
-        {
-            "feature-extraction": RwkvModel,
-            "text-generation": RwkvForCausalLM,
-        }
-        if is_torch_available()
-        else {}
+        {"feature-extraction": RwkvModel, "text-generation": RwkvForCausalLM} if is_torch_available() else {}
     )
     # all_generative_model_classes = (RwkvForCausalLM,) if is_torch_available() else ()
     fx_compatible = False

diff --git a/tests/models/swiftformer/test_modeling_swiftformer.py b/tests/models/swiftformer/test_modeling_swiftformer.py
@@ -139,7 +139,6 @@ class SwiftFormerModelTest(ModelTesterMixin, PipelineTesterMixin, unittest.TestC
     """
 
     all_model_classes = (SwiftFormerModel, SwiftFormerForImageClassification) if is_torch_available() else ()
-
     pipeline_model_mapping = (
         {"feature-extraction": SwiftFormerModel, "image-classification": SwiftFormerForImageClassification}
         if is_torch_available()

diff --git a/tests/test_pipeline_mixin.py b/tests/test_pipeline_mixin.py
@@ -263,6 +263,15 @@ def run_pipeline_test(self, task, repo_name, model_architecture, tokenizer_name,
             )
             return
 
+        pipeline_test_class_name = pipeline_test_mapping[task]["test"].__name__
+        if self.is_pipeline_test_to_skip_more(pipeline_test_class_name, model.config, model, tokenizer, processor):
+            logger.warning(
+                f"{self.__class__.__name__}::test_pipeline_{task.replace('-', '_')} is skipped: test is "
+                f"currently known to fail for: model `{model_architecture.__name__}` | tokenizer "
+                f"`{tokenizer_name}` | processor `{processor_name}`."
+            )
+            return
+
         # validate
         validate_test_components(self, task, model, tokenizer, processor)
 
@@ -440,6 +449,10 @@ def test_pipeline_zero_shot_object_detection(self):
     def is_pipeline_test_to_skip(
         self, pipeline_test_casse_name, config_class, model_architecture, tokenizer_name, processor_name
     ):
+        """Skip some tests based on the classes or their names without the instantiated objects.
+
+        This is to avoid calling `from_pretrained` (so reducing the runtime) if we already know the tests will fail.
+        """
         # No fix is required for this case.
         if (
             pipeline_test_casse_name == "DocumentQuestionAnsweringPipelineTests"
@@ -451,6 +464,20 @@ def is_pipeline_test_to_skip(
 
         return False
 
+    def is_pipeline_test_to_skip_more(self, pipeline_test_casse_name, config, model, tokenizer, processor):  # noqa
+        """Skip some more tests based on the information from the instantiated objects."""
+        # No fix is required for this case.
+        if (
+            pipeline_test_casse_name == "QAPipelineTests"
+            and tokenizer is not None
+            and getattr(tokenizer, "pad_token", None) is None
+            and not tokenizer.__class__.__name__.endswith("Fast")
+        ):
+            # `QAPipelineTests` doesn't work with a slow tokenizer that has no pad token.
+            return True
+
+        return False
+
 
 def validate_test_components(test_case, task, model, tokenizer, processor):
     # TODO: Move this to tiny model creation script

diff --git a/tests/utils/tiny_model_summary.json b/tests/utils/tiny_model_summary.json
@@ -477,6 +477,16 @@
         ],
         "sha": "07073b31da84054fd12226e3cae4cb3beb2547f9"
     },
+    "BioGptForSequenceClassification": {
+        "tokenizer_classes": [
+            "BioGptTokenizer"
+        ],
+        "processor_classes": [],
+        "model_classes": [
+            "BioGptForSequenceClassification"
+        ],
+        "sha": "8e18ad6218abd795e050dec324a8c827ccedacb4"
+    },
     "BioGptForTokenClassification": {
         "tokenizer_classes": [
             "BioGptTokenizer"
@@ -2131,6 +2141,46 @@
         ],
         "sha": "31ebf1b7a0ef1fd5059b98e28e5ab1c366d2c482"
     },
+    "FocalNetBackbone": {
+        "tokenizer_classes": [],
+        "processor_classes": [
+            "BitImageProcessor"
+        ],
+        "model_classes": [
+            "FocalNetBackbone"
+        ],
+        "sha": "eb8c580969443cb87de7dd9a256deaface03692f"
+    },
+    "FocalNetForImageClassification": {
+        "tokenizer_classes": [],
+        "processor_classes": [
+            "BitImageProcessor"
+        ],
+        "model_classes": [
+            "FocalNetForImageClassification"
+        ],
+        "sha": "28d30ded26a3213e8fb7011a455afc3aa98b0a95"
+    },
+    "FocalNetForMaskedImageModeling": {
+        "tokenizer_classes": [],
+        "processor_classes": [
+            "BitImageProcessor"
+        ],
+        "model_classes": [
+            "FocalNetForMaskedImageModeling"
+        ],
+        "sha": "0ea7626d19c9dd2f3113d977f643a1babc720bd3"
+    },
+    "FocalNetModel": {
+        "tokenizer_classes": [],
+        "processor_classes": [
+            "BitImageProcessor"
+        ],
+        "model_classes": [
+            "FocalNetModel"
+        ],
+        "sha": "107b004e6aa14108a359b7d22bdb9aa141ec05d5"
+    },
     "FunnelBaseModel": {
         "tokenizer_classes": [
             "FunnelTokenizer",
@@ -2247,6 +2297,17 @@
         ],
         "sha": "24a8dbb48b1aa0ba2eba44324fcd0c78cca64dd4"
     },
+    "GPT2ForQuestionAnswering": {
+        "tokenizer_classes": [
+            "GPT2Tokenizer",
+            "GPT2TokenizerFast"
+        ],
+        "processor_classes": [],
+        "model_classes": [
+            "GPT2ForQuestionAnswering"
+        ],
+        "sha": "a5bdd6bd4d79feece85ea9a8bd4ee5fe54c1d45b"
+    },
     "GPT2ForSequenceClassification": {
         "tokenizer_classes": [
             "GPT2Tokenizer",
@@ -2397,6 +2458,17 @@
         ],
         "sha": "e88934e402c15195dd99b2947632415dd7645268"
     },
+    "GPTNeoForQuestionAnswering": {
+        "tokenizer_classes": [
+            "GPT2Tokenizer",
+            "GPT2TokenizerFast"
+        ],
+        "processor_classes": [],
+        "model_classes": [
+            "GPTNeoForQuestionAnswering"
+        ],
+        "sha": "623883e94bd08caf9b3f839b98debeea72d5bc2b"
+    },
     "GPTNeoForSequenceClassification": {
         "tokenizer_classes": [
             "GPT2Tokenizer",
@@ -2408,6 +2480,17 @@
         ],
         "sha": "bf2090d5d91a70eb37ba51fbdcf23afc7031fea8"
     },
+    "GPTNeoForTokenClassification": {
+        "tokenizer_classes": [
+            "GPT2Tokenizer",
+            "GPT2TokenizerFast"
+        ],
+        "processor_classes": [],
+        "model_classes": [
+            "GPTNeoForTokenClassification"
+        ],
+        "sha": "d5208e73e24a1671219776b50fe5f96e0e4cd218"
+    },
     "GPTNeoModel": {
         "tokenizer_classes": [
             "GPT2Tokenizer",
@@ -2429,6 +2512,16 @@
         ],
         "sha": "0229cfaaa843c6b492ac2abffabb00f1ff1936f8"
     },
+    "GPTNeoXForQuestionAnswering": {
+        "tokenizer_classes": [
+            "GPTNeoXTokenizerFast"
+        ],
+        "processor_classes": [],
+        "model_classes": [
+            "GPTNeoXForQuestionAnswering"
+        ],
+        "sha": "7d2f08c959c211129952ee03b5562add09fe6864"
+    },
     "GPTNeoXForSequenceClassification": {
         "tokenizer_classes": [
             "GPTNeoXTokenizerFast"
@@ -2439,6 +2532,16 @@
         ],
         "sha": "17c4b845ee2e0bb780ca2dea2d59a3d9d5d3c651"
     },
+    "GPTNeoXForTokenClassification": {
+        "tokenizer_classes": [
+            "GPTNeoXTokenizerFast"
+        ],
+        "processor_classes": [],
+        "model_classes": [
+            "GPTNeoXForTokenClassification"
+        ],
+        "sha": "3aa4fe8a562f32230041d6d3616aa5ecc3f30192"
+    },
     "GPTNeoXJapaneseForCausalLM": {
         "tokenizer_classes": [
             "GPTNeoXJapaneseTokenizer"
@@ -4922,6 +5025,26 @@
         ],
         "sha": "4830db38fd310404c5ab70bd00684eca0bc06ca8"
     },
+    "RwkvForCausalLM": {
+        "tokenizer_classes": [
+            "GPTNeoXTokenizerFast"
+        ],
+        "processor_classes": [],
+        "model_classes": [
+            "RwkvForCausalLM"
+        ],
+        "sha": "2f452fd46b39e39b1a6a95fa1d8232405bbb3e96"
+    },
+    "RwkvModel": {
+        "tokenizer_classes": [
+            "GPTNeoXTokenizerFast"
+        ],
+        "processor_classes": [],
+        "model_classes": [
+            "RwkvModel"
+        ],
+        "sha": "88a52c9437dc3c06f65a8252490be7eb91197804"
+    },
     "SEWDForCTC": {
         "tokenizer_classes": [
             "Wav2Vec2CTCTokenizer"
@@ -5196,6 +5319,26 @@
         ],
         "sha": "e0a3ac56a4047da3f921638252ead5e44438bbdb"
     },
+    "SwiftFormerForImageClassification": {
+        "tokenizer_classes": [],
+        "processor_classes": [
+            "ViTImageProcessor"
+        ],
+        "model_classes": [
+            "SwiftFormerForImageClassification"
+        ],
+        "sha": "a249b14a525d29e675b6e4af4baacd9ba7df7598"
+    },
+    "SwiftFormerModel": {
+        "tokenizer_classes": [],
+        "processor_classes": [
+            "ViTImageProcessor"
+        ],
+        "model_classes": [
+            "SwiftFormerModel"
+        ],
+        "sha": "25ba2d88c770533f8c69811d2a454a00c1d09f5d"
+    },
     "Swin2SRModel": {
         "tokenizer_classes": [],
         "processor_classes": [