Adds HuggingFace pretrained encoder unit tests

ludwig/encoders/text_encoders.py

@@ -1338,7 +1338,7 @@ def input_dtype(self):
 @DeveloperAPI
 @register_encoder("camembert", TEXT)
 class CamemBERTEncoder(HFTextEncoder):
-    DEFAULT_MODEL_NAME = "jplu/camembert-base"
+    DEFAULT_MODEL_NAME = "camembert-base"
 
     def __init__(
         self,

ludwig/schema/metadata/configs/encoders.yaml

@@ -1008,7 +1008,7 @@ CamemBERT:
         short_description:
             Language model trained on large French text corpus.
         long_description:
-            The camembert encoder loads a pretrained CamemBERT (default jplu/tf-camembert-base) model using
+            The camembert encoder loads a pretrained CamemBERT (default camembert-base) model using
             the Hugging Face transformers package. CamemBERT is pre-trained on 138GB of French text.
         literature_references:
             - https://arxiv.org/abs/1911.03894

tests/ludwig/encoders/test_text_encoders.py

@@ -3,7 +3,61 @@
 
 from ludwig.encoders import text_encoders
 from tests.integration_tests.parameter_update_utils import check_module_parameters_updated
-from tests.integration_tests.utils import slow
+
+
+@pytest.mark.parametrize(
+    "encoder_cls",
+    [
+        text_encoders.ALBERTEncoder,
+        text_encoders.BERTEncoder,
+        text_encoders.XLMEncoder,
+        text_encoders.GPTEncoder,
+        text_encoders.RoBERTaEncoder,
+        text_encoders.GPT2Encoder,
+        text_encoders.DistilBERTEncoder,
+        text_encoders.TransformerXLEncoder,
+        text_encoders.CTRLEncoder,
+        text_encoders.CamemBERTEncoder,
+        text_encoders.MT5Encoder,
+        text_encoders.XLMRoBERTaEncoder,
+        text_encoders.LongformerEncoder,
+        text_encoders.ELECTRAEncoder,
+        text_encoders.FlauBERTEncoder,
+        text_encoders.T5Encoder,
+        text_encoders.XLNetEncoder,
+        text_encoders.DistilBERTEncoder,
+    ],
+)
+def test_hf_pretrained_default_exists(tmpdir, encoder_cls: text_encoders.HFTextEncoder):
+    """Test that the default pretrained model exists on the HuggingFace Hub.
+
+    This test merely checks that the default model name is valid. It does not check
+    the model end-to-end, as that would require downloading the model weights, which
+    can cause problems in the CI due to memory/runtime constraints.
+
+    TODO: add an end-to-end test for pretrained HF encoders.
+    """
+    from huggingface_hub import HfApi
+
+    hf_api = HfApi()
+    hf_api.model_info(encoder_cls.DEFAULT_MODEL_NAME)
+
+
+@pytest.mark.parametrize("pretrained_model_name_or_path", ["bert-base-uncased"])
+@pytest.mark.parametrize("reduce_output", [None, "sum", "cls_pooled"])
+@pytest.mark.parametrize("max_sequence_length", [20])
+def test_auto_transformer_encoder(
+    tmpdir, pretrained_model_name_or_path: str, reduce_output: str, max_sequence_length: int
+):
+    encoder = text_encoders.AutoTransformerEncoder(
+        pretrained_model_name_or_path=pretrained_model_name_or_path,
+        reduce_output=reduce_output,
+        max_sequence_length=max_sequence_length,
+        pretrained_kwargs=dict(cache_dir=tmpdir),
+    )
+    inputs = torch.rand((2, max_sequence_length)).type(encoder.input_dtype)
+    outputs = encoder(inputs)
+    assert outputs["encoder_output"].shape[1:] == encoder.output_shape
 
 
 @pytest.mark.parametrize("use_pretrained", [False])
@@ -34,7 +88,6 @@ def test_bert_encoder(use_pretrained: bool, reduce_output: str, max_sequence_len
     assert outputs["encoder_output"].shape[1:] == bert.output_shape
 
 
-@slow
 @pytest.mark.parametrize("use_pretrained", [False])
 @pytest.mark.parametrize("reduce_output", ["last", "sum", "mean"])
 @pytest.mark.parametrize("max_sequence_length", [20])
@@ -49,7 +102,6 @@ def test_xlm_encoder(use_pretrained: bool, reduce_output: str, max_sequence_leng
     assert outputs["encoder_output"].shape[1:] == xlm_encoder.output_shape
 
 
-@slow
 @pytest.mark.parametrize("use_pretrained", [False])
 @pytest.mark.parametrize("reduce_output", [None, "sum"])
 @pytest.mark.parametrize("max_sequence_length", [20])
@@ -78,8 +130,7 @@ def test_roberta_encoder(use_pretrained: bool, reduce_output: str, max_sequence_
     assert outputs["encoder_output"].shape[1:] == roberta_encoder.output_shape
 
 
-@slow
-@pytest.mark.parametrize("use_pretrained", [True, False])
+@pytest.mark.parametrize("use_pretrained", [False])
 @pytest.mark.parametrize("reduce_output", [None, "sum"])
 @pytest.mark.parametrize("max_sequence_length", [20])
 def test_gpt2_encoder(use_pretrained: bool, reduce_output: str, max_sequence_length: int):
@@ -121,7 +172,6 @@ def test_transfoxl_encoder(use_pretrained: bool, reduce_output: str, max_sequenc
     assert outputs["encoder_output"].shape[1:] == transfo.output_shape
 
 
-@slow
 @pytest.mark.parametrize("use_pretrained", [False])
 @pytest.mark.parametrize("reduce_output", [None, "sum"])
 @pytest.mark.parametrize("max_sequence_length", [20])
@@ -136,7 +186,6 @@ def test_ctrl_encoder(use_pretrained: bool, reduce_output: str, max_sequence_len
     assert outputs["encoder_output"].shape[1:] == encoder.output_shape
 
 
-@slow
 @pytest.mark.parametrize("use_pretrained", [False])
 @pytest.mark.parametrize("reduce_output", [None, "cls_pooled"])
 @pytest.mark.parametrize("max_sequence_length", [20])
@@ -165,7 +214,6 @@ def test_mt5_encoder(use_pretrained: bool, reduce_output: str, max_sequence_leng
     assert outputs["encoder_output"].shape[1:] == mt5_encoder.output_shape
 
 
-@slow
 @pytest.mark.parametrize("use_pretrained", [False])
 @pytest.mark.parametrize("reduce_output", [None, "sum"])
 @pytest.mark.parametrize("max_sequence_length", [20])
@@ -192,7 +240,6 @@ def test_longformer_encoder(use_pretrained: bool, reduce_output: str, max_sequen
     assert outputs["encoder_output"].shape[1:] == encoder.output_shape
 
 
-@slow
 @pytest.mark.parametrize("use_pretrained", [False])
 @pytest.mark.parametrize("reduce_output", [None, "sum"])
 @pytest.mark.parametrize("max_sequence_length", [20])
@@ -205,21 +252,6 @@ def test_electra_encoder(use_pretrained: bool, reduce_output: str, max_sequence_
     assert outputs["encoder_output"].shape[1:] == encoder.output_shape
 
 
-@pytest.mark.parametrize("pretrained_model_name_or_path", ["bert-base-uncased"])
-@pytest.mark.parametrize("reduce_output", [None, "sum", "cls_pooled"])
-@pytest.mark.parametrize("max_sequence_length", [20])
-def test_auto_transformer_encoder(pretrained_model_name_or_path: str, reduce_output: str, max_sequence_length: int):
-    encoder = text_encoders.AutoTransformerEncoder(
-        pretrained_model_name_or_path=pretrained_model_name_or_path,
-        reduce_output=reduce_output,
-        max_sequence_length=max_sequence_length,
-    )
-    inputs = torch.rand((2, max_sequence_length)).type(encoder.input_dtype)
-    outputs = encoder(inputs)
-    assert outputs["encoder_output"].shape[1:] == encoder.output_shape
-
-
-@slow
 @pytest.mark.parametrize("use_pretrained", [False])
 @pytest.mark.parametrize("reduce_output", [None, "sum"])
 @pytest.mark.parametrize("max_sequence_length", [20])
@@ -244,7 +276,6 @@ def test_t5_encoder(use_pretrained: bool, reduce_output: str, max_sequence_lengt
     assert outputs["encoder_output"].shape[1:] == encoder.output_shape
 
 
-@slow
 @pytest.mark.parametrize("use_pretrained", [False])
 @pytest.mark.parametrize("reduce_output", [None, "sum"])
 @pytest.mark.parametrize("max_sequence_length", [20])