[BartTokenizer] add prepare s2s batch

src/transformers/tokenization_bart.py

@@ -45,6 +45,65 @@ class BartTokenizer(RobertaTokenizer):
         "merges_file": {m: merges_url for m in _all_bart_models},
     }
 
+    def prepare_seq2seq_batch(
+        self,
+        src_texts: List[str],
+        tgt_texts: Optional[List[str]] = None,
+        max_length: Optional[int] = None,
+        max_target_length: Optional[int] = None,
+        padding: str = "longest",
+        return_tensors: str = "None",
+        **kwargs,
+    ) -> BatchEncoding:
+        """Prepare a batch that can be passed directly to an instance of BartModel.
+        Args:
+            src_texts (:obj:`List[str]`):
+                list of src texts
+            tgt_texts (:obj:`List[str]`, `optional`):
+                list of tgt texts
+            max_length (:obj:`int`, `optional`):
+                maximum length for the source text which defers to the config value of 1024 for facebook/bart*
+            max_target_length (:obj:`int`, `optional`):
+                maximum length for the target text which defers to the config value of 1024 for facebook/bart*
+            padding (:obj:`str`, `optional`, defaults to "longest"):
+                strategy for padding `input_ids` and `decoder_input_ids`. Should be "max_length" or "longest".
+            return_tensors (:obj:`str`, `optional`):
+                Can be set to ‘tf’, ‘pt’ or ‘np’ to return respectively TensorFlow `tf.constant`, PyTorch `torch.Tensor` or Numpy :oj: np.ndarray instead of a list of python integers.
+            **kwargs:
+                passed to self.__call__
+        Returns:
+            :class:`~transformers.BatchEncoding`: with keys input_ids, attention_mask, decoder_input_ids, decoder_attention_mask.
+        """
+        if max_length is None:
+            max_length = self.model_max_length
+        model_inputs: BatchEncoding = self(
+            src_texts,
+            add_special_tokens=True,
+            return_tensors=return_tensors,
+            max_length=max_length,
+            padding=padding,
+            truncation=True,
+            **kwargs,
+        )
+        if tgt_texts is None:
+            return model_inputs
+        # Process tgt_texts
+        if max_target_length is None:
+            max_target_length = max_length
+        decoder_inputs: BatchEncoding = self(
+            tgt_texts,
+            add_special_tokens=True,
+            return_tensors=return_tensors,
+            padding=padding,
+            max_length=max_target_length,
+            truncation=True,
+            **kwargs,
+        )
+        for k, v in decoder_inputs.items():
+            model_inputs[f"decoder_{k}"] = v
+
+        return model_inputs
+
 
 class BartTokenizerFast(RobertaTokenizerFast):
     # merges and vocab same as Roberta

tests/test_modeling_bart.py

@@ -18,7 +18,8 @@
 
 import timeout_decorator  # noqa
 
-from transformers import is_torch_available
+from transformers import BatchEncoding, is_torch_available
+from transformers.file_utils import cached_property
 from transformers.testing_utils import require_torch, slow, torch_device
 
 from .test_configuration_common import ConfigTester
@@ -415,6 +416,10 @@ def _long_tensor(tok_lst):
 
 @require_torch
 class BartModelIntegrationTests(unittest.TestCase):
+    @cached_property
+    def default_tokenizer(self):
+        return BartTokenizer.from_pretrained("facebook/bart-large")
+
     @slow
     def test_inference_no_head(self):
         model = BartModel.from_pretrained("facebook/bart-large").to(torch_device)
@@ -559,6 +564,76 @@ def test_cnn_summarization_same_as_fairseq(self):
         # TODO(SS): run fairseq again with num_beams=2, min_len=20.
         # TODO(SS): add test case that hits max_length
 
+    def test_prepare_seq2seq_batch(self):
+        tokenizer = self.default_tokenizer
+        src_text = ["A long paragraph for summrization.", "Another paragraph for summrization."]
+        tgt_text = [
+            "Summary of the text.",
+            "Another summary.",
+        ]
+        expected_src_tokens = [0, 250, 251, 17818, 13, 32933, 21645, 1258, 4, 2]
+        batch = tokenizer.prepare_seq2seq_batch(
+            src_text, tgt_texts=tgt_text, max_length=len(expected_src_tokens), return_tensors="pt"
+        )
+        self.assertIsInstance(batch, BatchEncoding)
+
+        self.assertEqual((2, 10), batch.input_ids.shape)
+        self.assertEqual((2, 10), batch.attention_mask.shape)
+        result = batch.input_ids.tolist()[0]
+        self.assertListEqual(expected_src_tokens, result)
+        # Test that special tokens are reset
+
+    def test_empty_target_text(self):
+        tokenizer = self.default_tokenizer
+        src_text = ["A long paragraph for summrization.", "Another paragraph for summrization."]
+        batch = tokenizer.prepare_seq2seq_batch(src_text, return_tensors="pt")
+        # check if input_ids are returned and no decoder_input_ids
+        self.assertIn("input_ids", batch)
+        self.assertIn("attention_mask", batch)
+        self.assertNotIn("decoder_input_ids", batch)
+        self.assertNotIn("decoder_attention_mask", batch)
+
+    def test_max_target_length(self):
+        tokenizer = self.default_tokenizer
+        src_text = ["A long paragraph for summrization.", "Another paragraph for summrization."]
+        tgt_text = [
+            "Summary of the text.",
+            "Another summary.",
+        ]
+        batch = tokenizer.prepare_seq2seq_batch(
+            src_text, tgt_texts=tgt_text, max_target_length=32, padding="max_length", return_tensors="pt"
+        )
+        self.assertEqual(32, batch["decoder_input_ids"].shape[1])
+        self.assertEqual(32, batch["decoder_attention_mask"].shape[1])
+
+        # test None max_target_length
+        batch = tokenizer.prepare_seq2seq_batch(
+            src_text, tgt_texts=tgt_text, max_length=32, padding="max_length", return_tensors="pt"
+        )
+        self.assertEqual(32, batch["decoder_input_ids"].shape[1])
+        self.assertEqual(32, batch["decoder_attention_mask"].shape[1])
+
+    def test_outputs_not_longer_than_maxlen(self):
+        tokenizer = self.default_tokenizer
+
+        batch = tokenizer.prepare_seq2seq_batch(["I am a small frog" * 1024, "I am a small frog"], return_tensors="pt")
+        self.assertIsInstance(batch, BatchEncoding)
+        self.assertEqual(batch.input_ids.shape, (2, 1024))
+
+    def test_special_tokens(self):
+        tokenizer = self.default_tokenizer
+        src_text = ["A long paragraph for summrization."]
+        tgt_text = [
+            "Summary of the text.",
+        ]
+        batch = tokenizer.prepare_seq2seq_batch(src_text, tgt_texts=tgt_text, return_tensors="pt")
+        input_ids = batch["input_ids"]
+        decoder_input_ids = batch["decoder_input_ids"]
+        self.assertTrue((input_ids[:, 0] == tokenizer.bos_token_id).all().item())
+        self.assertTrue((decoder_input_ids[:, 0] == tokenizer.bos_token_id).all().item())
+        self.assertTrue((input_ids[:, -1] == tokenizer.eos_token_id).all().item())
+        self.assertTrue((decoder_input_ids[:, -1] == tokenizer.eos_token_id).all().item())
+
 
 @require_torch
 class TestSinusoidalPositionalEmbeddings(unittest.TestCase):