Bug fix for permutation language modelling (huggingface#8409)

stas00 · Nov 10, 2020 · 23262ab · 23262ab
1 parent e7a23e8
commit 23262ab
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/src/transformers/data/data_collator.py b/src/transformers/data/data_collator.py
@@ -579,7 +579,7 @@ def mask_tokens(self, inputs: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor,
             masked_indices.masked_fill_(padding_mask, value=0.0)
 
         # Mask indicating non-functional tokens, where functional tokens are [SEP], [CLS], padding, etc.
-        non_func_mask = ~(padding_mask & special_tokens_mask)
+        non_func_mask = ~(padding_mask | special_tokens_mask)
 
         inputs[masked_indices] = self.tokenizer.mask_token_id
         labels[~masked_indices] = -100  # We only compute loss on masked tokens