move 2 convert fnc. to PreTrainedSentencepieceTokenizer

src/transformers/models/albert/tokenization_albert.py

@@ -197,14 +197,6 @@ def _tokenize(self, text: str) -> List[str]:
 
         return new_pieces
 
-    def _convert_token_to_id(self, token):
-        """Converts a token (str) in an id using the vocab."""
-        return self.sp_model.PieceToId(token)
-
-    def _convert_id_to_token(self, index):
-        """Converts an index (integer) in a token (str) using the vocab."""
-        return self.sp_model.IdToPiece(index)
-
     def build_inputs_with_special_tokens(
         self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None
     ) -> List[int]:

src/transformers/models/bert_generation/tokenization_bert_generation.py

@@ -100,15 +100,6 @@ def __init__(
             **kwargs,
         )
 
-    def _convert_token_to_id(self, token):
-        """Converts a token (str) in an id using the vocab."""
-        return self.sp_model.piece_to_id(token)
-
-    def _convert_id_to_token(self, index):
-        """Converts an index (integer) in a token (str) using the vocab."""
-        token = self.sp_model.IdToPiece(index)
-        return token
-
     def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> Tuple[str]:
         if not os.path.isdir(save_directory):
             logger.error(f"Vocabulary path ({save_directory}) should be a directory")

src/transformers/models/big_bird/tokenization_big_bird.py

@@ -130,15 +130,6 @@ def __init__(
             **kwargs,
         )
 
-    def _convert_token_to_id(self, token):
-        """Converts a token (str) in an id using the vocab."""
-        return self.sp_model.piece_to_id(token)
-
-    def _convert_id_to_token(self, index):
-        """Converts an index (integer) in a token (str) using the vocab."""
-        token = self.sp_model.IdToPiece(index)
-        return token
-
     def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> Tuple[str]:
         if not os.path.isdir(save_directory):
             logger.error(f"Vocabulary path ({save_directory}) should be a directory")

src/transformers/models/reformer/tokenization_reformer.py

@@ -103,10 +103,6 @@ def __init__(
             **kwargs,
         )
 
-    def _convert_token_to_id(self, token):
-        """Converts a token (str) in an id using the vocab."""
-        return self.sp_model.piece_to_id(token)
-
     def _convert_id_to_token(self, index):
         """Converts an index (integer) in a token (str) using the vocab."""
         if index < self.sp_model.get_piece_size():

src/transformers/models/xlnet/tokenization_xlnet.py

@@ -200,14 +200,6 @@ def _tokenize(self, text: str) -> List[str]:
 
         return new_pieces
 
-    def _convert_token_to_id(self, token):
-        """Converts a token (str) in an id using the vocab."""
-        return self.sp_model.PieceToId(token)
-
-    def _convert_id_to_token(self, index):
-        """Converts an index (integer) in a token (str) using the vocab."""
-        return self.sp_model.IdToPiece(index)
-
     def build_inputs_with_special_tokens(
         self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None
     ) -> List[int]:

src/transformers/tokenization_utils.py

@@ -799,6 +799,14 @@ def get_vocab(self):
         vocab.update(self.added_tokens_encoder)
         return vocab
 
+    def _convert_token_to_id(self, token):
+        """Converts a token (str) in an id using the vocab."""
+        return self.sp_model.PieceToId(token)
+
+    def _convert_id_to_token(self, index):
+        """Converts an index (integer) in a token (str) using the vocab."""
+        return self.sp_model.IdToPiece(index)
+
     def __getstate__(self):
         state = self.__dict__.copy()
         state["sp_model"] = None