Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Allow add_tokens for ESM #28535

Merged
merged 5 commits into from
Jan 19, 2024
Merged

Allow add_tokens for ESM #28535

Changes from 1 commit
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
8d4cfeb
Allow non-special tokens to be added
Rocketknight1 Jan 16, 2024
6acba9b
Add test, fix token adding code
Rocketknight1 Jan 16, 2024
b3fa7e1
Revert changes to id_to_token and token_to_id
Rocketknight1 Jan 17, 2024
7fdb3f2
Update the ESM tokenizer to be a bit more standardized
Rocketknight1 Jan 18, 2024
2dca270
Update src/transformers/models/esm/tokenization_esm.py
Rocketknight1 Jan 19, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 5 additions & 15 deletions src/transformers/models/esm/tokenization_esm.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,9 +14,7 @@
# limitations under the License.
"""Tokenization classes for ESM."""
import os
from typing import List, Optional, Union

from transformers import AddedToken
from typing import List, Optional

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging
Expand Down Expand Up @@ -92,11 +90,10 @@ def _convert_token_to_id(self, token: str) -> int:
def _tokenize(self, text, **kwargs):
return text.split()

def get_vocab_size(self, with_added_tokens=False):
return len(self._id_to_token)

def get_vocab(self):
return {token: i for i, token in enumerate(self.all_tokens)}
base_vocab = {token: i for i, token in enumerate(self.all_tokens)}
Rocketknight1 marked this conversation as resolved.
Show resolved Hide resolved
base_vocab.update(self.added_tokens_encoder)
return base_vocab

def token_to_id(self, token: str) -> int:
return self._token_to_id.get(token, self._token_to_id.get(self.unk_token))
Expand Down Expand Up @@ -157,11 +154,4 @@ def save_vocabulary(self, save_directory, filename_prefix):

@property
def vocab_size(self) -> int:
return self.get_vocab_size(with_added_tokens=False)

def _add_tokens(self, new_tokens: Union[List[str], List[AddedToken]], special_tokens: bool = False) -> int:
result = super()._add_tokens(new_tokens, special_tokens=special_tokens)
for token in new_tokens:
if token and token not in self.all_tokens:
self.all_tokens.append(token)
return result
return len(self.all_tokens)
Loading