Revert "Fix + Test (huggingface#8049)"

This reverts commit 2ad8c38.
fabiocapsouza · Nov 15, 2020 · d12a3e4 · d12a3e4
1 parent 670bd0a
commit d12a3e4
Show file tree

Hide file tree

Showing 2 changed files with 0 additions and 12 deletions.
diff --git a/src/transformers/tokenization_blenderbot.py b/src/transformers/tokenization_blenderbot.py
@@ -166,9 +166,6 @@ def bpe(self, token: str) -> str:
         tokens = token.split(" ")
         words = []
         for token in tokens:
-            if not len(token):
-                continue
-
             token = token.lower()
             word = tuple(token)
             word = tuple(list(word[:-1]) + [word[-1] + "</w>"])

diff --git a/tests/test_tokenization_blenderbot.py b/tests/test_tokenization_blenderbot.py
@@ -75,15 +75,6 @@ def test_special_tokens_small_tok(self):
         assert src_text != decoded  # I wish it did!
         assert decoded == "i am a small frog ."
 
-    def test_empty_word_small_tok(self):
-        tok = BlenderbotSmallTokenizer.from_pretrained("facebook/blenderbot-90M")
-        src_text = "I am a small frog ."
-        src_text_dot = "."
-        encoded = tok(src_text)["input_ids"]
-        encoded_dot = tok(src_text_dot)["input_ids"]
-
-        assert encoded[-1] == encoded_dot[0]
-
 
 class Blenderbot3BTokenizerTests(unittest.TestCase):
     @cached_property