fix(tokenizer): 修复TokenTokenizer无法解决空格的问题

Closes #37
xiangking · Mar 26, 2022 · 4956c88 · 4956c88
1 parent 72391f8
commit 4956c88
Showing 1 changed file with 7 additions and 2 deletions.
diff --git a/ark_nlp/processor/tokenizer/transfomer.py b/ark_nlp/processor/tokenizer/transfomer.py
@@ -229,8 +229,13 @@ class TokenTokenizer(TransfomerTokenizer):
 
     def tokenize(self, text, **kwargs):
         tokens = []
-        text = ' '.join([token_ for token_ in text])
-        tokens = self.vocab.tokenize(text)
+        for token_ in text:
+            tokenized_token_ = self.vocab.tokenize(token_)
+            if tokenized_token_ == []:
+                tokens.extend([token_])
+            else:
+                tokens.extend(tokenized_token_)
+
         return tokens
 
     def sequence_to_ids(self, sequence, **kwargs):