diff --git a/torchtext/transforms.py b/torchtext/transforms.py index 3a91af3963..65c5924b04 100644 --- a/torchtext/transforms.py +++ b/torchtext/transforms.py @@ -365,7 +365,7 @@ def _tokenize(self, text: str) -> List[str]: A list of bpe token ids represents each bpe tokens For example: "awesome,awe" - --> bpe --> bpe tokens: ["aw", "esome"], [","], ["aw", e] + --> bpe --> bpe tokens: ["aw", "esome"], [","], ["aw", "e"] --> bpe encode --> bpe token ids: [707, 5927, 11, 707, 68] """ text = text.lower().strip()