Update src/transformers/convert_slow_tokenizer.py

huggingface · Jan 18, 2024 · 39e0faf · 39e0faf
1 parent 06260b1
commit 39e0faf
Showing 1 changed file with 0 additions and 1 deletion.
diff --git a/src/transformers/convert_slow_tokenizer.py b/src/transformers/convert_slow_tokenizer.py
@@ -553,7 +553,6 @@ def tokenizer(self, proto):
     def normalizer(self, proto):
         precompiled_charsmap = proto.normalizer_spec.precompiled_charsmap
         _normalizers = [
-            normalizers.Strip(left=False, right=True),
             normalizers.Replace(Regex(" {2,}"), "▁"),
         ]
         if not precompiled_charsmap: