From 9a65f9f71991cbdff36018c4859ee69021cb31b9 Mon Sep 17 00:00:00 2001 From: Arthur Zucker Date: Tue, 14 May 2024 15:02:01 +0200 Subject: [PATCH] test --- src/transformers/tokenization_utils_base.py | 11 ++++++----- src/transformers/tokenization_utils_fast.py | 4 ++-- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/src/transformers/tokenization_utils_base.py b/src/transformers/tokenization_utils_base.py index 2c16bc53e084..1775d8d2c695 100644 --- a/src/transformers/tokenization_utils_base.py +++ b/src/transformers/tokenization_utils_base.py @@ -2905,7 +2905,7 @@ def _call_one( return_offsets_mapping: bool = False, return_length: bool = False, verbose: bool = True, - split_special_tokens:bool = False, + split_special_tokens: bool = False, **kwargs, ) -> BatchEncoding: # Input type checking for clearer error @@ -3094,7 +3094,7 @@ def _encode_plus( return_offsets_mapping: bool = False, return_length: bool = False, verbose: bool = True, - split_special_tokens:bool = False, + split_special_tokens: bool = False, **kwargs, ) -> BatchEncoding: raise NotImplementedError @@ -3125,7 +3125,7 @@ def batch_encode_plus( return_offsets_mapping: bool = False, return_length: bool = False, verbose: bool = True, - split_special_tokens:bool = False, + split_special_tokens: bool = False, **kwargs, ) -> BatchEncoding: """ @@ -3170,7 +3170,8 @@ def batch_encode_plus( return_special_tokens_mask=return_special_tokens_mask, return_offsets_mapping=return_offsets_mapping, return_length=return_length, - verbose=verbose,split_special_tokens=split_special_tokens, + verbose=verbose, + split_special_tokens=split_special_tokens, **kwargs, ) @@ -3199,7 +3200,7 @@ def _batch_encode_plus( return_offsets_mapping: bool = False, return_length: bool = False, verbose: bool = True, - split_special_tokens:bool = False, + split_special_tokens: bool = False, **kwargs, ) -> BatchEncoding: raise NotImplementedError diff --git a/src/transformers/tokenization_utils_fast.py b/src/transformers/tokenization_utils_fast.py index f0c8ccd097c2..d55cd9809910 100644 --- a/src/transformers/tokenization_utils_fast.py +++ b/src/transformers/tokenization_utils_fast.py @@ -486,7 +486,7 @@ def _batch_encode_plus( return_offsets_mapping: bool = False, return_length: bool = False, verbose: bool = True, - split_special_tokens:bool = False, + split_special_tokens: bool = False, ) -> BatchEncoding: if not isinstance(batch_text_or_text_pairs, (tuple, list)): raise TypeError( @@ -574,7 +574,7 @@ def _encode_plus( return_offsets_mapping: bool = False, return_length: bool = False, verbose: bool = True, - split_special_tokens:bool = False, + split_special_tokens: bool = False, **kwargs, ) -> BatchEncoding: batched_input = [(text, text_pair)] if text_pair else [text]