Skip to content

Commit

Permalink
Revert "fix: create a copy for tokenizer object (huggingface#18408)"
Browse files Browse the repository at this point in the history
This reverts commit df5e423.
  • Loading branch information
xa committed Dec 17, 2022
1 parent 7032e02 commit 97bd2c3
Showing 1 changed file with 1 addition and 2 deletions.
3 changes: 1 addition & 2 deletions src/transformers/tokenization_utils_fast.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,6 @@
Tokenization classes for fast tokenizers (provided by HuggingFace's tokenizers library). For slow (python) tokenizers
see tokenization_utils.py
"""
import copy
import json
import os
from collections import defaultdict
Expand Down Expand Up @@ -105,7 +104,7 @@ def __init__(self, *args, **kwargs):
)

if tokenizer_object is not None:
fast_tokenizer = copy.deepcopy(tokenizer_object)
fast_tokenizer = tokenizer_object
elif fast_tokenizer_file is not None and not from_slow:
# We have a serialization from tokenizers which let us directly build the backend
fast_tokenizer = TokenizerFast.from_file(fast_tokenizer_file)
Expand Down

0 comments on commit 97bd2c3

Please sign in to comment.