Update deprecated/unused dependencies 🧹 🧹

setup.py

@@ -69,33 +69,14 @@
 
 import os
 import re
-import shutil
-from pathlib import Path
 
 from setuptools import Command, find_packages, setup
 
 
-# Remove stale transformers.egg-info directory to avoid https://github.com/pypa/pip/issues/5466
-stale_egg_info = Path(__file__).parent / "transformers.egg-info"
-if stale_egg_info.exists():
-    print(
-        (
-            "Warning: {} exists.\n\n"
-            "If you recently updated transformers to 3.0 or later, this is expected,\n"
-            "but it may prevent transformers from installing in editable mode.\n\n"
-            "This directory is automatically generated by Python's packaging tools.\n"
-            "I will remove it now.\n\n"
-            "See https://github.com/pypa/pip/issues/5466 for details.\n"
-        ).format(stale_egg_info)
-    )
-    shutil.rmtree(stale_egg_info)
-
-
 # IMPORTANT:
 # 1. all dependencies should be listed here with their version requirements if any
 # 2. once modified, run: `make deps_table_update` to update src/transformers/dependency_versions_table.py
 _deps = [
-    "Pillow>=10.0.1,<=15.0",
     "accelerate>=0.26.0",
     "av",
     "beautifulsoup4",
@@ -144,6 +125,7 @@
     "packaging>=20.0",
     "parameterized",
     "phonemizer",
+    "Pillow>=10.0.1,<=15.0",
     "protobuf",
     "psutil",
     "pyyaml>=5.1",
@@ -345,6 +327,7 @@ def run(self):
     )
     + extras["retrieval"]
     + extras["modelcreation"]
+    + extras["tiktoken"]
 )
 
 extras["deepspeed-testing"] = extras["deepspeed"] + extras["testing"] + extras["optuna"] + extras["sentencepiece"]

src/transformers/dependency_versions_table.py

@@ -2,7 +2,6 @@
 # 1. modify the `_deps` dict in setup.py
 # 2. run `make deps_table_update``
 deps = {
-    "Pillow": "Pillow>=10.0.1,<=15.0",
     "accelerate": "accelerate>=0.26.0",
     "av": "av",
     "beautifulsoup4": "beautifulsoup4",
@@ -50,6 +49,7 @@
     "packaging": "packaging>=20.0",
     "parameterized": "parameterized",
     "phonemizer": "phonemizer",
+    "Pillow": "Pillow>=10.0.1,<=15.0",
     "protobuf": "protobuf",
     "psutil": "psutil",
     "pyyaml": "pyyaml>=5.1",

tests/models/llama/test_tokenization_llama.py

@@ -839,13 +839,13 @@ def test_special_tokens_strip(self):
         self.assertEqual(tokens, ["▁No", "<s>", "▁He"])  # spaces are eaten by rstrip / lstrip
 
 
-@require_tiktoken
-@require_read_token
 class TikTokenIntegrationTests(unittest.TestCase):
     """
     A class that regroups important test to make sure that we properly handle the special tokens.
     """
 
+    @require_tiktoken
+    @require_read_token
     def test_tiktoken_llama(self):
         model_path = "hf-internal-testing/llama-3-8b-internal"
         subfolder = "original"