[CHORE] Unpin tokenizers. (#3322)

Let the version resolution pick the right version.

chromadb/test/client/test_http_client_v1_compatability.py

@@ -42,15 +42,7 @@ def test_http_client_bw_compatibility() -> None:
 
     old_version = "0.5.11"  # Module with known v1 client
 
-    # Version <3.9 requires bounding tokenizers<=0.20.3
-    # TOOD(hammadb): This code is duplicated in test_cross_version_persist.py
-    # for expediency on 11/27/2024 I am copy pasting rather than refactoring
-    # to DRY. Refactor later.
-    (major, minor, _) = pysys.version_info[:3]
-    if major == 3 and minor < 9:
-        install_version(old_version, {"tokenizers": "<=0.20.3"})
-    else:
-        install_version(old_version, {})
+    install_version(old_version, {})
 
     ctx = multiprocessing.get_context("spawn")
     conn1, conn2 = multiprocessing.Pipe()

chromadb/test/property/test_cross_version_persist.py

@@ -150,13 +150,7 @@ def version_settings(request) -> Generator[Tuple[str, Settings], None, None]:
     configuration = request.param
     version = configuration[0]
 
-    # Version <3.9 requires bounding tokenizers<=0.20.3
-    (major, minor, patch) = sys.version_info[:3]
-    if major == 3 and minor < 9:
-        install_version(version, {"tokenizers": "<=0.20.3"})
-    else:
-        install_version(version, {})
-
+    install_version(version, {})
     yield configuration
     # Cleanup the installed version
     path = get_path_to_version_install(version)

pyproject.toml

@@ -28,7 +28,7 @@ dependencies = [
   'opentelemetry-exporter-otlp-proto-grpc>=1.2.0',
   'opentelemetry-instrumentation-fastapi>=0.41b0',
   'opentelemetry-sdk>=1.2.0',
-  'tokenizers >= 0.13.2, <= 0.20.3',
+  'tokenizers >= 0.13.2',
   'pypika >= 0.48.9',
   'tqdm >= 4.65.0',
   'overrides >= 7.3.1',

requirements.txt

@@ -21,7 +21,7 @@ pypika>=0.48.9
 PyYAML>=6.0.0
 rich>=10.11.0
 tenacity>=8.2.3
-tokenizers>=0.13.2,<=0.20.3
+tokenizers>=0.13.2
 tqdm>=4.65.0
 typer>=0.9.0
 typing_extensions>=4.5.0