[Cherry-Pick] Fix requirements.txt (#9756)

* fix requirments.txt for py38 * update
PaddlePaddle · Jan 8, 2025 · bd9fe73 · bd9fe73
1 parent 9982539
commit bd9fe73
Show file tree

Hide file tree

Showing 3 changed files with 3 additions and 2 deletions.
diff --git a/requirements-dev.txt b/requirements-dev.txt
@@ -29,4 +29,4 @@ data
 wget
 huggingface_hub>=0.19.2
 tiktoken
-tokenizers>=0.21,<0.22
+tokenizers<0.21
diff --git a/requirements.txt b/requirements.txt
@@ -26,4 +26,4 @@ jinja2
 regex
 numpy<=1.26.4
 tiktoken
-tokenizers>=0.21,<0.22
+tokenizers<0.21
diff --git a/scripts/unit_test/ci_unit.sh b/scripts/unit_test/ci_unit.sh
@@ -28,6 +28,7 @@ install_requirements() {
     python -m pip config --user set global.trusted-host pip.baidu-int.com
     python -m pip install -r requirements.txt
     python -m pip install -r requirements-dev.txt
+    python -m pip install tokenizers>=0.21,<0.22 # update tokenizers for transformers
     python -m pip install -r tests/requirements.txt
     python -m pip install -r paddlenlp/experimental/autonlp/requirements.txt 
     python -m pip uninstall paddlepaddle paddlepaddle_gpu -y