tokenizers from fork

Wovchena · May 23, 2024 · 7e23930 · 7e23930
1 parent 1fe85b9
commit 7e23930
Show file tree

Hide file tree

Showing 2 changed files with 3 additions and 3 deletions.
diff --git a/.github/workflows/genai_python_lib.yml b/.github/workflows/genai_python_lib.yml
@@ -20,7 +20,7 @@ jobs:
       # build system doesn't. Install ./requirements-build.txt to detect possible conflicts.
       - run: source ./ov/setupvars.sh && python -m pip install ./thirdparty/openvino_tokenizers/[transformers] -r ./requirements-build.txt
       - run: PYTHONPATH=./src/python/ python -c "from openvino_genai import LLMPipeline"
-      - run: source ./ov/setupvars.sh && CMAKE_BUILD_PARALLEL_LEVEL= python -m pip install --pre . --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly
+      - run: source ./ov/setupvars.sh && CMAKE_BUILD_PARALLEL_LEVEL="" python -m pip install --pre . --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly
       - run: python -c "from openvino_genai import LLMPipeline"
       - name: GenAI Python API tests
         run: |
@@ -54,5 +54,5 @@ jobs:
       # build system doesn't. Install ./requirements-build.txt to detect possible conflicts.
       - run: call w_openvino_toolkit_windows_2024.2.0.dev20240515_x86_64\setupvars.bat && python -m pip install ./thirdparty/openvino_tokenizers/[transformers] -r ./requirements-build.txt
       - run: set "PYTHONPATH=./src/python;" && call w_openvino_toolkit_windows_2024.2.0.dev20240515_x86_64\setupvars.bat && python -c "from openvino_genai import LLMPipeline"  # cmd evaluates variables in a different way. Setting PYTHONPATH before setupvars.bat instead of doing that after solves that.
-      - run: set CMAKE_BUILD_PARALLEL_LEVEL= && call w_openvino_toolkit_windows_2024.2.0.dev20240515_x86_64\setupvars.bat && python -m pip install .
+      - run: set CMAKE_BUILD_PARALLEL_LEVEL="" && call w_openvino_toolkit_windows_2024.2.0.dev20240515_x86_64\setupvars.bat && python -m pip install .
       - run: python -c "from openvino_genai import LLMPipeline"
diff --git a/thirdparty/openvino_tokenizers b/thirdparty/openvino_tokenizers
+2 −2		.github/workflows/linux.yml
+2 −2		.github/workflows/mac.yml
+2 −2		.github/workflows/windows.yml
+1 −1		CMakeLists.txt
+22 −9		README.md
+40 −0		benchmark/README.md
+282 −0		benchmark/benchmark.py
+9 −3		pyproject.toml
+1 −1		python/openvino_tokenizers/cli.py
+75 −23		python/openvino_tokenizers/hf_parser.py
+4 −7		python/openvino_tokenizers/str_pack.py
+5 −4		python/openvino_tokenizers/tiktoken_parser.py
+115 −15		python/openvino_tokenizers/tokenizer_pipeline.py
+72 −39		src/CMakeLists.txt
+55 −46		src/bpe_tokenizer.cpp
+25 −3		src/bpe_tokenizer.hpp
+1 −1		src/case_fold.cpp
+7 −12		src/equal_str.cpp
+1 −1		src/normalize_unicode.cpp
+3 −1		src/ov_extension.cpp
+1 −1		src/regex_normalization.hpp
+71 −15		src/regex_split.cpp
+10 −1		src/regex_split.hpp
+39 −5		src/sentence_piece.cpp
+1 −1		src/string_tensor_pack.cpp
+5 −5		src/string_tensor_unpack.cpp
+218 −0		src/string_to_hash_bucket.cpp
+41 −0		src/string_to_hash_bucket.hpp
+34 −7		src/tensorflow_translators.cpp
+1 −0		src/tensorflow_translators.hpp
+1 −0		src/tokenizer.hpp
+1 −1		src/utils.cpp
+1 −1		tests/pass_rates.json
+18 −10		tests/tokenizer_differential_fuzzing.py
+3 −0		tests/tokenizers_test.py