source

Wovchena · May 24, 2024 · 8bcf504 · 8bcf504
1 parent f4e444f
commit 8bcf504
Show file tree

Hide file tree

Showing 2 changed files with 5 additions and 5 deletions.
diff --git a/.github/workflows/genai_package.yml b/.github/workflows/genai_package.yml
@@ -22,9 +22,9 @@ jobs:
       - run: source ./ov/setupvars.sh && cmake --install ./build/ --config ${{ matrix.build-type }} --prefix ov
       - run: ov/samples/cpp/build_samples.sh -i ${{ github.workspace }}/s\ pace
         if: ${{ 'Release' == matrix.build-type }}  # build_samples enforces Release build
-      - run: python -m pip install --upgrade-strategy eager -r ./text_generation/causal_lm/cpp/requirements.txt
+      - run: source ./ov/setupvars.sh && python -m pip install --upgrade-strategy eager -r ./text_generation/causal_lm/cpp/requirements.txt
         if: ${{ 'Release' == matrix.build-type }}
-      - run: python -m pip install ./thirdparty/openvino_tokenizers/[transformers] --pre --extra-index-url https://storage.openvinotoolkit.org/
+      - run: source ./ov/setupvars.sh && python -m pip install ./thirdparty/openvino_tokenizers/[transformers] --pre --extra-index-url https://storage.openvinotoolkit.org/
         if: ${{ 'Release' == matrix.build-type }}
       - run: source ./ov/setupvars.sh && optimum-cli export openvino --trust-remote-code --weight-format fp16 --model TinyLlama/TinyLlama-1.1B-Chat-v1.0 TinyLlama-1.1B-Chat-v1.0
         if: ${{ 'Release' == matrix.build-type }}
@@ -53,9 +53,9 @@ jobs:
       - run: call w_openvino_toolkit_windows_2024.2.0.dev20240515_x86_64\setupvars.bat && cmake --install ./build/ --config ${{ matrix.build-type }} --prefix w_openvino_toolkit_windows_2024.2.0.dev20240515_x86_64
       - run: call w_openvino_toolkit_windows_2024.2.0.dev20240515_x86_64\samples\cpp\build_samples_msvc.bat -i "${{ github.workspace }}/samples_install"
         if: ${{ 'Release' == matrix.build-type }}  # build_samples enforces Release build
-      - run: python -m pip install --upgrade-strategy eager -r ./text_generation/causal_lm/cpp/requirements.txt
+      - run: call w_openvino_toolkit_windows_2024.2.0.dev20240515_x86_64\setupvars.bat && python -m pip install --upgrade-strategy eager -r ./text_generation/causal_lm/cpp/requirements.txt
         if: ${{ 'Release' == matrix.build-type }}
-      - run: python -m pip install ./thirdparty/openvino_tokenizers/[transformers] --pre --extra-index-url https://storage.openvinotoolkit.org/
+      - run: call w_openvino_toolkit_windows_2024.2.0.dev20240515_x86_64\setupvars.bat && python -m pip install ./thirdparty/openvino_tokenizers/[transformers] --pre --extra-index-url https://storage.openvinotoolkit.org/
         if: ${{ 'Release' == matrix.build-type }}
       - run: call w_openvino_toolkit_windows_2024.2.0.dev20240515_x86_64\setupvars.bat && optimum-cli export openvino --trust-remote-code --weight-format fp16 --model TinyLlama/TinyLlama-1.1B-Chat-v1.0 TinyLlama-1.1B-Chat-v1.0
         if: ${{ 'Release' == matrix.build-type }}

diff --git a/thirdparty/openvino_tokenizers b/thirdparty/openvino_tokenizers
+22 −9		README.md
+72 −22		python/openvino_tokenizers/hf_parser.py
+4 −7		python/openvino_tokenizers/str_pack.py
+5 −4		python/openvino_tokenizers/tiktoken_parser.py
+115 −15		python/openvino_tokenizers/tokenizer_pipeline.py
+55 −46		src/bpe_tokenizer.cpp
+25 −3		src/bpe_tokenizer.hpp
+70 −14		src/regex_split.cpp
+10 −1		src/regex_split.hpp
+1 −1		tests/pass_rates.json
+3 −0		tests/tokenizers_test.py