Merge remote-tracking branch 'ilavrenov_upstream/ct-beam-search' into…

… penalties

.github/dependabot.yml

@@ -1,5 +1,9 @@
 version: 2
 updates:
+  - package-ecosystem: "pip"
+    directory: "./"
+    schedule:
+      interval: "weekly"
   - package-ecosystem: "pip"
     directory: "image_generation/stable_diffusion_1_5/cpp/scripts/"
     schedule:
@@ -8,6 +12,10 @@ updates:
     directory: "image_generation/lcm_dreamshaper_v7/cpp/scripts/"
     schedule:
       interval: "weekly"
+  - package-ecosystem: "pip"
+    directory: "./tests/python_tests/"
+    schedule:
+      interval: "weekly"
   - package-ecosystem: "pip"
     directory: "text_generation/causal_lm/cpp/"
     schedule:

.github/workflows/genai_package.yml

@@ -0,0 +1,98 @@
+name: genai_package
+on: pull_request
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref || github.ref_name }}
+  cancel-in-progress: true
+jobs:
+  ubuntu_genai_package:
+    strategy:
+      matrix:
+        build-type: [Release, Debug]
+    runs-on: ubuntu-20.04
+    env:
+      CMAKE_BUILD_PARALLEL_LEVEL: null
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: recursive
+      - uses: actions/setup-python@v4
+        with:
+          python-version: 3.8
+      - run: mkdir ./ov/
+      - run: curl https://storage.openvinotoolkit.org/repositories/openvino/packages/pre-release/2024.2.0rc1/linux/l_openvino_toolkit_ubuntu20_2024.2.0.dev20240524_x86_64.tgz | tar --directory ./ov/ --strip-components 1 -xz
+      - run: sudo ./ov/install_dependencies/install_openvino_dependencies.sh
+      - run: source ./ov/setupvars.sh && cmake -DCMAKE_BUILD_TYPE=${{ matrix.build-type }} -S ./ -B ./build/
+      - run: source ./ov/setupvars.sh && cmake --build ./build/ --config ${{ matrix.build-type }} --target package -j
+      - run: source ./ov/setupvars.sh && cmake --install ./build/ --config ${{ matrix.build-type }} --prefix ov
+      - run: ov/samples/cpp/build_samples.sh -i ${{ github.workspace }}/s\ pace
+        if: ${{ 'Release' == matrix.build-type }}  # build_samples enforces Release build
+      - run: source ./ov/setupvars.sh && cmake -DCMAKE_BUILD_TYPE=${{ matrix.build-type }} -S ./ov/samples/cpp/ -B ./samples\ build/ && cmake --build ./samples\ build/ --config ${{ matrix.build-type }} -j && cmake --install ./samples\ build/ --config ${{ matrix.build-type }} --component samples_bin --prefix s\ pace
+        if: ${{ 'Release' != matrix.build-type }}
+      - run: source ./ov/setupvars.sh && python -m pip install ./thirdparty/openvino_tokenizers/[transformers] --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/pre-release
+      - run: source ./ov/setupvars.sh && python -m pip install --upgrade-strategy eager -r ./samples/cpp/requirements.txt
+      - run: source ./ov/setupvars.sh && optimum-cli export openvino --trust-remote-code --weight-format fp16 --model TinyLlama/TinyLlama-1.1B-Chat-v1.0 TinyLlama-1.1B-Chat-v1.0
+      - run: source ./ov/setupvars.sh && timeout 50s ${{ github.workspace }}/s\ pace/samples_bin/greedy_causal_lm ./TinyLlama-1.1B-Chat-v1.0/ ""
+
+  macos_genai_package:
+    strategy:
+      matrix:
+        build-type: [Release, Debug]
+    runs-on: macos-12
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: recursive
+      - uses: actions/setup-python@v4
+        with:
+          python-version: 3.8
+      - run: mkdir ./ov/
+      - run: curl https://storage.openvinotoolkit.org/repositories/openvino/packages/pre-release/2024.2.0rc2/macos/m_openvino_toolkit_macos_12_6_2024.2.0.dev20240529_x86_64.tgz | tar --directory ./ov/ --strip-components 1 -xz
+      - run: brew install coreutils scons
+      - run: source ./ov/setupvars.sh && cmake -DCMAKE_BUILD_TYPE=${{ matrix.build-type }} -S ./ -B ./build/
+      - run: source ./ov/setupvars.sh && cmake --build ./build/ --config ${{ matrix.build-type }} --target package -j
+      - run: source ./ov/setupvars.sh && cmake --install ./build/ --config ${{ matrix.build-type }} --prefix ov
+      - run: ov/samples/cpp/build_samples.sh -i ${{ github.workspace }}/s\ pace
+        if: ${{ 'Release' == matrix.build-type }}  # build_samples enforces Release build
+      - run: source ./ov/setupvars.sh && python -m pip install ./thirdparty/openvino_tokenizers/[transformers] --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/pre-release
+        if: ${{ 'Release' == matrix.build-type }}
+      - run: source ./ov/setupvars.sh && python -m pip install --upgrade-strategy eager -r ./samples/cpp/requirements.txt
+        if: ${{ 'Release' == matrix.build-type }}
+      - run: source ./ov/setupvars.sh && optimum-cli export openvino --trust-remote-code --weight-format fp16 --model TinyLlama/TinyLlama-1.1B-Chat-v1.0 TinyLlama-1.1B-Chat-v1.0
+        if: ${{ 'Release' == matrix.build-type }}
+      - run: source ./ov/setupvars.sh && timeout 50s ${{ github.workspace }}/s\ pace/samples_bin/greedy_causal_lm ./TinyLlama-1.1B-Chat-v1.0/ ""
+        if: ${{ 'Release' == matrix.build-type }}
+
+  windows_genai_package:
+    strategy:
+      matrix:
+        build-type: [Release, Debug]
+    runs-on: windows-latest
+    env:
+      CMAKE_BUILD_PARALLEL_LEVEL: null
+    defaults:
+      run:
+        shell: cmd
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: recursive
+      - uses: actions/setup-python@v4
+        with:
+          python-version: 3.8
+      - run: curl --output ov.zip https://storage.openvinotoolkit.org/repositories/openvino/packages/pre-release/2024.2.0rc1/windows/w_openvino_toolkit_windows_2024.2.0.dev20240524_x86_64.zip
+      - run: unzip ov.zip
+      # Shorten the next setupvars calls.
+      - run: mklink /D ov w_openvino_toolkit_windows_2024.2.0.dev20240524_x86_64
+      - run: call ov\setupvars.bat && cmake -DCMAKE_BUILD_TYPE=${{ matrix.build-type }} -S ./ -B ./build/
+      - run: call ov\setupvars.bat && cmake --build ./build/ --config ${{ matrix.build-type }} --target package -j
+      - run: call ov\setupvars.bat && cmake --install ./build/ --config ${{ matrix.build-type }} --prefix ov
+      - run: call ov\samples\cpp\build_samples_msvc.bat -i "${{ github.workspace }}/samples_install"
+        if: ${{ false && 'Release' == matrix.build-type }}  # build_samples enforces Release build
+      - run: call ov\setupvars.bat && python -m pip install ./thirdparty/openvino_tokenizers/[transformers] --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/pre-release
+        if: ${{ false && 'Release' == matrix.build-type }}
+      - run: call ov\setupvars.bat && python -m pip install --upgrade-strategy eager -r ./samples/cpp/requirements.txt
+        if: ${{ false && 'Release' == matrix.build-type }}
+      - run: call ov\setupvars.bat && optimum-cli export openvino --trust-remote-code --weight-format fp16 --model TinyLlama/TinyLlama-1.1B-Chat-v1.0 TinyLlama-1.1B-Chat-v1.0
+        if: ${{ false && 'Release' == matrix.build-type }}
+      - run: call ov\setupvars.bat && "${{ github.workspace }}/samples_install/samples_bin/greedy_causal_lm" .\TinyLlama-1.1B-Chat-v1.0\ ""
+        if: ${{ false && 'Release' == matrix.build-type }}

.github/workflows/genai_python_lib.yml

@@ -0,0 +1,85 @@
+name: genai_python_lib
+on: pull_request
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref || github.ref_name }}
+  cancel-in-progress: true
+jobs:
+  ubuntu_genai_python_lib:
+    # A tokenizers' dependency fails to compile on ubuntu-20 n CenOS7 env.
+    runs-on: ubuntu-22.04
+    env:
+      # A tokenizers' dependency fails to compile with Ninja in CenOS7 env.
+      CMAKE_GENERATOR: Unix Makefiles
+      CMAKE_BUILD_PARALLEL_LEVEL: null
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: recursive
+      - uses: actions/setup-python@v4
+        with:
+          python-version: 3.8
+      - run: mkdir ./ov/
+      # Install CentOS7 instead of Ubuntu to match PyPI distribution ABI.
+      - run: curl https://storage.openvinotoolkit.org/repositories/openvino/packages/pre-release/2024.2.0rc1/linux/l_openvino_toolkit_centos7_2024.2.0.dev20240524_x86_64.tgz | tar --directory ./ov/ --strip-components 1 -xz
+      - run: sudo ./ov/install_dependencies/install_openvino_dependencies.sh
+      - run: source ./ov/setupvars.sh && cmake -DCMAKE_BUILD_TYPE=Release -S ./ -B ./build/
+      - run: source ./ov/setupvars.sh && cmake --build ./build/ --config Release -j
+      # GitHub Actions already provides what is listed in ./requirements-build.txt but the internal
+      # build system doesn't. Install ./requirements-build.txt to detect possible conflicts.
+      - run: source ./ov/setupvars.sh && python -m pip install ./thirdparty/openvino_tokenizers/[transformers] -r ./requirements-build.txt -r ./tests/python_tests/requirements.txt --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/pre-release --upgrade-strategy eager
+      - run: source ./ov/setupvars.sh && PYTHONPATH=./build/:$PYTHONPATH python -m pytest ./tests/python_tests/test_generate_api.py -m precommit
+      - run: source ./ov/setupvars.sh && python -m pip install . --config-settings=build-dir="build" --verbose
+      - run: python -m pytest ./tests/python_tests/test_generate_api.py -m precommit
+
+  macos_genai_python_lib:
+    runs-on: macos-12
+    env:
+      # A tokenizers' dependency fails to compile with Ninja.
+      CMAKE_GENERATOR: Unix Makefiles
+      CMAKE_BUILD_PARALLEL_LEVEL: null
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: recursive
+      - uses: actions/setup-python@v4
+        with:
+          python-version: 3.8
+      - run: mkdir ./ov/
+      - run: curl https://storage.openvinotoolkit.org/repositories/openvino/packages/pre-release/2024.2.0rc2/macos/m_openvino_toolkit_macos_12_6_2024.2.0.dev20240529_x86_64.tgz | tar --directory ./ov/ --strip-components 1 -xz
+      - run: brew install coreutils scons
+      - run: source ./ov/setupvars.sh && cmake -DCMAKE_BUILD_TYPE=Release -S ./ -B ./build/
+      - run: source ./ov/setupvars.sh && cmake --build ./build/ --config Release -j
+      # GitHub Actions already provides what is listed in ./requirements-build.txt but the internal
+      # build system doesn't. Install ./requirements-build.txt to detect possible conflicts.
+      - run: source ./ov/setupvars.sh && python -m pip install ./thirdparty/openvino_tokenizers/[transformers] -r ./requirements-build.txt -r ./tests/python_tests/requirements.txt --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/pre-release --upgrade-strategy eager
+      - run: source ./ov/setupvars.sh && PYTHONPATH=./build/:$PYTHONPATH python -m pytest ./tests/python_tests/test_generate_api.py -m precommit
+      - run: source ./ov/setupvars.sh && python -m pip install . --config-settings=build-dir="build" --verbose
+      - run: python -c "from openvino_genai import LLMPipeline"
+      - run: python -m pytest ./tests/python_tests/test_generate_api.py -m precommit
+
+  windows_genai_python_lib:
+    if: false
+    runs-on: windows-latest
+    env:
+      CMAKE_BUILD_PARALLEL_LEVEL: null
+    defaults:
+      run:
+        shell: cmd
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: recursive
+      - uses: actions/setup-python@v4
+        with:
+          python-version: 3.8
+      - run: curl --output ov.zip https://storage.openvinotoolkit.org/repositories/openvino/packages/pre-release/2024.2.0rc1/windows/w_openvino_toolkit_windows_2024.2.0.dev20240524_x86_64.zip
+      - run: unzip ov.zip
+      # Shorten the next setupvars calls.
+      - run: mklink /D ov w_openvino_toolkit_windows_2024.2.0.dev20240524_x86_64
+      - run: call ./ov/setupvars.bat && cmake -DCMAKE_BUILD_TYPE=Release -S ./ -B ./build/
+      - run: call ./ov/setupvars.bat && cmake --build ./build/ --config Release -j
+      - run: call ./ov/setupvars.bat && python -m pip install ./thirdparty/openvino_tokenizers/[transformers] -r ./requirements-build.txt -r ./tests/python_tests/requirements.txt --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/pre-release --upgrade-strategy eager
+      # cmd evaluates variables in a different way. Setting PYTHONPATH before setupvars.bat instead of doing that after solves that.
+      - run: set "PYTHONPATH=./build/" && call ./ov/setupvars.bat && python -m pytest ./tests/python_tests/test_generate_api.py -m precommit
+      - run: call ./ov/setupvars.bat && python -m pip install . --config-settings=build-dir="build" --verbose
+      - run: python -m pytest ./tests/python_tests/test_generate_api.py -m precommit

.github/workflows/lcm_dreamshaper_cpp.yml

@@ -40,15 +40,15 @@ jobs:
         run: |
           conda activate openvino_lcm_cpp
           conda update -c conda-forge --all
-          conda install -c conda-forge openvino=2024.1.0 c-compiler cxx-compiler git make cmake
+          conda install -c conda-forge -c conda-forge/label/openvino_dev openvino==2024.2.0.dev20240513 c-compiler cxx-compiler git make cmake
           conda env config vars set LD_LIBRARY_PATH=$CONDA_PREFIX/lib:$LD_LIBRARY_PATH
       
       - name: Install python dependencies
         working-directory: ${{ env.working_directory }}
         run: |
           conda activate openvino_lcm_cpp
-          python -m pip install -r requirements.txt
           python -m pip install ../../../thirdparty/openvino_tokenizers/[transformers]
+          python -m pip install -r requirements.txt
           
       - name: Download and convert model and tokenizer
         working-directory: ${{ env.working_directory }}
@@ -85,15 +85,15 @@ jobs:
         run: |
           conda activate openvino_lcm_cpp
           conda update -c conda-forge --all
-          conda install -c conda-forge openvino=2024.1.0 c-compiler cxx-compiler git make cmake
+          conda install -c conda-forge -c conda-forge/label/openvino_dev openvino==2024.2.0.dev20240513 c-compiler cxx-compiler git make cmake
           conda env config vars set LD_LIBRARY_PATH=$CONDA_PREFIX/lib:$LD_LIBRARY_PATH
       
       - name: Install python dependencies
         working-directory: ${{ env.working_directory }}
         run: |
           conda activate openvino_lcm_cpp
-          python -m pip install -r requirements.txt
           python -m pip install ../../../thirdparty/openvino_tokenizers/[transformers]
+          python -m pip install -r requirements.txt
 
       - name: Download and convert model and tokenizer
         working-directory: ${{ env.working_directory }}

.github/workflows/llm_bench-python.yml

@@ -39,7 +39,7 @@ jobs:
       run: |
         python -m pip install --upgrade pip
         python -m pip install flake8 pytest black
-        pip install -r ${{ env.LLM_BENCH_PYPATH }}/requirements.txt
+        GIT_CLONE_PROTECTION_ACTIVE=false pip install -r ${{ env.LLM_BENCH_PYPATH }}/requirements.txt
         pip install openvino-nightly
 
     - name: Lint with flake8
@@ -73,7 +73,7 @@ jobs:
           python-version: 3.8
       - name: Test stateful
         run: |
-          python -m pip install -r llm_bench/python/requirements.txt
+          GIT_CLONE_PROTECTION_ACTIVE=false python -m pip install -r llm_bench/python/requirements.txt
           python -m pip uninstall --yes openvino
           python -m pip install openvino-nightly
           python llm_bench/python/convert.py --model_id TinyLlama/TinyLlama-1.1B-Chat-v1.0 --output_dir . --stateful

.github/workflows/stable_diffusion_1_5_cpp.yml

@@ -39,15 +39,15 @@ jobs:
       - name: Install OpenVINO and other conda dependencies
         run: |
           conda activate openvino_sd_cpp
-          conda install -c conda-forge openvino=2024.1.0 c-compiler cxx-compiler git make cmake
+          conda install -c conda-forge -c conda-forge/label/openvino_dev openvino==2024.2.0.dev20240513 c-compiler cxx-compiler git make cmake
           conda env config vars set LD_LIBRARY_PATH=$CONDA_PREFIX/lib:$LD_LIBRARY_PATH
 
       - name: Install python dependencies
         working-directory: ${{ env.working_directory }}
         run: |
           conda activate openvino_sd_cpp
-          python -m pip install -r requirements.txt
           python -m pip install ../../../thirdparty/openvino_tokenizers/[transformers]
+          python -m pip install -r requirements.txt
 
       - name: Download and convert model and tokenizer
         working-directory: ${{ env.working_directory }}
@@ -83,14 +83,14 @@ jobs:
         - name: Install OpenVINO and other conda dependencies
           run: |
             conda activate openvino_sd_cpp
-            conda install -c conda-forge openvino=2024.1.0 c-compiler cxx-compiler git make cmake
+            conda install -c conda-forge -c conda-forge/label/openvino_dev openvino==2024.2.0.dev20240513 c-compiler cxx-compiler git make cmake
   
         - name: Install python dependencies
           working-directory: ${{ env.working_directory }}
           run: |
             conda activate openvino_sd_cpp
-            python -m pip install -r requirements.txt
             python -m pip install ../../../thirdparty/openvino_tokenizers/[transformers]
+            python -m pip install -r requirements.txt
   
         - name: Download and convert model and tokenizer
           working-directory: ${{ env.working_directory }}

.gitignore

@@ -39,4 +39,4 @@ CMakeUserPresets.json
 *.?env*
 *.pyc
 .env
-__pycache__
+__pycache__

CMakeLists.txt

@@ -0,0 +1,46 @@
+# Copyright (C) 2018-2024 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+#
+
+cmake_minimum_required(VERSION 3.23.0)  # The requirement comes from Jinja2Cpp
+
+# Multi config generators such as Visual Studio ignore CMAKE_BUILD_TYPE. Multi config generators are configured with
+# CMAKE_CONFIGURATION_TYPES, but limiting options in it completely removes such build options
+get_property(GENERATOR_IS_MULTI_CONFIG_VAR GLOBAL PROPERTY GENERATOR_IS_MULTI_CONFIG)
+if(CMAKE_GENERATOR STREQUAL "Ninja Multi-Config")
+    # 'Ninja Multi-Config' specific, see:
+    # https://cmake.org/cmake/help/latest/variable/CMAKE_DEFAULT_BUILD_TYPE.html
+    set(CMAKE_DEFAULT_BUILD_TYPE "Release" CACHE STRING "CMake default build type")
+elseif(NOT GENERATOR_IS_MULTI_CONFIG_VAR AND NOT DEFINED CMAKE_BUILD_TYPE)
+    message(STATUS "CMAKE_BUILD_TYPE is not defined, 'Release' will be used")
+    # Setting CMAKE_BUILD_TYPE as CACHE must go before project(). Otherwise project() sets its value and set() doesn't take an effect
+    set(CMAKE_BUILD_TYPE Release CACHE STRING "Choose the type of build, options are: None Debug Release RelWithDebInfo MinSizeRel ...")
+endif()
+
+project(OpenVINOGenAI VERSION 2024.2.0.0)
+
+add_subdirectory(./thirdparty/)
+add_subdirectory(src)
+add_subdirectory(samples/cpp/beam_search_causal_lm/)
+add_subdirectory(samples/cpp/chat_sample/)
+add_subdirectory(samples/cpp/greedy_causal_lm/)
+add_subdirectory(samples/cpp/multinomial_causal_lm/)
+add_subdirectory(samples/cpp/prompt_lookup_decoding_lm/)
+add_subdirectory(samples/cpp/speculative_decoding_lm/)
+
+add_subdirectory(text_generation/causal_lm/cpp/continuous_batching)
+
+install(DIRECTORY
+        ./samples/cpp/beam_search_causal_lm
+        ./samples/cpp/chat_sample
+        ./samples/cpp/greedy_causal_lm
+        ./samples/cpp/multinomial_causal_lm
+        # Don't install prompt_lookup_decoding_lm and speculative_decoding_lm because they don't use openvino_genai library and arent verifyed yet.
+    DESTINATION samples/cpp/ COMPONENT cpp_samples_genai)
+install(FILES ./samples/cpp/requirements.txt DESTINATION samples/cpp/ COMPONENT cpp_samples_genai)
+install(FILES LICENSE DESTINATION licensing COMPONENT licensing_genai RENAME LICENSE-GENAI)
+install(FILES third-party-programs.txt DESTINATION licensing COMPONENT licensing_genai RENAME third-party-programs-genai.txt)
+if(MSVC AND NOT DEFINED CPACK_GENERATOR)
+    set(CPACK_GENERATOR "ZIP")
+endif()
+include(CPack)

image_generation/lcm_dreamshaper_v7/cpp/README.md

@@ -18,7 +18,7 @@ Prepare a python environment and install dependencies:
 conda create -n openvino_lcm_cpp python==3.10
 conda activate openvino_lcm_cpp
 conda update -c conda-forge --all
-conda install -c conda-forge openvino=2024.1.0 c-compiler cxx-compiler git make cmake
+conda install -c conda-forge openvino=2024.2.0 c-compiler cxx-compiler git make cmake
 # Ensure that Conda standard libraries are used
 conda env config vars set LD_LIBRARY_PATH=$CONDA_PREFIX/lib:$LD_LIBRARY_PATH
 ```

image_generation/lcm_dreamshaper_v7/cpp/requirements.txt

@@ -1,4 +1,4 @@
 --extra-index-url https://download.pytorch.org/whl/cpu
 torch==2.2.2+cpu
 diffusers==0.27.2
-optimum-intel[openvino] @ git+https://github.com/huggingface/optimum-intel.git@fb1b35bef23242d65b2fb057c4a7ac78a7cfd4c3
+optimum-intel[openvino]==1.17.0