Made an option to build tokenizers

samples/cpp/beam_search_causal_lm/CMakeLists.txt

@@ -5,14 +5,16 @@ find_package(OpenVINOGenAI REQUIRED PATHS
     "${CMAKE_BINARY_DIR}"  # Reuse the package from the build.
     ${OpenVINO_DIR}  # GenAI may be installed alogside OpenVINO.
 )
+
 add_executable(beam_search_causal_lm beam_search_causal_lm.cpp)
 target_link_libraries(beam_search_causal_lm PRIVATE openvino::genai)
 set_target_properties(beam_search_causal_lm PROPERTIES
     COMPILE_PDB_NAME beam_search_causal_lm
     # Ensure out of box LC_RPATH on macOS with SIP
     INSTALL_RPATH_USE_LINK_PATH ON)
 target_compile_features(beam_search_causal_lm PRIVATE cxx_std_11)
+
 install(TARGETS beam_search_causal_lm
-    RUNTIME DESTINATION samples_bin/
-    COMPONENT samples_bin
-    EXCLUDE_FROM_ALL)
+        RUNTIME DESTINATION samples_bin/
+        COMPONENT samples_bin
+        EXCLUDE_FROM_ALL)

samples/cpp/benchmark_genai/CMakeLists.txt

@@ -25,6 +25,6 @@ set_target_properties(benchmark_genai PROPERTIES
     INSTALL_RPATH_USE_LINK_PATH ON)
 
 install(TARGETS benchmark_genai
-    RUNTIME DESTINATION samples_bin/
-    COMPONENT samples_bin
-    EXCLUDE_FROM_ALL)
+        RUNTIME DESTINATION samples_bin/
+        COMPONENT samples_bin
+        EXCLUDE_FROM_ALL)

samples/cpp/chat_sample/CMakeLists.txt

@@ -5,14 +5,16 @@ find_package(OpenVINOGenAI REQUIRED PATHS
     "${CMAKE_BINARY_DIR}"  # Reuse the package from the build.
     ${OpenVINO_DIR}  # GenAI may be installed alogside OpenVINO.
 )
+
 add_executable(chat_sample chat_sample.cpp)
 target_link_libraries(chat_sample PRIVATE openvino::genai)
 set_target_properties(chat_sample PROPERTIES
     COMPILE_PDB_NAME chat_sample
     # Ensure out of box LC_RPATH on macOS with SIP
     INSTALL_RPATH_USE_LINK_PATH ON)
 target_compile_features(chat_sample PRIVATE cxx_std_11)
+
 install(TARGETS chat_sample
-    RUNTIME DESTINATION samples_bin/
-    COMPONENT samples_bin
-    EXCLUDE_FROM_ALL)
+        RUNTIME DESTINATION samples_bin/
+        COMPONENT samples_bin
+        EXCLUDE_FROM_ALL)

samples/cpp/greedy_causal_lm/CMakeLists.txt

@@ -5,14 +5,16 @@ find_package(OpenVINOGenAI REQUIRED PATHS
     "${CMAKE_BINARY_DIR}"  # Reuse the package from the build.
     ${OpenVINO_DIR}  # GenAI may be installed alogside OpenVINO.
 )
+
 add_executable(greedy_causal_lm greedy_causal_lm.cpp)
 target_link_libraries(greedy_causal_lm PRIVATE openvino::genai)
 set_target_properties(greedy_causal_lm PROPERTIES
     COMPILE_PDB_NAME greedy_causal_lm
     # Ensure out of box LC_RPATH on macOS with SIP
     INSTALL_RPATH_USE_LINK_PATH ON)
 target_compile_features(greedy_causal_lm PRIVATE cxx_std_11)
+
 install(TARGETS greedy_causal_lm
-    RUNTIME DESTINATION samples_bin/
-    COMPONENT samples_bin
-    EXCLUDE_FROM_ALL)
+        RUNTIME DESTINATION samples_bin/
+        COMPONENT samples_bin
+        EXCLUDE_FROM_ALL)

samples/cpp/multinomial_causal_lm/CMakeLists.txt

@@ -5,14 +5,16 @@ find_package(OpenVINOGenAI REQUIRED PATHS
     "${CMAKE_BINARY_DIR}"  # Reuse the package from the build.
     ${OpenVINO_DIR}  # GenAI may be installed alogside OpenVINO.
 )
+
 add_executable(multinomial_causal_lm multinomial_causal_lm.cpp)
 target_link_libraries(multinomial_causal_lm PRIVATE openvino::genai)
 set_target_properties(multinomial_causal_lm PROPERTIES
     COMPILE_PDB_NAME multinomial_causal_lm
     # Ensure out of box LC_RPATH on macOS with SIP
     INSTALL_RPATH_USE_LINK_PATH ON)
 target_compile_features(multinomial_causal_lm PRIVATE cxx_std_11)
+
 install(TARGETS multinomial_causal_lm
-    RUNTIME DESTINATION samples_bin/
-    COMPONENT samples_bin
-    EXCLUDE_FROM_ALL)
+        RUNTIME DESTINATION samples_bin/
+        COMPONENT samples_bin
+        EXCLUDE_FROM_ALL)

samples/cpp/prompt_lookup_decoding_lm/CMakeLists.txt

@@ -1,23 +1,28 @@
 # Copyright (C) 2023-2024 Intel Corporation
 # SPDX-License-Identifier: Apache-2.0
 
-if(TARGET openvino_tokenizers)
-    set(OPENVINO_TOKENIZERS_PATH $<TARGET_FILE:openvino_tokenizers>)
-else()
-    message(FATAL_ERROR "multinomial_causal_lm must be compiled as part of OpenVIINOGenAI to have the path to openvino_tokenizers hardcoded.")
-endif()
-find_package(OpenVINO REQUIRED COMPONENTS Runtime)
+find_package(OpenVINOGenAI REQUIRED PATHS
+    "${CMAKE_BINARY_DIR}"  # Reuse the package from the build.
+    ${OpenVINO_DIR}  # GenAI may be installed alogside OpenVINO.
+)
+
 find_package(TBB REQUIRED COMPONENTS tbb)
+
 add_executable(prompt_lookup_decoding_lm prompt_lookup_decoding_lm.cpp)
 target_link_libraries(prompt_lookup_decoding_lm PRIVATE openvino::runtime TBB::tbb)
 set_target_properties(prompt_lookup_decoding_lm PROPERTIES
     COMPILE_PDB_NAME prompt_lookup_decoding_lm
     # Ensure out of box LC_RPATH on macOS with SIP
     INSTALL_RPATH_USE_LINK_PATH ON)
-target_compile_definitions(prompt_lookup_decoding_lm PRIVATE OPENVINO_TOKENIZERS_PATH="${OPENVINO_TOKENIZERS_PATH}")
 target_compile_features(prompt_lookup_decoding_lm PRIVATE cxx_std_17)
 
+get_target_property(genai_imported openvino::genai IMPORTED_LOCATION)
+set(OPENVINO_TOKENIZERS_PATH $<IF:$<BOOL:${genai_imported}>,${genai_imported},$<TARGET_FILE_DIR:openvino::genai>>)
+set(OPENVINO_TOKENIZERS_FILENAME "${CMAKE_SHARED_LIBRARY_PREFIX}openvino_tokenizer${CMAKE_SHARED_LIBRARY_SUFFIX}")
+target_compile_definitions(prompt_lookup_decoding_lm PRIVATE
+    OPENVINO_TOKENIZERS_PATH="${OPENVINO_TOKENIZERS_PATH}/${OPENVINO_TOKENIZERS_FILENAME}")
+
 install(TARGETS prompt_lookup_decoding_lm
-    RUNTIME DESTINATION samples_bin/
-    COMPONENT samples_bin
-    EXCLUDE_FROM_ALL)
+        RUNTIME DESTINATION samples_bin/
+        COMPONENT samples_bin
+        EXCLUDE_FROM_ALL)

samples/cpp/speculative_decoding_lm/CMakeLists.txt

@@ -1,23 +1,28 @@
 # Copyright (C) 2023-2024 Intel Corporation
 # SPDX-License-Identifier: Apache-2.0
 
-if(TARGET openvino_tokenizers)
-    set(OPENVINO_TOKENIZERS_PATH $<TARGET_FILE:openvino_tokenizers>)
-else()
-    message(FATAL_ERROR "multinomial_causal_lm must be compiled as part of OpenVIINOGenAI to have the path to openvino_tokenizers hardcoded.")
-endif()
-find_package(OpenVINO REQUIRED COMPONENTS Runtime)
+find_package(OpenVINOGenAI REQUIRED PATHS
+    "${CMAKE_BINARY_DIR}"  # Reuse the package from the build.
+    ${OpenVINO_DIR}  # GenAI may be installed alogside OpenVINO.
+)
+
 find_package(TBB REQUIRED COMPONENTS tbb)
+
 add_executable(speculative_decoding_lm speculative_decoding_lm.cpp)
 target_link_libraries(speculative_decoding_lm PRIVATE openvino::runtime TBB::tbb)
 set_target_properties(speculative_decoding_lm PROPERTIES
     COMPILE_PDB_NAME speculative_decoding_lm
     # Ensure out of box LC_RPATH on macOS with SIP
     INSTALL_RPATH_USE_LINK_PATH ON)
-target_compile_definitions(speculative_decoding_lm PRIVATE OPENVINO_TOKENIZERS_PATH="${OPENVINO_TOKENIZERS_PATH}")
 target_compile_features(speculative_decoding_lm PRIVATE cxx_std_17)
 
+get_target_property(genai_imported openvino::genai IMPORTED_LOCATION)
+set(OPENVINO_TOKENIZERS_PATH $<IF:$<BOOL:${genai_imported}>,${genai_imported},$<TARGET_FILE_DIR:openvino::genai>>)
+set(OPENVINO_TOKENIZERS_FILENAME "${CMAKE_SHARED_LIBRARY_PREFIX}openvino_tokenizer${CMAKE_SHARED_LIBRARY_SUFFIX}")
+target_compile_definitions(speculative_decoding_lm PRIVATE
+    OPENVINO_TOKENIZERS_PATH="${OPENVINO_TOKENIZERS_PATH}/${OPENVINO_TOKENIZERS_FILENAME}")
+
 install(TARGETS speculative_decoding_lm
-    RUNTIME DESTINATION samples_bin/
-    COMPONENT samples_bin
-    EXCLUDE_FROM_ALL)
+        RUNTIME DESTINATION samples_bin/
+        COMPONENT samples_bin
+        EXCLUDE_FROM_ALL)

src/cpp/src/tokenizers_path.cpp

@@ -59,10 +59,12 @@ std::string get_ov_genai_library_path() {
 std::filesystem::path with_openvino_tokenizers(const std::filesystem::path& path) {
 #ifdef _WIN32
     constexpr char tokenizers[] = "openvino_tokenizers.dll";
-#elif __linux__
+#elif defined(__linux__)
     constexpr char tokenizers[] = "libopenvino_tokenizers.so";
-#elif __APPLE__
+#elif defined(__APPLE__)
     constexpr char tokenizers[] = "libopenvino_tokenizers.dylib";
+#else
+#    error "Unsupported OS"
 #endif
     return path.parent_path() / tokenizers;
 }

thirdparty/CMakeLists.txt

@@ -2,33 +2,37 @@
 # SPDX-License-Identifier: Apache-2.0
 #
 
-add_subdirectory(./openvino_tokenizers/ "${CMAKE_BINARY_DIR}/openvino_tokenizers/")
-# Put binaries to a single dir to mimic package structure.
-set_target_properties(openvino_tokenizers PROPERTIES
-    # Generator expressions to disable appending a per-configuration subdirectory (Release, Debug).
-    # ARCHIVE_OUTPUT is irrelevant. It's here just to keep all the artifacts in one place.
-    ARCHIVE_OUTPUT_DIRECTORY "$<1:${CMAKE_BINARY_DIR}/openvino_genai/>"
-    LIBRARY_OUTPUT_DIRECTORY "$<1:${CMAKE_BINARY_DIR}/openvino_genai/>"
-    RUNTIME_OUTPUT_DIRECTORY "$<1:${CMAKE_BINARY_DIR}/openvino_genai/>"
-)
-if(TARGET core_tokenizers)
-    set_target_properties(core_tokenizers PROPERTIES
+option(BUILD_TOKENIZERS "Build OpenVINO Tokenizers together with OpenVINO GenAI" ON)
+
+if(BUILD_TOKENIZERS)
+    add_subdirectory(./openvino_tokenizers/ "${CMAKE_BINARY_DIR}/openvino_tokenizers/")
+    # Put binaries to a single dir to mimic package structure.
+    set_target_properties(openvino_tokenizers PROPERTIES
+        # Generator expressions to disable appending a per-configuration subdirectory (Release, Debug).
+        # ARCHIVE_OUTPUT is irrelevant. It's here just to keep all the artifacts in one place.
         ARCHIVE_OUTPUT_DIRECTORY "$<1:${CMAKE_BINARY_DIR}/openvino_genai/>"
         LIBRARY_OUTPUT_DIRECTORY "$<1:${CMAKE_BINARY_DIR}/openvino_genai/>"
         RUNTIME_OUTPUT_DIRECTORY "$<1:${CMAKE_BINARY_DIR}/openvino_genai/>"
     )
-else()
-    # Prebuilt dependencies
-    if(WIN32)
-        set(extra_libs "${CMAKE_BINARY_DIR}/_deps/fast_tokenizer-src/lib/core_tokenizers.dll"
-                       "${CMAKE_BINARY_DIR}/_deps/fast_tokenizer-src/third_party/lib/icudt70.dll"
-                       "${CMAKE_BINARY_DIR}/_deps/fast_tokenizer-src/third_party/lib/icuuc70.dll")
-    elseif(LINUX)
-        set(extra_libs "${CMAKE_BINARY_DIR}/_deps/fast_tokenizer-src/lib/libcore_tokenizers.so")
-    elseif(APPLE)
-        set(extra_libs "${CMAKE_BINARY_DIR}/_deps/fast_tokenizer-src/lib/libcore_tokenizers.dylib")
+    if(TARGET core_tokenizers)
+        set_target_properties(core_tokenizers PROPERTIES
+            ARCHIVE_OUTPUT_DIRECTORY "$<1:${CMAKE_BINARY_DIR}/openvino_genai/>"
+            LIBRARY_OUTPUT_DIRECTORY "$<1:${CMAKE_BINARY_DIR}/openvino_genai/>"
+            RUNTIME_OUTPUT_DIRECTORY "$<1:${CMAKE_BINARY_DIR}/openvino_genai/>"
+        )
+    else()
+        # Prebuilt dependencies
+        if(WIN32)
+            set(extra_libs "${CMAKE_BINARY_DIR}/_deps/fast_tokenizer-src/lib/core_tokenizers.dll"
+                        "${CMAKE_BINARY_DIR}/_deps/fast_tokenizer-src/third_party/lib/icudt70.dll"
+                        "${CMAKE_BINARY_DIR}/_deps/fast_tokenizer-src/third_party/lib/icuuc70.dll")
+        elseif(LINUX)
+            set(extra_libs "${CMAKE_BINARY_DIR}/_deps/fast_tokenizer-src/lib/libcore_tokenizers.so")
+        elseif(APPLE)
+            set(extra_libs "${CMAKE_BINARY_DIR}/_deps/fast_tokenizer-src/lib/libcore_tokenizers.dylib")
+        endif()
+        add_custom_command(OUTPUT "${extra_libs}"
+            COMMAND "${CMAKE_COMMAND}" -E copy "${extra_libs}" "${CMAKE_BINARY_DIR}/openvino_genai/"
+            DEPENDS openvino_tokenizers)
     endif()
-    add_custom_command(OUTPUT "${extra_libs}"
-        COMMAND "${CMAKE_COMMAND}" -E copy "${extra_libs}" "${CMAKE_BINARY_DIR}/openvino_genai/"
-        DEPENDS openvino_tokenizers)
 endif()