From 043b9df95e5dfc5e80ccbdf52926afaa0991664b Mon Sep 17 00:00:00 2001 From: fmz Date: Tue, 23 Jul 2024 06:57:43 -0700 Subject: [PATCH] Minor fixes --- CMakePresets.json | 256 +++++++-------------------- examples/CMakeLists.txt | 2 +- examples/speculative/speculative.cpp | 6 +- ggml/src/ggml.c | 4 +- include/llama.h | 2 + src/llama.cpp | 9 + 6 files changed, 79 insertions(+), 200 deletions(-) diff --git a/CMakePresets.json b/CMakePresets.json index ae2bf25c12786d..bdad38952d3cbe 100644 --- a/CMakePresets.json +++ b/CMakePresets.json @@ -1,197 +1,65 @@ { - "version": 4, - "configurePresets": [ - { - "name": "base", - "hidden": true, - "generator": "Ninja", - "binaryDir": "${sourceDir}/build-${presetName}", - "cacheVariables": { - "CMAKE_EXPORT_COMPILE_COMMANDS": "ON", - "CMAKE_INSTALL_RPATH": "$ORIGIN;$ORIGIN/.." - } - }, - { - "name": "sycl-base", - "hidden": true, - "generator": "Ninja", - "binaryDir": "${sourceDir}/build-${presetName}", - "cacheVariables": { - "CMAKE_EXPORT_COMPILE_COMMANDS": "ON", - "CMAKE_CXX_COMPILER": "icx", - "CMAKE_C_COMPILER": "cl", - "GGML_SYCL": "ON", - "CMAKE_INSTALL_RPATH": "$ORIGIN;$ORIGIN/.." - } - }, - { - "name": "debug", - "hidden": true, - "cacheVariables": { - "CMAKE_BUILD_TYPE": "Debug" - } - }, - { - "name": "release", - "hidden": true, - "cacheVariables": { - "CMAKE_BUILD_TYPE": "Release" - } - }, - { - "name": "reldbg", - "hidden": true, - "cacheVariables": { - "CMAKE_BUILD_TYPE": "RelWithDebInfo" - } - }, - { - "name": "static", - "hidden": true, - "cacheVariables": { - "GGML_STATIC": "ON" - } - }, - { - "name": "arm64-windows-msvc", - "hidden": true, - "architecture": { - "value": "arm64", - "strategy": "external" - }, - "toolset": { - "value": "host=x86_64", - "strategy": "external" - }, - "cacheVariables": { - "CMAKE_TOOLCHAIN_FILE": "${sourceDir}/cmake/arm64-windows-msvc.cmake" - } - }, - { - "name": "arm64-windows-llvm", - "hidden": true, - "architecture": { - "value": "arm64", - "strategy": "external" - }, - "toolset": { - "value": "host=x86_64", - "strategy": "external" - }, - "cacheVariables": { - "CMAKE_TOOLCHAIN_FILE": "${sourceDir}/cmake/arm64-windows-llvm.cmake" - } - }, - { - "name": "arm64-windows-llvm-debug", - "inherits": [ - "base", - "arm64-windows-llvm", - "debug" - ] - }, - { - "name": "arm64-windows-llvm-release", - "inherits": [ - "base", - "arm64-windows-llvm", - "reldbg" - ] - }, - { - "name": "arm64-windows-llvm+static-release", - "inherits": [ - "base", - "arm64-windows-llvm", - "reldbg", - "static" - ] - }, - { - "name": "arm64-windows-msvc-debug", - "inherits": [ - "base", - "arm64-windows-msvc", - "debug" - ] - }, - { - "name": "arm64-windows-msvc-release", - "inherits": [ - "base", - "arm64-windows-msvc", - "reldbg" - ] - }, - { - "name": "arm64-windows-msvc+static-release", - "inherits": [ - "base", - "arm64-windows-msvc", - "reldbg", - "static" - ] - }, - { - "name": "x64-windows-msvc-debug", - "inherits": [ - "base", - "debug" - ] - }, - { - "name": "x64-windows-msvc-release", - "inherits": [ - "base", - "reldbg" - ] - }, - { - "name": "x64-windows-msvc+static-release", - "inherits": [ - "base", - "reldbg", - "static" - ] - }, - { - "name": "x64-windows-sycl-debug", - "inherits": [ - "sycl-base", - "debug" - ] - }, - { - "name": "x64-windows-sycl-release", - "inherits": [ - "sycl-base", - "release" - ] - }, - { - "name": "clang10", - "displayName": "Clang 10.0.0 x86_64-pc-linux-gnu", - "description": "Using compilers: C = /usr/bin/clang, CXX = /usr/bin/clang++", - "binaryDir": "${sourceDir}/out/build/${presetName}", - "cacheVariables": { - "CMAKE_INSTALL_PREFIX": "${sourceDir}/out/install/${presetName}", - "CMAKE_C_COMPILER": "/usr/bin/clang", - "CMAKE_CXX_COMPILER": "/usr/bin/clang++", - "CMAKE_RC_COMPILER": "/usr/bin/llvm-rc-10", - "CMAKE_BUILD_TYPE": "Debug" - } - }, - { - "name": "gcc8.4", - "displayName": "GCC 8.4.0 x86_64-linux-gnu", - "description": "Using compilers: C = /usr/bin/gcc, CXX = /usr/bin/g++", - "binaryDir": "${sourceDir}/out/build/${presetName}", - "cacheVariables": { - "CMAKE_INSTALL_PREFIX": "${sourceDir}/out/install/${presetName}", - "CMAKE_C_COMPILER": "/usr/bin/gcc", - "CMAKE_CXX_COMPILER": "/usr/bin/g++", - "CMAKE_BUILD_TYPE": "Debug" - } + "version": 4, + "configurePresets": [ + { + "name": "base", + "hidden": true, + "generator": "Ninja", + "binaryDir": "${sourceDir}/build-${presetName}", + "cacheVariables": { + "CMAKE_EXPORT_COMPILE_COMMANDS": "ON", + "CMAKE_INSTALL_RPATH": "$ORIGIN;$ORIGIN/.." } - ] + }, + { + "name": "sycl-base", + "hidden": true, + "generator": "Ninja", + "binaryDir": "${sourceDir}/build-${presetName}", + "cacheVariables": { + "CMAKE_EXPORT_COMPILE_COMMANDS": "ON", + "CMAKE_CXX_COMPILER": "icx", + "CMAKE_C_COMPILER": "cl", + "GGML_SYCL": "ON", + "CMAKE_INSTALL_RPATH": "$ORIGIN;$ORIGIN/.." + } + }, + { "name": "debug", "hidden": true, "cacheVariables": { "CMAKE_BUILD_TYPE": "Debug" } }, + { "name": "release", "hidden": true, "cacheVariables": { "CMAKE_BUILD_TYPE": "Release" } }, + { "name": "reldbg", "hidden": true, "cacheVariables": { "CMAKE_BUILD_TYPE": "RelWithDebInfo" } }, + { "name": "static", "hidden": true, "cacheVariables": { "GGML_STATIC": "ON" } }, + + { + "name": "arm64-windows-msvc", "hidden": true, + "architecture": { "value": "arm64", "strategy": "external" }, + "toolset": { "value": "host=x86_64", "strategy": "external" }, + "cacheVariables": { + "CMAKE_TOOLCHAIN_FILE": "${sourceDir}/cmake/arm64-windows-msvc.cmake" + } + }, + + { + "name": "arm64-windows-llvm", "hidden": true, + "architecture": { "value": "arm64", "strategy": "external" }, + "toolset": { "value": "host=x86_64", "strategy": "external" }, + "cacheVariables": { + "CMAKE_TOOLCHAIN_FILE": "${sourceDir}/cmake/arm64-windows-llvm.cmake" + } + }, + + { "name": "arm64-windows-llvm-debug" , "inherits": [ "base", "arm64-windows-llvm", "debug" ] }, + { "name": "arm64-windows-llvm-release", "inherits": [ "base", "arm64-windows-llvm", "reldbg" ] }, + { "name": "arm64-windows-llvm+static-release", "inherits": [ "base", "arm64-windows-llvm", "reldbg", "static" ] }, + + { "name": "arm64-windows-msvc-debug" , "inherits": [ "base", "arm64-windows-msvc", "debug" ] }, + { "name": "arm64-windows-msvc-release", "inherits": [ "base", "arm64-windows-msvc", "reldbg" ] }, + { "name": "arm64-windows-msvc+static-release", "inherits": [ "base", "arm64-windows-msvc", "reldbg", "static" ] }, + + { "name": "x64-windows-msvc-debug" , "inherits": [ "base", "debug" ] }, + { "name": "x64-windows-msvc-release", "inherits": [ "base", "reldbg" ] }, + { "name": "x64-windows-msvc+static-release", "inherits": [ "base", "reldbg", "static" ] }, + + { "name": "x64-windows-sycl-debug" , "inherits": [ "sycl-base", "debug" ] }, + { "name": "x64-windows-sycl-release", "inherits": [ "sycl-base", "release" ] } + ] } diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt index 335bc5b99cc90d..155743639adfda 100644 --- a/examples/CMakeLists.txt +++ b/examples/CMakeLists.txt @@ -51,7 +51,7 @@ else() endif() add_subdirectory(save-load-state) add_subdirectory(simple) - #add_subdirectory(speculative) + add_subdirectory(speculative) add_subdirectory(tokenize) add_subdirectory(train-text-from-scratch) endif() diff --git a/examples/speculative/speculative.cpp b/examples/speculative/speculative.cpp index 0939a1a6a7a384..a403c58b5ff6d5 100644 --- a/examples/speculative/speculative.cpp +++ b/examples/speculative/speculative.cpp @@ -71,10 +71,10 @@ int main(int argc, char ** argv) { // load the draft model params.model = params.model_draft; params.n_gpu_layers = params.n_gpu_layers_draft; - if (params.n_threads_draft > 0) { - params.n_threads = params.n_threads_draft; + if (params.draft_cpuparams.n_threads > 0) { + params.cpuparams.n_threads = params.draft_cpuparams.n_threads; } - params.n_threads_batch = params.n_threads_batch_draft; + params.cpuparams_batch.n_threads = params.draft_cpuparams_batch.n_threads; std::tie(model_dft, ctx_dft) = llama_init_from_gpt_params(params); const bool vocab_type_tgt = llama_vocab_type(model_tgt); diff --git a/ggml/src/ggml.c b/ggml/src/ggml.c index dd8163ae97d0cd..76a4fb26032675 100644 --- a/ggml/src/ggml.c +++ b/ggml/src/ggml.c @@ -18751,7 +18751,7 @@ static bool __thread_affinity(const bool * mask) { for (uint32_t i = 0; i < GGML_MAX_N_THREADS; i++) { if (mask[i]) { - printf("Thread %lx: adding %d to cpuset\n", pthread_self(), i); + GGML_PRINT_DEBUG("Thread %lx: adding %d to cpuset\n", pthread_self(), i); CPU_SET(i, &cpuset); } } @@ -19144,7 +19144,7 @@ static bool ggml_graph_compute_check_for_work(struct ggml_compute_state * state) if (threadpool->poll) { while (!threadpool->new_work && !threadpool->stop && !threadpool->pause) { // No new work. Yield and keep polling. - //__cpu_relax(); + __cpu_relax(); } } else { ggml_mutex_lock_shared(&threadpool->mutex); diff --git a/include/llama.h b/include/llama.h index 9a2ccb1710f9a5..88b56e6a15ac29 100644 --- a/include/llama.h +++ b/include/llama.h @@ -441,6 +441,8 @@ extern "C" { LLAMA_API void llama_detach_batch_threadpool(struct llama_context * ctx); LLAMA_API void llama_detach_threadpools(struct llama_context * ctx); + // Pauses all attached threadpools + LLAMA_API void llama_pause_threadpools(struct llama_context * ctx); // Call once at the end of the program - currently only used for MPI LLAMA_API void llama_backend_free(void); diff --git a/src/llama.cpp b/src/llama.cpp index eaf969350d9970..9ec0073d133752 100644 --- a/src/llama.cpp +++ b/src/llama.cpp @@ -16416,6 +16416,15 @@ void llama_detach_threadpools(struct llama_context * ctx) { llama_detach_batch_threadpool(ctx); } +void llama_pause_threadpools(struct llama_context * ctx) { + if (ctx->threadpool) { + ggml_pause_threadpool(ctx->threadpool); + } + if (ctx->threadpool_batch) { + ggml_pause_threadpool(ctx->threadpool_batch); + } +} + void llama_backend_free(void) { ggml_quantize_free(); }